IQ.Pilot Release Commit @ 661a2de

tools/replay/.gitignore

@@ -3,3 +3,4 @@ moc_*
 
 replay
 tests/test_replay
+tests/test_api

tools/replay/SConscript

@@ -22,3 +22,4 @@ replay_env.Program("replay", ["main.cc"], LIBS=replay_libs, FRAMEWORKS=base_fram
 
 if GetOption('extras'):
   replay_env.Program('tests/test_replay', ['tests/test_replay.cc'], LIBS=replay_libs)
+  replay_env.Program('tests/test_api', ['tests/test_runner.cc', 'tests/test_api.cc'], LIBS=replay_libs)

tools/replay/api.cc

@@ -160,4 +160,54 @@ std::string httpGet(const std::string &url, long *response_code) {
   return res == CURLE_OK ? readBuffer : std::string{};
 }
 
+std::string apiResponse(const std::string &body, long response_code) {
+  if (response_code == 401 || response_code == 403) {
+    return R"({"error": "unauthorized"})";
+  }
+  if (response_code == 404) {
+    return R"({"error": "not_found"})";
+  }
+  if (body.empty() || response_code < 200 || response_code >= 300) {
+    return R"({"error": "network"})";
+  }
+  return body;
+}
+
+std::string routeFilesPath(const std::string &route) {
+  return "v1/route/" + route + "/files";
+}
+
+std::string devicesPath() {
+  return "v1/me/devices/";
+}
+
+std::string deviceRoutesPath(const std::string &dongle_id, int64_t start_ms, int64_t end_ms, bool preserved) {
+  if (preserved) {
+    return "v1/devices/" + dongle_id + "/routes/preserved";
+  }
+
+  std::string query;
+  if (start_ms > 0) query += "start=" + std::to_string(start_ms);
+  if (end_ms > 0) query += (query.empty() ? "" : "&") + ("end=" + std::to_string(end_ms));
+  return "v1/devices/" + dongle_id + "/routes_segments" + (query.empty() ? "" : "?" + query);
+}
+
+static std::string apiCall(const std::string &path) {
+  long response_code = 0;
+  const std::string body = httpGet(BASE_URL + "/" + path, &response_code);
+  return apiResponse(body, response_code);
+}
+
+std::string getRouteFiles(const std::string &route) {
+  return apiCall(routeFilesPath(route));
+}
+
+std::string getDevices() {
+  return apiCall(devicesPath());
+}
+
+std::string getDeviceRoutes(const std::string &dongle_id, int64_t start_ms, int64_t end_ms, bool preserved) {
+  return apiCall(deviceRoutesPath(dongle_id, start_ms, end_ms, preserved));
+}
+
 }  // namespace CommaApi

tools/replay/api.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <curl/curl.h>
+#include <cstdint>
 #include <string>
 
 #include "common/util.h"
@@ -12,4 +13,21 @@ const std::string BASE_URL = util::getenv("API_HOST", "https://api-iqlabs.konn3k
 std::string create_token(bool use_jwt, const json11::Json& payloads = {}, int expiry = 3600);
 std::string httpGet(const std::string &url, long *response_code = nullptr);
 
+// konn3kt equivalents of upstream's PyDownloader API helpers. Same endpoints and
+// same response shapes, but served over the C++/libcurl path instead of shelling
+// into tools/lib. On failure they return {"error": "<code>"} so callers can tell
+// unauthorized from a transport error without a second out-param.
+std::string getRouteFiles(const std::string &route);
+std::string getDevices();
+std::string getDeviceRoutes(const std::string &dongle_id, int64_t start_ms = 0, int64_t end_ms = 0, bool preserved = false);
+
+// Endpoint paths, relative to BASE_URL. Split out from the fetchers so the konn3kt
+// routing can be asserted without network access.
+std::string routeFilesPath(const std::string &route);
+std::string devicesPath();
+std::string deviceRoutesPath(const std::string &dongle_id, int64_t start_ms, int64_t end_ms, bool preserved);
+
+// Maps an HTTP status + body to either the body or an {"error": ...} envelope.
+std::string apiResponse(const std::string &body, long response_code);
+
 }  // namespace CommaApi2

tools/replay/framereader.cc

@@ -6,7 +6,7 @@
 #include <utility>
 
 #include "common/util.h"
-#include "third_party/libyuv/include/libyuv.h"
+#include "common/yuv.h"
 #include "tools/replay/util.h"
 #include "system/hardware/hw.h"
 
@@ -257,12 +257,12 @@ bool FFmpegVideoDecoder::copyBuffer(AVFrame *f, VisionBuf *buf) {
       memcpy(buf->uv + i*buf->stride, f->data[1] + i*f->linesize[1], width);
     }
   } else {
-    libyuv::I420ToNV12(f->data[0], f->linesize[0],
-                       f->data[1], f->linesize[1],
-                       f->data[2], f->linesize[2],
-                       buf->y, buf->stride,
-                       buf->uv, buf->stride,
-                       width, height);
+    yuv::i420_to_nv12(f->data[0], f->linesize[0],
+                      f->data[1], f->linesize[1],
+                      f->data[2], f->linesize[2],
+                      buf->y, buf->stride,
+                      buf->uv, buf->stride,
+                      width, height);
   }
   return true;
 }

tools/replay/logreader.cc

@@ -1,19 +1,31 @@
 #include "tools/replay/logreader.h"
 
 #include <algorithm>
+#include <chrono>
 #include <utility>
 #include "tools/replay/filereader.h"
 #include "tools/replay/util.h"
 #include "common/util.h"
 
 bool LogReader::load(const std::string &url, std::atomic<bool> *abort, bool local_cache, int chunk_size, int retries) {
+  using Clock = std::chrono::steady_clock;
+  compressed_size_ = decompressed_size_ = 0;
+  download_seconds_ = decompress_seconds_ = parse_seconds_ = 0.0;
+
+  const auto download_start = Clock::now();
   std::string data = FileReader(local_cache, chunk_size, retries).read(url, abort);
+  download_seconds_ = std::chrono::duration<double>(Clock::now() - download_start).count();
+  compressed_size_ = decompressed_size_ = data.size();
+
   if (!data.empty()) {
+    const auto decompress_start = Clock::now();
     if (url.find(".bz2") != std::string::npos || util::starts_with(data, "BZh9")) {
       data = decompressBZ2(data, abort);
     } else if (url.find(".zst") != std::string::npos || util::starts_with(data, "\x28\xB5\x2F\xFD")) {
       data = decompressZST(data, abort);
     }
+    decompress_seconds_ = std::chrono::duration<double>(Clock::now() - decompress_start).count();
+    decompressed_size_ = data.size();
   }
 
   bool success = !data.empty() && load(data.data(), data.size(), abort);
@@ -23,6 +35,8 @@ bool LogReader::load(const std::string &url, std::atomic<bool> *abort, bool loca
 }
 
 bool LogReader::load(const char *data, size_t size, std::atomic<bool> *abort) {
+  using Clock = std::chrono::steady_clock;
+  const auto parse_start = Clock::now();
   try {
     events.reserve(65000);
     kj::ArrayPtr<const capnp::word> words((const capnp::word *)data, size / sizeof(capnp::word));
@@ -65,6 +79,8 @@ bool LogReader::load(const char *data, size_t size, std::atomic<bool> *abort) {
     migrateOldEvents();
   }
 
+  parse_seconds_ = std::chrono::duration<double>(Clock::now() - parse_start).count();
+
   if (!events.empty() && !(abort && *abort)) {
     events.shrink_to_fit();
     std::sort(events.begin(), events.end());

tools/replay/logreader.h

@@ -32,6 +32,15 @@ public:
   bool load(const char *data, size_t size, std::atomic<bool> *abort = nullptr);
   std::vector<Event> events;
 
+  // Per-load instrumentation, consumed by jotpluggler's route load stats panel.
+  // Unlike upstream these are measured around iqpilot's own FileReader/decompress
+  // steps rather than around a Python downloader, so decompress is timed separately.
+  uint64_t compressed_size() const { return compressed_size_; }
+  uint64_t decompressed_size() const { return decompressed_size_; }
+  double download_seconds() const { return download_seconds_; }
+  double decompress_seconds() const { return decompress_seconds_; }
+  double parse_seconds() const { return parse_seconds_; }
+
 private:
   void migrateOldEvents();
 
@@ -39,4 +48,9 @@ private:
   bool requires_migration = true;
   std::vector<bool> filters_;
   MonotonicBuffer buffer_{1024 * 1024};
+  uint64_t compressed_size_ = 0;
+  uint64_t decompressed_size_ = 0;
+  double download_seconds_ = 0.0;
+  double decompress_seconds_ = 0.0;
+  double parse_seconds_ = 0.0;
 };

tools/replay/tests/test_api.cc

@@ -0,0 +1,72 @@
+#include "catch2/catch.hpp"
+
+#include "tools/replay/api.h"
+
+// These cover the konn3kt endpoints that replaced upstream's PyDownloader shell-outs.
+// Cabana's remote-routes dialog and jotpluggler's route-file listing both route through
+// here, so a silent change to a path or to the error envelope breaks konn3kt without
+// breaking the build.
+
+TEST_CASE("konn3kt api paths") {
+  SECTION("route files") {
+    REQUIRE(CommaApi2::routeFilesPath("a2a0ccea32023010|2023-07-27--13-01-19") ==
+            "v1/route/a2a0ccea32023010|2023-07-27--13-01-19/files");
+  }
+
+  SECTION("devices") {
+    REQUIRE(CommaApi2::devicesPath() == "v1/me/devices/");
+  }
+
+  SECTION("device routes over a time range") {
+    REQUIRE(CommaApi2::deviceRoutesPath("dongle", 1000, 2000, false) ==
+            "v1/devices/dongle/routes_segments?start=1000&end=2000");
+  }
+
+  SECTION("device routes with only one bound") {
+    REQUIRE(CommaApi2::deviceRoutesPath("dongle", 1000, 0, false) ==
+            "v1/devices/dongle/routes_segments?start=1000");
+    REQUIRE(CommaApi2::deviceRoutesPath("dongle", 0, 2000, false) ==
+            "v1/devices/dongle/routes_segments?end=2000");
+  }
+
+  SECTION("device routes with no bounds omits the query entirely") {
+    REQUIRE(CommaApi2::deviceRoutesPath("dongle", 0, 0, false) ==
+            "v1/devices/dongle/routes_segments");
+  }
+
+  SECTION("preserved routes ignore the time range") {
+    REQUIRE(CommaApi2::deviceRoutesPath("dongle", 1000, 2000, true) ==
+            "v1/devices/dongle/routes/preserved");
+  }
+}
+
+TEST_CASE("konn3kt api error envelope") {
+  // Cabana and jotpluggler both branch on these exact strings.
+  SECTION("success passes the body through") {
+    REQUIRE(CommaApi2::apiResponse("[{\"dongle_id\": \"abc\"}]", 200) == "[{\"dongle_id\": \"abc\"}]");
+  }
+
+  SECTION("401 and 403 both map to unauthorized") {
+    REQUIRE(CommaApi2::apiResponse("", 401) == R"({"error": "unauthorized"})");
+    REQUIRE(CommaApi2::apiResponse("nope", 403) == R"({"error": "unauthorized"})");
+  }
+
+  SECTION("404 maps to not_found") {
+    REQUIRE(CommaApi2::apiResponse("", 404) == R"({"error": "not_found"})");
+  }
+
+  SECTION("transport failure and 5xx map to network") {
+    REQUIRE(CommaApi2::apiResponse("", 0) == R"({"error": "network"})");
+    REQUIRE(CommaApi2::apiResponse("boom", 500) == R"({"error": "network"})");
+  }
+
+  SECTION("a 200 with an empty body is still an error") {
+    REQUIRE(CommaApi2::apiResponse("", 200) == R"({"error": "network"})");
+  }
+}
+
+TEST_CASE("konn3kt base url") {
+  // Everything is built off BASE_URL; an accidental revert to comma's host would
+  // silently point every tool at api.comma.ai.
+  REQUIRE(CommaApi2::BASE_URL.find("comma.ai") == std::string::npos);
+}

tools/replay/tests/test_runner.cc

@@ -0,0 +1,2 @@
+#define CATCH_CONFIG_MAIN
+#include "catch2/catch.hpp"