IQ.Pilot Release Commit @ d2ce8a8

iqpilot/tools/lib/url_file.py

@@ -21,6 +21,10 @@ CACHE_SIZE = 10 * 1024 * 1024 * 1024  # total cache size in GB
 logging.getLogger("urllib3").setLevel(logging.WARNING)
 
 
+USER_AGENT = os.getenv("IQPILOT_HTTP_USER_AGENT", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+                       "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36")
+
+
 def _env_int(name: str, default: int) -> int:
   raw = os.getenv(name)
   if raw is None:
@@ -113,6 +117,10 @@ class URLFile:
     pass
 
   def _request(self, method: str, url: str, headers: dict[str, str] | None = None) -> BaseHTTPResponse:
+    # the data host is behind cloudflare, which answers a default urllib3 agent
+    # with a 1010 block. It reads as 403 Forbidden on a correctly signed url, so
+    # it looks like an auth problem and is not one.
+    headers = {**(headers or {}), "User-Agent": USER_AGENT}
     try:
       return URLFile.pool_manager().request(method, url, timeout=self._timeout, headers=headers)
     except MaxRetryError as e:

iqpilot/tools/replay/camera.cc

@@ -70,6 +70,9 @@ void CameraServer::cameraThread(Camera &cam) {
           .timestamp_eof = eidx.getTimestampEof(),
       };
       vipc_server_->send(yuv, &extra);
+      if (++sent_count_ % 100 == 1) {
+        rInfo("camera[%d] vipc send #%lu frame_id=%u seg_frame=%d", cam.type, sent_count_, frame_id, segment_id);
+      }
     } else {
       rError("camera[%d] failed to get frame: %lu", cam.type, segment_id);
     }

iqpilot/tools/replay/camera.h

@@ -38,5 +38,6 @@ protected:
       {.type = WideRoadCam, .stream_type = VISION_STREAM_WIDE_ROAD},
   };
   std::atomic<int> publishing_ = 0;
+  uint64_t sent_count_ = 0;
   std::unique_ptr<VisionIpcServer> vipc_server_;
 };

iqpilot/tools/replay/framereader.cc

@@ -107,11 +107,21 @@ bool FrameReader::loadFromFile(CameraType type, const std::string &file, bool no
   packets_info.reserve(60 * 20);  // 20fps, one minute
   while (!(abort && *abort) && av_read_frame(input_ctx, &pkt) == 0) {
     if (pkt.stream_index == video_stream_idx_) {
-      packets_info.emplace_back(PacketInfo{.flags = pkt.flags, .pos = pkt.pos});
+      packets_info.emplace_back(PacketInfo{.flags = pkt.flags, .pos = pkt.pos, .ts = pkt.dts});
     }
     av_packet_unref(&pkt);
   }
-  avio_seek(input_ctx->pb, 0, SEEK_SET);
+  // IQ.Pilot camera files are (fragmented) MP4: rewinding the raw pb leaves the
+  // mov demuxer's sample cursor at EOF, so rewind through the demuxer instead.
+  // comma's raw HEVC bitstreams have no index; only the pb rewind works there.
+  if (!packets_info.empty() &&
+      avformat_seek_file(input_ctx, video_stream_idx_, INT64_MIN,
+                         packets_info.front().ts, packets_info.front().ts, 0) < 0) {
+    avformat_flush(input_ctx);
+    avio_seek(input_ctx->pb, 0, SEEK_SET);
+  }
+  rInfo("frame index built: %zu packets, fmt=%s", packets_info.size(),
+        input_ctx->iformat ? input_ctx->iformat->name : "?");
   return !packets_info.empty();
 }
 
@@ -147,6 +157,10 @@ bool FFmpegVideoDecoder::open(AVCodecParameters *codecpar, bool hw_decoder) {
   }
   width = (decoder_ctx->width + 3) & ~3;
   height = decoder_ctx->height;
+  // frame-threaded software decode: single-threaded can't hold 2x1928x1208@20
+  // on this SoC. The added output delay is absorbed by the EAGAIN-aware loop.
+  decoder_ctx->thread_count = 3;
+  decoder_ctx->thread_type = FF_THREAD_FRAME;
 
   if (hw_decoder && !initHardwareDecoder(HW_DEVICE_TYPE)) {
     rWarning("No device with hardware decoder found. fallback to CPU decoding.");
@@ -188,6 +202,14 @@ bool FFmpegVideoDecoder::initHardwareDecoder(AVHWDeviceType hw_device_type) {
 bool FFmpegVideoDecoder::decode(FrameReader *reader, int idx, VisionBuf *buf) {
   int current_idx = idx;
   if (idx != reader->prev_idx + 1) {
+    if (idx > reader->prev_idx && idx - reader->prev_idx <= 300) {
+      // forward catch-up: the decoder is already positioned at prev_idx+1, and
+      // sequential decode is cheaper and (for raw H.264) more reliable than a
+      // byte seek plus keyframe re-decode
+      current_idx = reader->prev_idx + 1;
+      reader->prev_idx = idx;
+      goto read_packets;
+    }
     // seeking to the nearest key frame
     for (int i = idx; i >= 0; --i) {
       if (reader->packets_info[i].flags & AV_PKT_FLAG_KEY) {
@@ -198,6 +220,12 @@ bool FFmpegVideoDecoder::decode(FrameReader *reader, int idx, VisionBuf *buf) {
 
     auto pos = reader->packets_info[current_idx].pos;
     int ret = avformat_seek_file(reader->input_ctx, 0, pos, pos, pos, AVSEEK_FLAG_BYTE);
+    if (ret < 0) {
+      // mp4 containers reject byte seeks; seek the keyframe by timestamp
+      // through the mov index instead
+      auto ts = reader->packets_info[current_idx].ts;
+      ret = avformat_seek_file(reader->input_ctx, reader->video_stream_idx_, INT64_MIN, ts, ts, 0);
+    }
     if (ret < 0) {
       rError("Failed to seek to byte position %lld: %d", pos, AVERROR(ret));
       return false;
@@ -206,26 +234,42 @@ bool FFmpegVideoDecoder::decode(FrameReader *reader, int idx, VisionBuf *buf) {
   }
   reader->prev_idx = idx;
 
+read_packets:
+  // H.264 has decoder delay: the first packets may legitimately yield no frame
+  // yet (EAGAIN), and one packet can release several buffered frames. comma's
+  // zero-delay HEVC never exercised either case.
   AVPacket pkt;
-  while (av_read_frame(reader->input_ctx, &pkt) >= 0) {
-    // Skip non-video packets
+  int rf_ret;
+  while ((rf_ret = av_read_frame(reader->input_ctx, &pkt)) >= 0) {
     if (pkt.stream_index != reader->video_stream_idx_) {
       av_packet_unref(&pkt);
       continue;
     }
-
-    AVFrame *frame = decodeFrame(&pkt);
+    int ret = avcodec_send_packet(decoder_ctx, &pkt);
     av_packet_unref(&pkt);
-    if (!frame) {
-      rError("Failed to decode frame at index %d", current_idx);
+    if (ret < 0) {
+      rError("Error sending a packet for decoding: %d", ret);
       return false;
     }
-
-    if (current_idx++ == idx) {
-      return copyBuffer(frame, buf);
+    while ((ret = avcodec_receive_frame(decoder_ctx, av_frame_)) == 0) {
+      AVFrame *frame = av_frame_;
+      if (av_frame_->format == hw_pix_fmt) {
+        if (av_hwframe_transfer_data(hw_frame_, av_frame_, 0) < 0) {
+          rError("error transferring frame data from GPU to CPU");
+          return false;
+        }
+        frame = hw_frame_;
+      }
+      if (current_idx++ == idx) {
+        return copyBuffer(frame, buf);
+      }
+    }
+    if (ret != AVERROR(EAGAIN)) {
+      rError("avcodec_receive_frame error: %d", ret);
+      return false;
     }
   }
-  rError("Failed to find frame at index %d", idx);
+  rError("Failed to find frame at index %d (read ret=%d)", idx, rf_ret);
   return false;
 }
 
@@ -268,20 +312,46 @@ bool FFmpegVideoDecoder::copyBuffer(AVFrame *f, VisionBuf *buf) {
 }
 
 #ifndef __APPLE__
+QcomVideoDecoder::~QcomVideoDecoder() {
+  if (bsf_) av_bsf_free(&bsf_);
+}
+
 bool QcomVideoDecoder::open(AVCodecParameters *codecpar, bool hw_decoder) {
-  if (codecpar->codec_id != AV_CODEC_ID_HEVC) {
-    rError("Hardware decoder only supports HEVC codec");
+  // msm_vidc decodes both; IQ.Pilot recordings are H.264 while comma's are HEVC
+  uint32_t v4l2_fmt;
+  if (codecpar->codec_id == AV_CODEC_ID_HEVC) {
+    v4l2_fmt = V4L2_PIX_FMT_HEVC;
+  } else if (codecpar->codec_id == AV_CODEC_ID_H264) {
+    v4l2_fmt = V4L2_PIX_FMT_H264;
+    if (codecpar->extradata && codecpar->extradata_size > 0) {
+      // mp4 carries AVCC (length-prefixed NALs, headers out-of-band); the V4L2
+      // decoder wants an Annex-B bitstream with in-band SPS/PPS
+      const AVBitStreamFilter *f = av_bsf_get_by_name("h264_mp4toannexb");
+      if (!f || av_bsf_alloc(f, &bsf_) < 0 ||
+          avcodec_parameters_copy(bsf_->par_in, codecpar) < 0 || av_bsf_init(bsf_) < 0) {
+        rError("failed to set up h264_mp4toannexb filter");
+        return false;
+      }
+    }
+  } else {
+    rError("Hardware decoder only supports HEVC and H.264 codecs");
     return false;
   }
   width = codecpar->width;
   height = codecpar->height;
-  msm_vidc.init(VIDEO_DEVICE, width, height, V4L2_PIX_FMT_HEVC);
+  msm_vidc.init(VIDEO_DEVICE, width, height, v4l2_fmt);
   return true;
 }
 
 bool QcomVideoDecoder::decode(FrameReader *reader, int idx, VisionBuf *buf) {
   int from_idx = idx;
   if (idx != reader->prev_idx + 1) {
+    if (idx > reader->prev_idx && idx - reader->prev_idx <= 300) {
+      // forward catch-up: the decoder is already positioned at prev_idx+1, and
+      // sequential decode is cheaper and (for raw H.264) more reliable than a
+      // byte seek plus keyframe re-decode
+      from_idx = reader->prev_idx + 1;
+    } else {
     // seeking to the nearest key frame
     for (int i = idx; i >= 0; --i) {
       if (reader->packets_info[i].flags & AV_PKT_FLAG_KEY) {
@@ -292,10 +362,17 @@ bool QcomVideoDecoder::decode(FrameReader *reader, int idx, VisionBuf *buf) {
 
     auto pos = reader->packets_info[from_idx].pos;
     int ret = avformat_seek_file(reader->input_ctx, 0, pos, pos, pos, AVSEEK_FLAG_BYTE);
+    if (ret < 0) {
+      // mp4 containers reject byte seeks; seek the keyframe by timestamp
+      // through the mov index instead
+      auto ts = reader->packets_info[from_idx].ts;
+      ret = avformat_seek_file(reader->input_ctx, reader->video_stream_idx_, INT64_MIN, ts, ts, 0);
+    }
     if (ret < 0) {
       rError("Failed to seek to byte position %lld: %d", pos, AVERROR(ret));
       return false;
     }
+    }
   }
   reader->prev_idx = idx;
   bool result = false;
@@ -303,6 +380,13 @@ bool QcomVideoDecoder::decode(FrameReader *reader, int idx, VisionBuf *buf) {
   msm_vidc.avctx = reader->input_ctx;
   for (int i = from_idx; i <= idx; ++i) {
     if (av_read_frame(reader->input_ctx, &pkt) == 0) {
+      if (bsf_ != nullptr) {
+        if (av_bsf_send_packet(bsf_, &pkt) < 0 || av_bsf_receive_packet(bsf_, &pkt) < 0) {
+          rError("h264_mp4toannexb failed at index %d", i);
+          av_packet_unref(&pkt);
+          return false;
+        }
+      }
       result = msm_vidc.decodeFrame(&pkt, buf) && (i == idx);
       av_packet_unref(&pkt);
     }

iqpilot/tools/replay/framereader.h

@@ -37,6 +37,7 @@ public:
   struct PacketInfo {
     int flags;
     int64_t pos;
+    int64_t ts;  // dts; byte pos is useless for seeking in mp4 containers
   };
   std::vector<PacketInfo> packets_info;
 };
@@ -72,11 +73,12 @@ private:
 class QcomVideoDecoder : public VideoDecoder {
 public:
   QcomVideoDecoder() {};
-  ~QcomVideoDecoder() override {};
+  ~QcomVideoDecoder() override;
   bool open(AVCodecParameters *codecpar, bool hw_decoder) override;
   bool decode(FrameReader *reader, int idx, VisionBuf *buf) override;
 
 private:
   MsmVidc msm_vidc = MsmVidc();
+  AVBSFContext *bsf_ = nullptr;  // AVCC (mp4) -> Annex-B for msm_vidc
 };
 #endif

iqpilot/tools/replay/main.cc

@@ -1,4 +1,5 @@
 #include <getopt.h>
+#include <unistd.h>
 
 #include <iomanip>
 #include <iostream>
@@ -176,6 +177,15 @@ int main(int argc, char *argv[]) {
     return 0;
   }
 
+  // REPLAY_HEADLESS: skip ncurses, which needs a real TTY and swallows all
+  // replay log output into its UI — required when driven from a service
+  if (getenv("REPLAY_HEADLESS") != nullptr) {
+    replay.start(config.start_seconds);
+    while (true) {
+      pause();
+    }
+  }
+
   ConsoleUI console_ui(&replay);
   replay.start(config.start_seconds);
   return console_ui.exec();

iqpilot/tools/replay/qcom_decoder.cc

@@ -43,7 +43,7 @@ bool MsmVidc::init(const char* dev, size_t width, size_t height, uint64_t codec)
   }
   subscribeEvents();
   v4l2_buf_type out_type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
-  setPlaneFormat(out_type, V4L2_PIX_FMT_HEVC); // Also allocates the output buffer
+  setPlaneFormat(out_type, codec); // Also allocates the output buffer
   setFPS(FPS);
   request_buffers(fd, out_type, OUTPUT_BUFFER_COUNT);
   util::safe_ioctl(fd, VIDIOC_STREAMON, &out_type, "VIDIOC_STREAMON OUTPUT failed");