IQ.Pilot Release Commit @ 24db8ae

iqpilot/iq_maps/tile_bundle_downloader.py

@@ -21,11 +21,21 @@ except Exception:  # ProprietaryModuleMissing or import errors in stripped build
   _private_base_urls = None
   _private_auth = None
 
-DEFAULT_TILE_BUNDLE_BASE_URL = "https://maps.konn3kt.com/iqosmd/v1"
+# Tile bundles live as LFS objects in the PRIVATE repo IQ.Lvbs/iqmaps (R2 is gone).
+# Anonymous access 404s by design; devices authenticate with the embedded read-only PAT
+# carried by the closed-source updater bundle (same fetch account as the OS images).
+DEFAULT_TILE_BUNDLE_BASE_URL = "https://git.konn3kt.com/IQ.Lvbs/iqmaps/raw/branch/master"
+FALLBACK_TILE_BUNDLE_BASE_URL = "https://gitlvb.teallvbs.xyz/IQ.Lvbs/iqmaps/raw/branch/master"
+
+# Gitea /raw NEVER returns LFS content -- it returns this pointer, and the real bytes come
+# from the LFS batch API (see _resolve_object_url).
+LFS_POINTER_MAGIC = b"version https://git-lfs"
 BASE_URL_PARAM = "OfflineTilesBaseUrl"
 PROGRESS_PARAM = "OfflineTilesDownloadProgress"
 REQUEST_PARAM = "OfflineTilesDownloadRequest"
 CHUNK_BYTES = 1 << 20
+# must match scripts/iqpilot/tile_factory/upload_bundles_lfs.py
+PART_BYTES = 90 * 1024 * 1024
 HTTP_TIMEOUT_S = 30.0
 STREAM_RETRIES = 8
 
@@ -44,9 +54,52 @@ def candidate_base_urls(params: Params) -> list[str]:
     except Exception:
       pass
   urls.append(DEFAULT_TILE_BUNDLE_BASE_URL)
+  urls.append(FALLBACK_TILE_BUNDLE_BASE_URL)
   return urls
 
 
+def _maps_auth_module():
+  """The read PAT lives in the compiled updater bundle (never in this open file)."""
+  try:
+    from iqpilot.system.proprietary_runtime._verified_import import import_verified_module
+    return import_verified_module("iqpilot_updater_private", "iqpilot_private.updater.git_remote")
+  except Exception:
+    pass
+  try:
+    import importlib
+    import os
+    import sys
+    root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+    bundle_python = os.path.join(root, "artifacts", "iqpilot_updater_private", "python")
+    if os.path.isdir(bundle_python):
+      if bundle_python not in sys.path:
+        sys.path.insert(0, bundle_python)
+      return importlib.import_module("iqpilot_private.updater.git_remote")
+  except Exception:
+    pass
+  return None
+
+
+def request_headers(url: str) -> dict:
+  mod = _maps_auth_module()
+  if mod is not None:
+    try:
+      headers = mod.map_tiles_headers(url)
+      if headers:
+        return headers
+    except Exception:
+      pass
+  try:
+    from iqpilot.common.git_creds import get_credentials
+    creds = get_credentials()
+    if creds and all(creds) and "/iq.lvbs/iqmaps" in url.lower():
+      import base64
+      return {"Authorization": "Basic " + base64.b64encode(f"{creds[0]}:{creds[1]}".encode()).decode()}
+  except Exception:
+    pass
+  return {}
+
+
 def request_auth() -> tuple[str, str] | None:
   if _private_auth is None:
     return None
@@ -56,8 +109,67 @@ def request_auth() -> tuple[str, str] | None:
     return None
 
 
+def _lfs_endpoint(base_url: str) -> str:
+  """<host>/<owner>/<repo>/raw/branch/<b> -> <host>/<owner>/<repo>.git/info/lfs"""
+  return base_url.split("/raw/", 1)[0] + ".git/info/lfs"
+
+
+def _resolve_oid_url(session: requests.Session, base_url: str, oid: str, size: int,
+                     headers: dict) -> tuple[str, dict]:
+  """Bundles are stored as bare LFS objects addressed by oid from the index -- no pointer
+  files, because committing one per part meant hundreds of concurrent commits per branch."""
+  batch = session.post(f"{_lfs_endpoint(base_url)}/objects/batch",
+                       data=json.dumps({"operation": "download", "transfers": ["basic"],
+                                        "objects": [{"oid": oid, "size": size}]}),
+                       headers={"Content-Type": "application/vnd.git-lfs+json",
+                                "Accept": "application/vnd.git-lfs+json", **headers},
+                       timeout=HTTP_TIMEOUT_S)
+  batch.raise_for_status()
+  entry = batch.json()["objects"][0]
+  if "actions" not in entry:
+    raise requests.RequestException(f"LFS object unavailable: {entry.get('error', oid)}")
+  action = entry["actions"]["download"]
+  return action["href"], action.get("header", {})
+
+
+def _resolve_object_url(session: requests.Session, url: str, headers: dict) -> tuple[str, dict]:
+  """Follow a Gitea LFS pointer to the real (pre-signed) object URL.
+
+  Returns the URL to stream plus any extra headers it needs. A plain host that serves the
+  bytes directly (local test server, static mirror) resolves to itself unchanged."""
+  probe = session.get(url, headers={**headers, "Accept-Encoding": None}, stream=True,
+                      timeout=HTTP_TIMEOUT_S)
+  probe.raise_for_status()
+  if int(probe.headers.get("content-length") or 0) >= 1024:
+    probe.close()
+    return url, {}
+  body = probe.content
+  probe.close()
+  if not body.startswith(LFS_POINTER_MAGIC):
+    return url, {}
+
+  meta = dict(line.split(" ", 1) for line in body.decode().strip().splitlines() if " " in line)
+  oid = meta["oid"].split(":", 1)[1]
+  size = int(meta["size"])
+  lfs_base = url.split("/raw/", 1)[0] + ".git/info/lfs"
+  batch = session.post(f"{lfs_base}/objects/batch",
+                       data=json.dumps({"operation": "download", "transfers": ["basic"],
+                                        "objects": [{"oid": oid, "size": size}]}),
+                       headers={"Content-Type": "application/vnd.git-lfs+json",
+                                "Accept": "application/vnd.git-lfs+json", **headers},
+                       timeout=HTTP_TIMEOUT_S)
+  batch.raise_for_status()
+  action = batch.json()["objects"][0]["actions"]["download"]
+  return action["href"], action.get("header", {})
+
+
 def fetch_index(base_url: str, session: requests.Session) -> dict:
-  response = session.get(f"{base_url}/index.json", timeout=HTTP_TIMEOUT_S, auth=request_auth())
+  index_url = f"{base_url}/index.json"
+  headers = request_headers(index_url)
+  # requests' auth= rewrites the Authorization header, so only fall back to it when the
+  # closed-source bundle gave us nothing.
+  response = session.get(index_url, timeout=HTTP_TIMEOUT_S, headers=headers,
+                         auth=None if headers else request_auth())
   response.raise_for_status()
   index = response.json()
   regions = index.get("regions")
@@ -164,7 +276,7 @@ class TileBundleDownloader:
     ok = self._download_file(
       selector, base_url, entry["path"], int(entry.get("bytes", 0)),
       str(entry.get("sha256", "")).strip().lower(), night_path,
-      progress_offset, progress_total,
+      progress_offset, progress_total, int(entry.get("parts", 1)), entry.get("objects"),
     )
     if not ok:
       return False
@@ -174,6 +286,7 @@ class TileBundleDownloader:
         str(entry.get("day_sha256", "")).strip().lower(),
         night_path.with_name("offline_day.mbtiles"),
         progress_offset + int(entry.get("bytes", 0)), progress_total,
+        int(entry.get("day_parts", 1)), entry.get("day_objects"),
       )
       if not day_ok:
         cloudlog.warning(f"iq_maps: day-style bundle failed for {selector}; night set installed")
@@ -183,8 +296,16 @@ class TileBundleDownloader:
 
   def _download_file(self, selector: str, base_url: str, remote_path: str, expected_bytes: int,
                      expected_sha: str, final_path: Path,
-                     progress_offset: int, progress_total: int) -> bool:
-    url = f"{base_url}/{remote_path.lstrip('/')}"
+                     progress_offset: int, progress_total: int, parts: int = 1,
+                     objects: list | None = None) -> bool:
+    # Bundles are published as <name>.pNN because Cloudflare caps proxied bodies at ~100MB.
+    # They stream back-to-back into ONE .part file: concatenating afterwards would need
+    # double the free space, which devices do not have.
+    base = f"{base_url}/{remote_path.lstrip('/')}"
+    if objects:
+      urls = [None] * len(objects)   # resolved per-attempt from the oid
+    else:
+      urls = [base] if parts <= 1 else [f"{base}.p{i:02d}" for i in range(parts)]
     part_path = final_path.with_name(final_path.name + ".part")
     part_path.parent.mkdir(parents=True, exist_ok=True)
 
@@ -207,28 +328,45 @@ class TileBundleDownloader:
             digest = hashlib.sha256()
             resume_from = 0
 
-        headers = {"Range": f"bytes={resume_from}-"} if resume_from else {}
-        auth = request_auth()
-        response = self.session.get(url, headers=headers, stream=True, timeout=HTTP_TIMEOUT_S, auth=auth)
-        if resume_from and response.status_code != 206:
-          digest = hashlib.sha256()
-          resume_from = 0
-          part_path.unlink(missing_ok=True)
-          if response.status_code == 416:
-            response = self.session.get(url, stream=True, timeout=HTTP_TIMEOUT_S, auth=auth)
-        response.raise_for_status()
-
+        # every part but the last is exactly PART_BYTES, so a byte offset maps to a part index
+        first_part = resume_from // PART_BYTES if len(urls) > 1 else 0
+        skip_in_part = resume_from - first_part * PART_BYTES if len(urls) > 1 else resume_from
         downloaded = resume_from
         mode = "ab" if resume_from else "wb"
         with open(part_path, mode) as f:
-          for chunk in response.iter_content(chunk_size=CHUNK_BYTES):
-            if self._should_abort():
-              cloudlog.warning(f"iq_maps: tile bundle download cancelled for {selector}")
-              return False
-            f.write(chunk)
-            digest.update(chunk)
-            downloaded += len(chunk)
-            self._publish_progress(selector, progress_offset + downloaded, progress_total, active=True)
+          for index in range(first_part, len(urls)):
+            if objects:
+              url_headers = request_headers(base_url)
+              auth = None if url_headers else request_auth()
+              object_url, object_headers = _resolve_oid_url(
+                self.session, base_url, objects[index]["oid"], int(objects[index]["size"]),
+                url_headers)
+            else:
+              url = urls[index]
+              url_headers = request_headers(url)
+              auth = None if url_headers else request_auth()
+              # Re-resolve per part: a pre-signed LFS object URL can expire mid-download.
+              object_url, object_headers = _resolve_object_url(self.session, url, url_headers)
+            headers = dict(object_headers)
+            offset = skip_in_part if index == first_part else 0
+            if offset:
+              headers["Range"] = f"bytes={offset}-"
+            response = self.session.get(object_url, headers=headers, stream=True,
+                                        timeout=HTTP_TIMEOUT_S, auth=auth)
+            if offset and response.status_code != 206:
+              # server ignored the range: restart this whole file cleanly
+              f.close()
+              part_path.unlink(missing_ok=True)
+              raise requests.RequestException(f"range not honoured for part {index}")
+            response.raise_for_status()
+            for chunk in response.iter_content(chunk_size=CHUNK_BYTES):
+              if self._should_abort():
+                cloudlog.warning(f"iq_maps: tile bundle download cancelled for {selector}")
+                return False
+              f.write(chunk)
+              digest.update(chunk)
+              downloaded += len(chunk)
+              self._publish_progress(selector, progress_offset + downloaded, progress_total, active=True)
         break
       except requests.RequestException as exc:
         last_error = exc