IQ.Pilot Prebuilt Release @ 27f668a

iqpilot/iq_maps/__init__.py

@@ -0,0 +1,5 @@
+import os
+from iqpilot.common.basedir import BASEDIR
+
+VENDOR_MAPD_BIN_DIR = os.path.join(BASEDIR, "iqpilot/third_party/mapd_pfeiferj")
+VENDOR_MAPD_PATH = os.path.join(VENDOR_MAPD_BIN_DIR, "mapd")

iqpilot/iq_maps/orchestrator.py

@@ -0,0 +1,419 @@
+#!/usr/bin/env python3
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
+"""
+import platform
+import os
+import glob
+import shutil
+import signal
+import subprocess
+import threading
+import time
+from datetime import datetime
+
+import iqpilot.cereal.messaging as messaging
+from iqpilot.cereal import custom
+from iqpilot.common.params import Params
+from iqpilot.common.realtime import Ratekeeper, config_realtime_process
+from iqpilot.common.swaglog import cloudlog
+from iqpilot.selfdrive.selfdrived.alertmanager import set_offroad_alert
+from iqpilot.system.hardware.hw import Paths
+from iqpilot.iq_maps import VENDOR_MAPD_BIN_DIR, VENDOR_MAPD_PATH
+from iqpilot.iq_maps.tile_bundle_downloader import TileBundleDownloader, region_bundle_installed
+from iqpilot.iq_maps.vendor_mapd_installer import VendorMapdInstaller
+
+OfflineMapAction = custom.MapdInputType
+_region_sync_worker: threading.Thread | None = None
+
+_active_proc_lock = threading.Lock()
+_active_proc: subprocess.Popen | None = None
+_shutdown = threading.Event()
+_tile_downloader: TileBundleDownloader | None = None
+
+
+def _vendor_fetch_pidfile() -> str:
+  return os.path.join(Paths.mapd_root(), ".vendor_fetch.pid")
+
+
+def _pid_is_vendor_fetch(pid: int) -> bool:
+  try:
+    with open(f"/proc/{pid}/cmdline", "rb") as f:
+      cmdline = f.read()
+  except OSError:
+    return False
+  return VENDOR_MAPD_PATH.encode() in cmdline
+
+
+def _reap_orphaned_vendor_fetch() -> None:
+  pidfile = _vendor_fetch_pidfile()
+  try:
+    with open(pidfile) as f:
+      pid = int(f.read().strip())
+  except (OSError, ValueError):
+    return
+  try:
+    if _pid_is_vendor_fetch(pid):
+      cloudlog.warning(f"iq_maps: reaping orphaned vendor-fetch mapd pid={pid} from a prior run")
+      os.kill(pid, signal.SIGTERM)
+      for _ in range(20):
+        time.sleep(0.1)
+        if not _pid_is_vendor_fetch(pid):
+          break
+      else:
+        os.kill(pid, signal.SIGKILL)
+  except ProcessLookupError:
+    pass
+  finally:
+    try:
+      os.remove(pidfile)
+    except OSError:
+      pass
+
+
+def _kill_active_proc() -> None:
+  with _active_proc_lock:
+    proc = _active_proc
+  if proc is None or proc.poll() is not None:
+    return
+  proc.terminate()
+  try:
+    proc.wait(timeout=3)
+  except Exception:
+    proc.kill()
+    try:
+      proc.wait(timeout=2)
+    except Exception:
+      pass
+
+
+def _handle_shutdown_signal(signum, _frame) -> None:
+  cloudlog.warning(f"iq_maps: mapd_manager received signal {signum}, cleaning up vendor-fetch subprocess")
+  _shutdown.set()
+  _kill_active_proc()
+  if _tile_downloader is not None:
+    _tile_downloader.cancel()
+  worker = _region_sync_worker
+  if worker is not None and worker.is_alive():
+    worker.join(timeout=3)
+  raise SystemExit(0)
+
+
+def _install_signal_handlers() -> None:
+  signal.signal(signal.SIGINT, _handle_shutdown_signal)
+  signal.signal(signal.SIGTERM, _handle_shutdown_signal)
+
+
+def ensure_vendor_runtime() -> None:
+  try:
+    VendorMapdInstaller().verify()
+  except Exception:
+    cloudlog.exception("iq_maps: vendor runtime verification failed")
+
+params = Params()
+mem_params = Params("/dev/shm/params") if platform.system() != "Darwin" else params
+
+
+def stale_region_artifacts() -> list[str]:
+  patterns = [
+    f"{Paths.mapd_root()}/db",
+    f"{Paths.mapd_root()}/v*"
+  ]
+  stale_paths: list[str] = []
+  for pattern in patterns:
+    for match in glob.glob(pattern):
+      stale_paths.append(match)
+      if os.path.isdir(match):
+        stale_paths.extend(glob.glob(match + '/**', recursive=True))
+  if not os.path.isfile(VENDOR_MAPD_PATH):
+    stale_paths.append(VENDOR_MAPD_PATH)
+  return stale_paths
+
+
+def purge_stale_region_artifacts(stale_paths: list[str]) -> None:
+  for candidate in stale_paths:
+    if candidate.endswith('/') and os.path.isfile(candidate[:-1]):
+      candidate = candidate[:-1]
+    if os.path.islink(candidate) or os.path.isfile(candidate):
+      os.remove(candidate)
+    elif os.path.isdir(candidate):
+      shutil.rmtree(candidate, ignore_errors=False)
+
+
+def _compose_region_selector(nations: list[str], states: list[str] | None = None) -> str:
+  requested_paths: list[str] = []
+  for state_code in (states or []):
+    code = str(state_code).strip().upper()
+    if code and code != "ALL":
+      requested_paths.append(f"us_state.{code}")
+  for nation_code in (nations or []):
+    code = str(nation_code).strip().upper()
+    if code:
+      requested_paths.append(f"nation.{code}")
+  return ",".join(requested_paths)
+
+
+def _fetch_tile_bundles(region_selector: str, abort_check=None) -> None:
+  global _tile_downloader
+  if not params.get_bool("OfflineOSMaps"):
+    return
+  selectors = [part for part in region_selector.split(",") if part]
+  if not selectors:
+    return
+  try:
+    _tile_downloader = TileBundleDownloader(params=params, mem_params=mem_params, abort_check=abort_check)
+    _tile_downloader.download_regions(selectors)
+  except Exception:
+    cloudlog.exception("iq_maps: tile bundle download failed")
+  finally:
+    _tile_downloader = None
+
+
+def _drive_vendor_fetch(region_selector: str, requested_regions: dict) -> None:
+  global _active_proc
+  proc = None
+  cancelled = False
+  try:
+    mem_params.put("OSMDownloadLocations", requested_regions)
+    proc = subprocess.Popen([VENDOR_MAPD_PATH], cwd=VENDOR_MAPD_BIN_DIR,
+                            stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+                            start_new_session=True)
+    with _active_proc_lock:
+      _active_proc = proc
+    with open(_vendor_fetch_pidfile(), "w") as f:
+      f.write(str(proc.pid))
+
+    pm = messaging.PubMaster(["mapdIn"])
+    sm = messaging.SubMaster(["mapdExtendedOut"])
+    time.sleep(4.0)
+
+    for _ in range(10):
+      msg = messaging.new_message("mapdIn")
+      msg.mapdIn.type = OfflineMapAction.download
+      msg.mapdIn.str = region_selector
+      pm.send("mapdIn", msg)
+      time.sleep(0.2)
+
+    started = False
+    deadline = time.monotonic() + 3600.0
+    while time.monotonic() < deadline and not _shutdown.is_set():
+      sm.update(500)
+      dp = sm["mapdExtendedOut"].downloadProgress
+      mem_params.put("OSMDownloadProgress", {
+        "active": bool(dp.active),
+        "total_files": int(dp.totalFiles),
+        "downloaded_files": int(dp.downloadedFiles),
+      })
+      if dp.active:
+        started = True
+      elif started:
+        break
+      if not mem_params.get("OSMDownloadLocations"):
+        cancelled = True
+        cancel = messaging.new_message("mapdIn")
+        cancel.mapdIn.type = OfflineMapAction.cancelDownload
+        pm.send("mapdIn", cancel)
+        break
+    cloudlog.info(f"iq_maps: vendor map download finished for {region_selector}")
+    if not cancelled and not _shutdown.is_set():
+      _fetch_tile_bundles(region_selector, abort_check=lambda: _shutdown.is_set() or not mem_params.get("OSMDownloadLocations"))
+  except Exception:
+    cloudlog.exception("iq_maps: vendor map download failed")
+  finally:
+    try:
+      mem_params.remove("OSMDownloadLocations")
+    except Exception:
+      pass
+    if proc is not None:
+      proc.terminate()
+      try:
+        proc.wait(timeout=5)
+      except Exception:
+        proc.kill()
+    with _active_proc_lock:
+      _active_proc = None
+    try:
+      os.remove(_vendor_fetch_pidfile())
+    except OSError:
+      pass
+
+
+def queue_region_refresh(nations: list[str], states: list[str] | None = None) -> None:
+  global _region_sync_worker
+  params.put("OsmDownloadedDate", str(datetime.now().timestamp()))
+  params.put_bool("OsmDbUpdatesCheck", False)
+
+  region_selector = _compose_region_selector(nations, states)
+  if not region_selector:
+    cloudlog.warning("iq_maps: no region selected for offline map download")
+    return
+  if _region_sync_worker is not None and _region_sync_worker.is_alive():
+    cloudlog.warning("iq_maps: vendor map download already in progress")
+    return
+
+  requested_regions = {"nations": nations, "states": states or [], "paths": region_selector}
+  cloudlog.info(f"iq_maps: starting vendor map download for {region_selector}")
+  _region_sync_worker = threading.Thread(
+    target=_drive_vendor_fetch,
+    args=(region_selector, requested_regions),
+    daemon=True,
+  )
+  _region_sync_worker.start()
+
+
+def normalize_region_selection(nations: list[str], states: list[str] | None = None) -> tuple[list[str], list[str]]:
+  normalized_nations = list(nations)
+  normalized_states = list(states or [])
+  lowered_states = {entry.lower() for entry in normalized_states}
+
+  if "US" in normalized_nations and normalized_states and "all" not in lowered_states:
+    normalized_nations = [entry for entry in normalized_nations if entry != "US"]
+  elif normalized_states:
+    normalized_states = [entry for entry in normalized_states if entry.lower() != "all"]
+
+  return normalized_nations, normalized_states
+
+
+_AUTO_RESTORE_INTERVAL_S = 1800.0
+_last_auto_restore_t = 0.0
+
+
+def region_data_missing() -> bool:
+  if not params.get_bool("OsmLocal"):
+    return False
+  if not params.get("OsmDownloadedDate"):
+    return False
+  if glob.glob(f"{Paths.mapd_root()}/db") or glob.glob(f"{Paths.mapd_root()}/v*"):
+    return False
+  if glob.glob(f"{Paths.mapd_root()}/offline/*/*"):
+    return False
+  country = params.get("OsmLocationName", return_default=True)
+  return bool(country)
+
+
+def configured_states() -> list[str]:
+  try:
+    states = params.get("OsmStateNames")
+    if isinstance(states, bytes):
+      import json as _json
+      states = _json.loads(states.decode("utf-8"))
+    if isinstance(states, str):
+      import json as _json
+      states = _json.loads(states)
+    if isinstance(states, list) and states:
+      return [str(s).strip().upper() for s in states if str(s).strip()]
+  except Exception:
+    pass
+  state = params.get("OsmStateName", return_default=True)
+  return [state] if state else []
+
+
+def maybe_auto_restore_region() -> None:
+  global _last_auto_restore_t
+  if not region_data_missing():
+    return
+  if _region_sync_worker is not None and _region_sync_worker.is_alive():
+    return
+  now = time.monotonic()
+  if now - _last_auto_restore_t < _AUTO_RESTORE_INTERVAL_S:
+    return
+  _last_auto_restore_t = now
+  country = params.get("OsmLocationName", return_default=True)
+  states = configured_states()
+  nations, states_filtered = normalize_region_selection([country], states)
+  cloudlog.warning(f"iq_maps: configured offline region {country}/{states} has no data on disk; auto-restoring")
+  queue_region_refresh(nations, states_filtered)
+
+
+_TILE_RESTORE_INTERVAL_S = 1800.0
+_last_tile_restore_t = 0.0
+_tile_only_worker: threading.Thread | None = None
+
+
+def _configured_region_selector() -> str:
+  country = params.get("OsmLocationName", return_default=True)
+  states = configured_states()
+  nations, states_filtered = normalize_region_selection([country] if country else [], states)
+  return _compose_region_selector(nations, states_filtered)
+
+
+def tile_bundles_missing() -> bool:
+  if not params.get_bool("OfflineOSMaps"):
+    return False
+  selector = _configured_region_selector()
+  if not selector:
+    return False
+  return any(not region_bundle_installed(part) for part in selector.split(",") if part)
+
+
+def maybe_restore_tile_bundles() -> None:
+  global _last_tile_restore_t, _tile_only_worker
+  if not tile_bundles_missing():
+    return
+  if _region_sync_worker is not None and _region_sync_worker.is_alive():
+    return
+  if _tile_only_worker is not None and _tile_only_worker.is_alive():
+    return
+  now = time.monotonic()
+  if now - _last_tile_restore_t < _TILE_RESTORE_INTERVAL_S:
+    return
+  _last_tile_restore_t = now
+  selector = _configured_region_selector()
+  cloudlog.warning(f"iq_maps: offline map tile bundles missing for {selector}; downloading")
+  _tile_only_worker = threading.Thread(
+    target=_fetch_tile_bundles,
+    args=(selector,),
+    kwargs={"abort_check": _shutdown.is_set},
+    daemon=True,
+  )
+  _tile_only_worker.start()
+
+
+def sync_osm_request_flags() -> None:
+  maybe_auto_restore_region()
+  maybe_restore_tile_bundles()
+  if params.get_bool("OsmDbUpdatesCheck"):
+    if _region_sync_worker is not None and _region_sync_worker.is_alive():
+      return
+    purge_stale_region_artifacts(stale_region_artifacts())
+    country = params.get("OsmLocationName", return_default=True)
+    states = configured_states()
+    filtered_nations, filtered_states = normalize_region_selection([country], states)
+    queue_region_refresh(filtered_nations, filtered_states)
+
+  if not mem_params.get("OSMDownloadBounds"):
+    mem_params.put("OSMDownloadBounds", "")
+
+  if not mem_params.get("LastGPSPosition"):
+    mem_params.put("LastGPSPosition", "{}")
+
+
+def run_loop():
+  ensure_vendor_runtime()
+  config_realtime_process([0, 1, 2, 3], 5)
+
+  rk = Ratekeeper(1, print_delay_threshold=None)
+
+  try:
+    os.mkdir(Paths.mapd_root())
+  except FileExistsError:
+    pass
+  except PermissionError:
+    cloudlog.exception(f"iq_maps: failed to make {Paths.mapd_root()}")
+  _reap_orphaned_vendor_fetch()
+  _install_signal_handlers()
+
+  while not _shutdown.is_set():
+    show_alert = stale_region_artifacts() and params.get_bool("OsmLocal")
+    set_offroad_alert("Offroad_OSMUpdateRequired", show_alert, "This alert will be cleared when new maps are downloaded.")
+
+    sync_osm_request_flags()
+    rk.keep_time()
+
+
+def main():
+  run_loop()
+
+
+if __name__ == "__main__":
+  main()

iqpilot/iq_maps/tile_bundle_downloader.py

@@ -0,0 +1,516 @@
+#!/usr/bin/env python3
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
+"""
+import hashlib
+import json
+import platform
+import threading
+import time
+from pathlib import Path
+
+import requests
+
+from iqpilot.common.params import Params
+from iqpilot.common.swaglog import cloudlog
+from iqpilot.ui.onroad.offline_tiles import offline_map_root
+
+try:
+  from iqpilot.iq_maps.tiles_auth import get_base_urls as _private_base_urls, get_requests_auth as _private_auth
+except Exception:  # ProprietaryModuleMissing or import errors in stripped builds
+  _private_base_urls = None
+  _private_auth = None
+
+# Tile bundles live as LFS objects in the PRIVATE repo IQ.Lvbs/iqmaps (R2 is gone).
+# Anonymous access 404s by design; devices authenticate with the embedded read-only PAT
+# carried by the closed-source updater bundle (same fetch account as the OS images).
+# Hugging Face is primary: it is CDN-served, so device downloads no longer come off the
+# gitea box's home uplink. The gitea copies stay as failover -- if HF ever suspends the
+# repo the fleet silently falls back instead of losing maps entirely.
+HF_TILE_BUNDLE_BASE_URL = "https://huggingface.co/datasets/T3vl/iqmaps/resolve/main"
+DEFAULT_TILE_BUNDLE_BASE_URL = "https://git.konn3kt.com/IQ.Lvbs/iqmaps/raw/branch/master"
+FALLBACK_TILE_BUNDLE_BASE_URL = "https://gitlvb.teallvbs.xyz/IQ.Lvbs/iqmaps/raw/branch/master"
+
+# Gitea /raw NEVER returns LFS content -- it returns this pointer, and the real bytes come
+# from the LFS batch API (see _resolve_object_url).
+LFS_POINTER_MAGIC = b"version https://git-lfs"
+BASE_URL_PARAM = "OfflineTilesBaseUrl"
+PROGRESS_PARAM = "OfflineTilesDownloadProgress"
+REQUEST_PARAM = "OfflineTilesDownloadRequest"
+CHUNK_BYTES = 1 << 20
+# must match scripts/iqpilot/tile_factory/upload_bundles_lfs.py
+PART_BYTES = 90 * 1024 * 1024
+HTTP_TIMEOUT_S = 30.0
+STREAM_RETRIES = 8
+
+
+def candidate_base_urls(params: Params) -> list[str]:
+  override = params.get(BASE_URL_PARAM)
+  if isinstance(override, bytes):
+    override = override.decode("utf-8", errors="ignore")
+  override = (override or "").strip()
+  if override:
+    return [override.rstrip("/")]
+  # HF first (CDN, and it keeps device traffic off the gitea box's uplink); the bundle's
+  # own endpoints and the self-hosted defaults follow as failover.
+  urls: list[str] = [HF_TILE_BUNDLE_BASE_URL]
+  if _private_base_urls is not None:
+    try:
+      urls.extend(url.rstrip("/") for url in _private_base_urls())
+    except Exception:
+      pass
+  urls.append(DEFAULT_TILE_BUNDLE_BASE_URL)
+  urls.append(FALLBACK_TILE_BUNDLE_BASE_URL)
+  seen: set[str] = set()
+  return [u for u in urls if not (u in seen or seen.add(u))]
+
+
+def _is_hf(url: str) -> bool:
+  return "huggingface.co" in url.lower()
+
+
+def _maps_auth_module():
+  """The read PAT lives in the compiled updater bundle (never in this open file)."""
+  try:
+    from iqpilot.system.proprietary_runtime._verified_import import import_verified_module
+    return import_verified_module("iqpilot_updater_private", "iqpilot_private.updater.git_remote")
+  except Exception:
+    pass
+  try:
+    import importlib
+    import os
+    import sys
+    root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+    bundle_python = os.path.join(root, "artifacts", "iqpilot_updater_private", "python")
+    if os.path.isdir(bundle_python):
+      if bundle_python not in sys.path:
+        sys.path.insert(0, bundle_python)
+      return importlib.import_module("iqpilot_private.updater.git_remote")
+  except Exception:
+    pass
+  return None
+
+
+def request_headers(url: str) -> dict:
+  mod = _maps_auth_module()
+  if mod is not None:
+    if _is_hf(url):
+      # HF wants a bearer token, not basic auth; a build whose bundle predates HF
+      # hosting simply gets nothing here and falls through to the gitea mirrors.
+      try:
+        token = mod.map_tiles_hf_token()
+        if token:
+          return {"Authorization": f"Bearer {token}"}
+      except Exception:
+        pass
+      return {}
+    try:
+      headers = mod.map_tiles_headers(url)
+      if headers:
+        return headers
+    except Exception:
+      pass
+  try:
+    from iqpilot.common.git_creds import get_credentials
+    creds = get_credentials()
+    if creds and all(creds) and "/iq.lvbs/iqmaps" in url.lower():
+      import base64
+      return {"Authorization": "Basic " + base64.b64encode(f"{creds[0]}:{creds[1]}".encode()).decode()}
+  except Exception:
+    pass
+  return {}
+
+
+def request_auth() -> tuple[str, str] | None:
+  if _private_auth is None:
+    return None
+  try:
+    return _private_auth()
+  except Exception:
+    return None
+
+
+def _lfs_endpoint(base_url: str) -> str:
+  """<host>/<owner>/<repo>/raw/branch/<b> -> <host>/<owner>/<repo>.git/info/lfs"""
+  return base_url.split("/raw/", 1)[0] + ".git/info/lfs"
+
+
+def _resolve_oid_url(session: requests.Session, base_url: str, oid: str, size: int,
+                     headers: dict) -> tuple[str, dict]:
+  """Bundles are stored as bare LFS objects addressed by oid from the index -- no pointer
+  files, because committing one per part meant hundreds of concurrent commits per branch."""
+  batch = session.post(f"{_lfs_endpoint(base_url)}/objects/batch",
+                       data=json.dumps({"operation": "download", "transfers": ["basic"],
+                                        "objects": [{"oid": oid, "size": size}]}),
+                       headers={"Content-Type": "application/vnd.git-lfs+json",
+                                "Accept": "application/vnd.git-lfs+json", **headers},
+                       timeout=HTTP_TIMEOUT_S)
+  batch.raise_for_status()
+  entry = batch.json()["objects"][0]
+  if "actions" not in entry:
+    raise requests.RequestException(f"LFS object unavailable: {entry.get('error', oid)}")
+  action = entry["actions"]["download"]
+  return action["href"], action.get("header", {})
+
+
+def _resolve_object_url(session: requests.Session, url: str, headers: dict) -> tuple[str, dict]:
+  """Follow a Gitea LFS pointer to the real (pre-signed) object URL.
+
+  Returns the URL to stream plus any extra headers it needs. A plain host that serves the
+  bytes directly (local test server, static mirror) resolves to itself unchanged."""
+  probe = session.get(url, headers={**headers, "Accept-Encoding": None}, stream=True,
+                      timeout=HTTP_TIMEOUT_S)
+  probe.raise_for_status()
+  # A host that serves the bytes itself still needs the caller's auth on the real GET --
+  # returning {} here sends the download out anonymous and a private host answers 401.
+  if int(probe.headers.get("content-length") or 0) >= 1024:
+    probe.close()
+    return url, dict(headers)
+  body = probe.content
+  probe.close()
+  if not body.startswith(LFS_POINTER_MAGIC):
+    return url, dict(headers)
+
+  meta = dict(line.split(" ", 1) for line in body.decode().strip().splitlines() if " " in line)
+  oid = meta["oid"].split(":", 1)[1]
+  size = int(meta["size"])
+  lfs_base = url.split("/raw/", 1)[0] + ".git/info/lfs"
+  batch = session.post(f"{lfs_base}/objects/batch",
+                       data=json.dumps({"operation": "download", "transfers": ["basic"],
+                                        "objects": [{"oid": oid, "size": size}]}),
+                       headers={"Content-Type": "application/vnd.git-lfs+json",
+                                "Accept": "application/vnd.git-lfs+json", **headers},
+                       timeout=HTTP_TIMEOUT_S)
+  batch.raise_for_status()
+  action = batch.json()["objects"][0]["actions"]["download"]
+  return action["href"], action.get("header", {})
+
+
+def fetch_index(base_url: str, session: requests.Session) -> dict:
+  index_url = f"{base_url}/index.json"
+  headers = request_headers(index_url)
+  # requests' auth= rewrites the Authorization header, so only fall back to it when the
+  # closed-source bundle gave us nothing.
+  response = session.get(index_url, timeout=HTTP_TIMEOUT_S, headers=headers,
+                         auth=None if headers else request_auth())
+  response.raise_for_status()
+  index = response.json()
+  regions = index.get("regions")
+  if not isinstance(regions, dict):
+    raise ValueError("tile bundle index has no regions")
+  return regions
+
+
+def region_bundle_dir(selector: str) -> Path:
+  return offline_map_root() / "regions" / selector
+
+
+def region_bundle_path(selector: str) -> Path:
+  return region_bundle_dir(selector) / "tiles" / "offline.mbtiles"
+
+
+def region_valhalla_path(selector: str) -> Path:
+  # valhalla mmaps this tar in place, so it stays uncompressed on disk
+  return region_bundle_dir(selector) / "valhalla" / "tiles.tar"
+
+
+def region_valhalla_installed(selector: str) -> bool:
+  return region_valhalla_path(selector).exists()
+
+
+def installed_valhalla_selectors() -> list[str]:
+  regions_root = offline_map_root() / "regions"
+  if not regions_root.exists():
+    return []
+  return sorted(
+    child.name for child in regions_root.iterdir()
+    if child.is_dir() and (child / "valhalla" / "tiles.tar").exists()
+  )
+
+
+def region_bundle_installed(selector: str) -> bool:
+  return region_bundle_path(selector).exists()
+
+
+def installed_region_selectors() -> list[str]:
+  regions_root = offline_map_root() / "regions"
+  if not regions_root.exists():
+    return []
+  return sorted(
+    child.name for child in regions_root.iterdir()
+    if child.is_dir() and (child / "tiles" / "offline.mbtiles").exists()
+  )
+
+
+def _hash_existing(path: Path) -> tuple["hashlib._Hash", int]:
+  digest = hashlib.sha256()
+  size = 0
+  with open(path, "rb") as f:
+    while True:
+      chunk = f.read(CHUNK_BYTES)
+      if not chunk:
+        break
+      digest.update(chunk)
+      size += len(chunk)
+  return digest, size
+
+
+def _write_manifest(selector: str, entry: dict) -> None:
+  manifest = {
+    "region": selector,
+    "version": entry.get("version", ""),
+    "mbtiles": {
+      "bounds": entry.get("bounds", ""),
+      "minzoom": entry.get("minzoom"),
+      "maxzoom": entry.get("maxzoom"),
+      "bytes": entry.get("bytes"),
+      "sha256": entry.get("sha256", ""),
+    },
+  }
+  if entry.get("day_path"):
+    manifest["mbtiles_day"] = {
+      "bytes": entry.get("day_bytes"),
+      "sha256": entry.get("day_sha256", ""),
+    }
+  manifest_path = region_bundle_dir(selector) / "manifest.json"
+  manifest_path.parent.mkdir(parents=True, exist_ok=True)
+  manifest_path.write_text(json.dumps(manifest, indent=2))
+
+
+class TileBundleDownloader:
+
+  def __init__(self, params: Params | None = None, mem_params: Params | None = None,
+               abort_check=None):
+    self.params = params if params is not None else Params()
+    if mem_params is not None:
+      self.mem_params = mem_params
+    else:
+      self.mem_params = Params("/dev/shm/params") if platform.system() != "Darwin" else self.params
+    self.session = requests.Session()
+    self._cancelled = threading.Event()
+    self._abort_check = abort_check
+
+  def cancel(self) -> None:
+    self._cancelled.set()
+
+  def _should_abort(self) -> bool:
+    if self._cancelled.is_set():
+      return True
+    if not self.mem_params.get(REQUEST_PARAM):
+      self._cancelled.set()
+      return True
+    if self._abort_check is not None and self._abort_check():
+      self._cancelled.set()
+      return True
+    return False
+
+  def _publish_progress(self, region: str, downloaded: int, total: int, active: bool) -> None:
+    self.mem_params.put(PROGRESS_PARAM, {
+      "active": active,
+      "region": region,
+      "downloaded_bytes": int(downloaded),
+      "total_bytes": int(total),
+    })
+
+  def _download_one(self, selector: str, entry: dict, base_url: str,
+                    progress_offset: int, progress_total: int) -> bool:
+    night_path = region_bundle_path(selector)
+    ok = self._download_file(
+      selector, base_url, entry["path"], int(entry.get("bytes", 0)),
+      str(entry.get("sha256", "")).strip().lower(), night_path,
+      progress_offset, progress_total, int(entry.get("parts", 1)), entry.get("objects"),
+    )
+    if not ok:
+      return False
+    if entry.get("day_path"):
+      day_ok = self._download_file(
+        selector, base_url, entry["day_path"], int(entry.get("day_bytes", 0)),
+        str(entry.get("day_sha256", "")).strip().lower(),
+        night_path.with_name("offline_day.mbtiles"),
+        progress_offset + int(entry.get("bytes", 0)), progress_total,
+        int(entry.get("day_parts", 1)), entry.get("day_objects"),
+      )
+      if not day_ok:
+        cloudlog.warning(f"iq_maps: day-style bundle failed for {selector}; night set installed")
+    if entry.get("valhalla_path"):
+      # routing is additive: a region whose extract is missing or corrupt must still end up
+      # with a usable map rather than failing the whole download
+      try:
+        nav_ok = self._download_file(
+          selector, base_url, entry["valhalla_path"], int(entry.get("valhalla_bytes", 0)),
+          str(entry.get("valhalla_sha256", "")).strip().lower(),
+          region_valhalla_path(selector),
+          progress_offset + int(entry.get("bytes", 0)) + int(entry.get("day_bytes", 0)),
+          progress_total, 1, entry.get("valhalla_objects"),
+        )
+      except Exception as exc:
+        nav_ok = False
+        cloudlog.warning(f"iq_maps: routing extract errored for {selector}: {exc}")
+      if not nav_ok:
+        cloudlog.warning(f"iq_maps: routing extract failed for {selector}; map tiles installed")
+    _write_manifest(selector, entry)
+    cloudlog.info(f"iq_maps: installed tile bundle {selector}")
+    return True
+
+  def _download_file(self, selector: str, base_url: str, remote_path: str, expected_bytes: int,
+                     expected_sha: str, final_path: Path,
+                     progress_offset: int, progress_total: int, parts: int = 1,
+                     objects: list | None = None) -> bool:
+    # Bundles are published as <name>.pNN because Cloudflare caps proxied bodies at ~100MB.
+    # They stream back-to-back into ONE .part file: concatenating afterwards would need
+    # double the free space, which devices do not have.
+    base = f"{base_url}/{remote_path.lstrip('/')}"
+    count = len(objects) if objects else parts
+    if objects and not _is_hf(base_url):
+      urls = [None] * count          # gitea: resolved per-attempt from the oid
+    else:
+      # HF (and plain mirrors) serve the same chunks as ordinary .pNN files
+      urls = [base] if count <= 1 else [f"{base}.p{i:02d}" for i in range(count)]
+    part_path = final_path.with_name(final_path.name + ".part")
+    part_path.parent.mkdir(parents=True, exist_ok=True)
+
+    downloaded = 0
+    digest = hashlib.sha256()
+    last_error: Exception | None = None
+    for attempt in range(STREAM_RETRIES):
+      if self._should_abort():
+        cloudlog.warning(f"iq_maps: tile bundle download cancelled for {selector}")
+        return False
+      if attempt:
+        time.sleep(min(30.0, 2.0 * attempt))
+      try:
+        digest = hashlib.sha256()
+        resume_from = 0
+        if part_path.exists():
+          digest, resume_from = _hash_existing(part_path)
+          if expected_bytes and resume_from > expected_bytes:
+            part_path.unlink()
+            digest = hashlib.sha256()
+            resume_from = 0
+
+        # every part but the last is exactly PART_BYTES, so a byte offset maps to a part index
+        first_part = resume_from // PART_BYTES if len(urls) > 1 else 0
+        skip_in_part = resume_from - first_part * PART_BYTES if len(urls) > 1 else resume_from
+        downloaded = resume_from
+        mode = "ab" if resume_from else "wb"
+        with open(part_path, mode) as f:
+          for index in range(first_part, len(urls)):
+            if objects and not _is_hf(base_url):
+              url_headers = request_headers(base_url)
+              auth = None if url_headers else request_auth()
+              object_url, object_headers = _resolve_oid_url(
+                self.session, base_url, objects[index]["oid"], int(objects[index]["size"]),
+                url_headers)
+            else:
+              url = urls[index]
+              url_headers = request_headers(url)
+              auth = None if url_headers else request_auth()
+              # Re-resolve per part: a pre-signed LFS object URL can expire mid-download.
+              object_url, object_headers = _resolve_object_url(self.session, url, url_headers)
+            headers = dict(object_headers)
+            offset = skip_in_part if index == first_part else 0
+            if offset:
+              headers["Range"] = f"bytes={offset}-"
+            response = self.session.get(object_url, headers=headers, stream=True,
+                                        timeout=HTTP_TIMEOUT_S, auth=auth)
+            if offset and response.status_code != 206:
+              # server ignored the range: restart this whole file cleanly
+              f.close()
+              part_path.unlink(missing_ok=True)
+              raise requests.RequestException(f"range not honoured for part {index}")
+            response.raise_for_status()
+            for chunk in response.iter_content(chunk_size=CHUNK_BYTES):
+              if self._should_abort():
+                cloudlog.warning(f"iq_maps: tile bundle download cancelled for {selector}")
+                return False
+              f.write(chunk)
+              digest.update(chunk)
+              downloaded += len(chunk)
+              self._publish_progress(selector, progress_offset + downloaded, progress_total, active=True)
+        break
+      except requests.RequestException as exc:
+        last_error = exc
+        cloudlog.warning(f"iq_maps: tile bundle stream interrupted for {selector} "
+                         + f"(attempt {attempt + 1}/{STREAM_RETRIES}): {exc}")
+    else:
+      raise requests.RequestException(f"stream failed after {STREAM_RETRIES} attempts") from last_error
+
+    if expected_bytes and downloaded != expected_bytes:
+      cloudlog.error(f"iq_maps: tile bundle size mismatch for {selector}: {downloaded} != {expected_bytes}")
+      part_path.unlink(missing_ok=True)
+      return False
+    if expected_sha and digest.hexdigest() != expected_sha:
+      cloudlog.error(f"iq_maps: tile bundle sha256 mismatch for {selector}")
+      part_path.unlink(missing_ok=True)
+      return False
+
+    part_path.replace(final_path)
+    return True
+
+  def download_regions(self, selectors: list[str]) -> bool:
+    self._cancelled.clear()
+    ok = True
+    try:
+      self.mem_params.put(REQUEST_PARAM, {"regions": list(selectors)})
+      regions = None
+      base_url = ""
+      for candidate in candidate_base_urls(self.params):
+        try:
+          regions = fetch_index(candidate, self.session)
+          base_url = candidate
+          break
+        except (requests.RequestException, ValueError, json.JSONDecodeError):
+          cloudlog.warning(f"iq_maps: tile bundle index unavailable at {candidate}")
+      if regions is None:
+        cloudlog.error("iq_maps: no tile bundle host reachable")
+        return False
+
+      wanted: list[tuple[str, dict]] = []
+      for selector in selectors:
+        entry = regions.get(selector)
+        if entry is None:
+          cloudlog.warning(f"iq_maps: no tile bundle published for {selector}")
+          ok = False
+          continue
+        if region_bundle_installed(selector) and self._installed_matches(selector, entry):
+          continue
+        wanted.append((selector, entry))
+
+      progress_total = sum(int(entry.get("bytes", 0)) + int(entry.get("day_bytes", 0)) for _, entry in wanted)
+      progress_offset = 0
+      for selector, entry in wanted:
+        if self._should_abort():
+          return False
+        try:
+          if not self._download_one(selector, entry, base_url, progress_offset, progress_total):
+            ok = False
+        except (requests.RequestException, OSError):
+          cloudlog.exception(f"iq_maps: tile bundle download failed for {selector}")
+          ok = False
+        progress_offset += int(entry.get("bytes", 0)) + int(entry.get("day_bytes", 0))
+      return ok
+    finally:
+      self._publish_progress("", 0, 0, active=False)
+      try:
+        self.mem_params.remove(REQUEST_PARAM)
+      except Exception:
+        pass
+
+  @staticmethod
+  def _installed_matches(selector: str, entry: dict) -> bool:
+    manifest_path = region_bundle_dir(selector) / "manifest.json"
+    try:
+      manifest = json.loads(manifest_path.read_text())
+    except (OSError, json.JSONDecodeError):
+      return False
+    installed_sha = str(manifest.get("mbtiles", {}).get("sha256", "")).strip().lower()
+    expected_sha = str(entry.get("sha256", "")).strip().lower()
+    if not expected_sha or installed_sha != expected_sha:
+      return False
+    if entry.get("day_path"):
+      day_file = region_bundle_dir(selector) / "tiles" / "offline_day.mbtiles"
+      installed_day = str(manifest.get("mbtiles_day", {}).get("sha256", "")).strip().lower()
+      expected_day = str(entry.get("day_sha256", "")).strip().lower()
+      if not day_file.exists() or installed_day != expected_day:
+        return False
+    return True

iqpilot/iq_maps/tiles_auth.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+"""
+Copyright (c) IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
+"""
+from iqpilot._proprietary_loader import ProprietaryModuleMissing, load_private_module
+
+try:
+  load_private_module(__name__, "iqpilot_private.maps.git_auth")
+except ProprietaryModuleMissing:
+  from iqpilot.maps_private_src.git_auth import *

iqpilot/iq_maps/update_vendor_version.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
+"""
+import argparse
+import os
+import re
+import sys
+
+from iqpilot.common.basedir import BASEDIR
+from iqpilot.iq_maps import VENDOR_MAPD_PATH
+from iqpilot.iq_maps.vendor_mapd_installer import (
+  VENDOR_RELEASE_TAG,
+  sha256_of_file,
+)
+
+_RELEASE_SYMBOL = "VENDOR_RELEASE_TAG"
+_INSTALLER_SRC = os.path.join(BASEDIR, "iqpilot", "iq_maps", "vendor_mapd_installer.py")
+HASH_FILE = os.path.join(BASEDIR, "iqpilot", "iq_maps", "tests", "mapd_hash")
+_HASH_FILE = HASH_FILE
+_TAG_ASSIGN = re.compile(rf'^{_RELEASE_SYMBOL}\s*=\s*["\'][^"\']*["\']', re.MULTILINE)
+
+
+def rewrite_pinned_tag(new_tag: str) -> bool:
+  with open(_INSTALLER_SRC) as f:
+    src = f.read()
+
+  patched, count = _TAG_ASSIGN.subn(f'{_RELEASE_SYMBOL} = "{new_tag}"', src, count=1)
+  if count != 1:
+    print(f"could not locate the {_RELEASE_SYMBOL} assignment in {_INSTALLER_SRC}; nothing written")
+    return False
+
+  with open(_INSTALLER_SRC, "w") as f:
+    f.write(patched)
+  print(f"pinned {_RELEASE_SYMBOL} -> {new_tag}")
+  return True
+
+
+def refresh_hash_file() -> None:
+  digest = sha256_of_file(VENDOR_MAPD_PATH)
+  with open(_HASH_FILE, "w") as f:
+    f.write(digest)
+  print(f"wrote binary hash {digest} -> {_HASH_FILE}")
+
+
+def main() -> int:
+  parser = argparse.ArgumentParser(description="Pin a new mapd release tag and refresh its hash")
+  parser.add_argument("--new_ver", type=str, help='e.g. --new_ver "v2.1.0"')
+  args = parser.parse_args()
+
+  if not args.new_ver:
+    parser.print_help()
+    print(f'\ncurrently pinned: {VENDOR_RELEASE_TAG} (unchanged)')
+    return 0
+
+  target = args.new_ver.strip()
+  if target == VENDOR_RELEASE_TAG:
+    reply = input(f"{target} is already the pinned tag — re-run anyway? (y/N): ").strip().lower()
+    if reply != "y":
+      print("aborted; nothing changed")
+      return 0
+
+  if not rewrite_pinned_tag(target):
+    return 1
+  refresh_hash_file()
+  return 0
+
+
+if __name__ == "__main__":
+  sys.exit(main())

iqpilot/iq_maps/vendor_mapd_installer.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
+"""
+import hashlib
+import os
+import sys
+
+from iqpilot.common.basedir import BASEDIR
+from iqpilot.common.params import Params
+from iqpilot.common.spinner import Spinner
+from iqpilot.common.swaglog import cloudlog
+from iqpilot.iq_maps import VENDOR_MAPD_PATH
+import iqpilot.system.sentry as sentry
+
+VENDOR_RELEASE_TAG = "v2.0.6-iq1"
+
+_VERSION_PARAM = "MapdVersion"
+_HASH_FILE = os.path.join(BASEDIR, "iqpilot", "iq_maps", "tests", "mapd_hash")
+QUARANTINE_PATH = VENDOR_MAPD_PATH + ".quarantined"
+
+
+def sha256_of_file(path: str) -> str:
+  digest = hashlib.sha256()
+  with open(path, "rb") as handle:
+    for block in iter(lambda: handle.read(1 << 20), b""):
+      digest.update(block)
+  return digest.hexdigest()
+
+
+def stamp_vendor_version(version: str, params: Params | None = None) -> None:
+  (params or Params()).put(_VERSION_PARAM, version)
+
+
+class VendorMapdInstaller:
+  def __init__(self, spinner_ref: Spinner | None = None, params: Params | None = None):
+    self._spinner = spinner_ref
+    self._params = params if params is not None else Params()
+
+  def get_installed_version(self) -> str:
+    return str(self._params.get(_VERSION_PARAM) or "")
+
+  def verify(self) -> bool:
+    expected = self._expected_hash()
+    if not expected:
+      cloudlog.error("iq_maps: pinned mapd hash missing, vendor binary cannot be verified")
+      return False
+
+    if not os.path.isfile(VENDOR_MAPD_PATH):
+      self._say("Offline maps engine missing; it will be restored by the next update.")
+      self._params.remove(_VERSION_PARAM)
+      return False
+
+    try:
+      current = sha256_of_file(VENDOR_MAPD_PATH)
+    except OSError:
+      cloudlog.exception("iq_maps: vendor mapd unreadable")
+      return False
+
+    if current == expected:
+      stamp_vendor_version(VENDOR_RELEASE_TAG, self._params)
+      try:
+        os.remove(QUARANTINE_PATH)
+      except OSError:
+        pass
+      self._say(f"Offline maps engine verified [{VENDOR_RELEASE_TAG}]")
+      return True
+
+    cloudlog.error(f"iq_maps: vendor mapd hash {current[:12]} != pinned {expected[:12]}, quarantining")
+    self._say("Offline maps engine failed verification; quarantined until the next update.")
+    try:
+      os.replace(VENDOR_MAPD_PATH, QUARANTINE_PATH)
+    except OSError:
+      cloudlog.exception("iq_maps: vendor mapd quarantine failed")
+      return False
+    self._params.remove(_VERSION_PARAM)
+    try:
+      raise RuntimeError(f"vendor mapd hash mismatch quarantined: {current}")
+    except RuntimeError as exc:
+      sentry.init(sentry.SentryProject.SELFDRIVE)
+      sentry.capture_exception(exc)
+    return False
+
+  def _expected_hash(self) -> str:
+    try:
+      with open(_HASH_FILE) as f:
+        return f.read().strip()
+    except OSError:
+      return ""
+
+  def _say(self, text: str) -> None:
+    if self._spinner is not None:
+      self._spinner.update(text)
+
+
+if __name__ == "__main__":
+  spinner = Spinner()
+  ok = VendorMapdInstaller(spinner).verify()
+  spinner.close()
+  sys.exit(0 if ok else 1)