IQ.Pilot Prebuilt Release @ da36e21

iqpilot/iq_maps/__init__.py

@@ -0,0 +1,5 @@
+import os
+from iqpilot.common.basedir import BASEDIR
+
+VENDOR_MAPD_BIN_DIR = os.path.join(BASEDIR, "iqpilot/third_party/mapd_pfeiferj")
+VENDOR_MAPD_PATH = os.path.join(VENDOR_MAPD_BIN_DIR, "mapd")

iqpilot/iq_maps/orchestrator.py

@@ -0,0 +1,463 @@
+#!/usr/bin/env python3
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
+"""
+import platform
+import os
+import glob
+import shutil
+import signal
+import subprocess
+import threading
+import time
+from datetime import datetime
+
+import iqpilot.cereal.messaging as messaging
+from iqpilot.cereal import custom
+from iqpilot.common.params import Params
+from iqpilot.common.realtime import Ratekeeper, config_realtime_process
+from iqpilot.common.swaglog import cloudlog
+from iqpilot.selfdrive.selfdrived.alertmanager import set_offroad_alert
+from iqpilot.system.hardware.hw import Paths
+from iqpilot.iq_maps import VENDOR_MAPD_BIN_DIR, VENDOR_MAPD_PATH
+from iqpilot.iq_maps.tile_bundle_downloader import TileBundleDownloader, region_bundle_installed
+from iqpilot.iq_maps.vendor_mapd_installer import VendorMapdInstaller
+
+OfflineMapAction = custom.MapdInputType
+_region_sync_worker: threading.Thread | None = None
+
+# mapd_manager only runs offroad (process_config.only_offroad) and the onroad
+# NativeProcess("mapd", ...) is started the instant `started` flips True. If a
+# vendor-map download is in flight at that exact moment, the two `mapd`
+# binaries end up pointed at the same Paths.mapd_root() tile directory at the
+# same time: this one still downloading/writing, the onroad one already
+# mmap-reading. Manager only sends SIGINT/SIGTERM to stop mapd_manager, which
+# by default only interrupts the main thread — the background download thread
+# and the vendor `mapd` subprocess it spawned are otherwise orphaned and keep
+# writing into the tile directory the onroad reader just opened, which is what
+# was segfaulting (-12) the onroad process in a tight restart loop. The lock +
+# pidfile below make sure that subprocess is always killed (on clean shutdown
+# via the signal handlers, and on the next boot if this process itself got
+# SIGKILLed) before anything else is allowed to read the tile directory.
+_active_proc_lock = threading.Lock()
+_active_proc: subprocess.Popen | None = None
+_shutdown = threading.Event()
+# Display-tile bundles for the offline on-screen map (separate asset from mapd's routing
+# data). Downloaded after the mapd fetch in the same worker so a region selection installs
+# both, and independently restorable when only the tile bundle is missing.
+_tile_downloader: TileBundleDownloader | None = None
+
+
+def _vendor_fetch_pidfile() -> str:
+  return os.path.join(Paths.mapd_root(), ".vendor_fetch.pid")
+
+
+def _pid_is_vendor_fetch(pid: int) -> bool:
+  try:
+    with open(f"/proc/{pid}/cmdline", "rb") as f:
+      cmdline = f.read()
+  except OSError:
+    return False
+  return VENDOR_MAPD_PATH.encode() in cmdline
+
+
+def _reap_orphaned_vendor_fetch() -> None:
+  """Kill any vendor-fetch mapd subprocess left running from a prior, uncleanly-terminated run."""
+  pidfile = _vendor_fetch_pidfile()
+  try:
+    with open(pidfile) as f:
+      pid = int(f.read().strip())
+  except (OSError, ValueError):
+    return
+  try:
+    if _pid_is_vendor_fetch(pid):
+      cloudlog.warning(f"iq_maps: reaping orphaned vendor-fetch mapd pid={pid} from a prior run")
+      os.kill(pid, signal.SIGTERM)
+      for _ in range(20):
+        time.sleep(0.1)
+        if not _pid_is_vendor_fetch(pid):
+          break
+      else:
+        os.kill(pid, signal.SIGKILL)
+  except ProcessLookupError:
+    pass
+  finally:
+    try:
+      os.remove(pidfile)
+    except OSError:
+      pass
+
+
+def _kill_active_proc() -> None:
+  with _active_proc_lock:
+    proc = _active_proc
+  if proc is None or proc.poll() is not None:
+    return
+  proc.terminate()
+  try:
+    proc.wait(timeout=3)
+  except Exception:
+    proc.kill()
+    try:
+      proc.wait(timeout=2)
+    except Exception:
+      pass
+
+
+def _handle_shutdown_signal(signum, _frame) -> None:
+  cloudlog.warning(f"iq_maps: mapd_manager received signal {signum}, cleaning up vendor-fetch subprocess")
+  _shutdown.set()
+  _kill_active_proc()
+  if _tile_downloader is not None:
+    _tile_downloader.cancel()
+  worker = _region_sync_worker
+  if worker is not None and worker.is_alive():
+    worker.join(timeout=3)
+  raise SystemExit(0)
+
+
+def _install_signal_handlers() -> None:
+  signal.signal(signal.SIGINT, _handle_shutdown_signal)
+  signal.signal(signal.SIGTERM, _handle_shutdown_signal)
+
+
+def ensure_vendor_runtime() -> None:
+  # verify-only: a hash-mismatched binary is quarantined, never replaced from
+  # the network — the updater restores the checked-in one
+  try:
+    VendorMapdInstaller().verify()
+  except Exception:
+    cloudlog.exception("iq_maps: vendor runtime verification failed")
+
+params = Params()
+mem_params = Params("/dev/shm/params") if platform.system() != "Darwin" else params
+
+
+def stale_region_artifacts() -> list[str]:
+  patterns = [
+    f"{Paths.mapd_root()}/db",
+    f"{Paths.mapd_root()}/v*"
+  ]
+  stale_paths: list[str] = []
+  for pattern in patterns:
+    for match in glob.glob(pattern):
+      stale_paths.append(match)
+      if os.path.isdir(match):
+        stale_paths.extend(glob.glob(match + '/**', recursive=True))
+  if not os.path.isfile(VENDOR_MAPD_PATH):
+    stale_paths.append(VENDOR_MAPD_PATH)
+  return stale_paths
+
+
+def purge_stale_region_artifacts(stale_paths: list[str]) -> None:
+  for candidate in stale_paths:
+    if candidate.endswith('/') and os.path.isfile(candidate[:-1]):
+      candidate = candidate[:-1]
+    if os.path.islink(candidate) or os.path.isfile(candidate):
+      os.remove(candidate)
+    elif os.path.isdir(candidate):
+      shutil.rmtree(candidate, ignore_errors=False)
+
+
+def _compose_region_selector(nations: list[str], states: list[str] | None = None) -> str:
+  requested_paths: list[str] = []
+  for state_code in (states or []):
+    code = str(state_code).strip().upper()
+    if code and code != "ALL":
+      requested_paths.append(f"us_state.{code}")
+  for nation_code in (nations or []):
+    code = str(nation_code).strip().upper()
+    if code:
+      requested_paths.append(f"nation.{code}")
+  return ",".join(requested_paths)
+
+
+def _fetch_tile_bundles(region_selector: str, abort_check=None) -> None:
+  """Download the offline on-screen map display tiles for the selected regions.
+
+  Separate asset from mapd's routing data: the on-screen map's OsmOfflineProvider reads
+  raster .mbtiles bundles, so a region selection installs both when OfflineOSMaps is on."""
+  global _tile_downloader
+  if not params.get_bool("OfflineOSMaps"):
+    return
+  selectors = [part for part in region_selector.split(",") if part]
+  if not selectors:
+    return
+  try:
+    _tile_downloader = TileBundleDownloader(params=params, mem_params=mem_params, abort_check=abort_check)
+    _tile_downloader.download_regions(selectors)
+  except Exception:
+    cloudlog.exception("iq_maps: tile bundle download failed")
+  finally:
+    _tile_downloader = None
+
+
+def _drive_vendor_fetch(region_selector: str, requested_regions: dict) -> None:
+  global _active_proc
+  proc = None
+  cancelled = False
+  try:
+    mem_params.put("OSMDownloadLocations", requested_regions)
+    proc = subprocess.Popen([VENDOR_MAPD_PATH], cwd=VENDOR_MAPD_BIN_DIR,
+                            stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+                            start_new_session=True)
+    with _active_proc_lock:
+      _active_proc = proc
+    with open(_vendor_fetch_pidfile(), "w") as f:
+      f.write(str(proc.pid))
+
+    pm = messaging.PubMaster(["mapdIn"])
+    sm = messaging.SubMaster(["mapdExtendedOut"])
+    time.sleep(4.0)
+
+    for _ in range(10):
+      msg = messaging.new_message("mapdIn")
+      msg.mapdIn.type = OfflineMapAction.download
+      msg.mapdIn.str = region_selector
+      pm.send("mapdIn", msg)
+      time.sleep(0.2)
+
+    started = False
+    deadline = time.monotonic() + 3600.0
+    while time.monotonic() < deadline and not _shutdown.is_set():
+      sm.update(500)
+      dp = sm["mapdExtendedOut"].downloadProgress
+      mem_params.put("OSMDownloadProgress", {
+        "active": bool(dp.active),
+        "total_files": int(dp.totalFiles),
+        "downloaded_files": int(dp.downloadedFiles),
+      })
+      if dp.active:
+        started = True
+      elif started:
+        break
+      if not mem_params.get("OSMDownloadLocations"):
+        cancelled = True
+        cancel = messaging.new_message("mapdIn")
+        cancel.mapdIn.type = OfflineMapAction.cancelDownload
+        pm.send("mapdIn", cancel)
+        break
+    cloudlog.info(f"iq_maps: vendor map download finished for {region_selector}")
+    if not cancelled and not _shutdown.is_set():
+      # OSMDownloadLocations stays set until the finally below, so the konn3kt cancel RPC
+      # (which removes it) aborts the tile phase exactly like it cancels the mapd phase.
+      _fetch_tile_bundles(region_selector, abort_check=lambda: _shutdown.is_set() or not mem_params.get("OSMDownloadLocations"))
+  except Exception:
+    cloudlog.exception("iq_maps: vendor map download failed")
+  finally:
+    try:
+      mem_params.remove("OSMDownloadLocations")
+    except Exception:
+      pass
+    if proc is not None:
+      proc.terminate()
+      try:
+        proc.wait(timeout=5)
+      except Exception:
+        proc.kill()
+    with _active_proc_lock:
+      _active_proc = None
+    try:
+      os.remove(_vendor_fetch_pidfile())
+    except OSError:
+      pass
+
+
+def queue_region_refresh(nations: list[str], states: list[str] | None = None) -> None:
+  global _region_sync_worker
+  params.put("OsmDownloadedDate", str(datetime.now().timestamp()))
+  params.put_bool("OsmDbUpdatesCheck", False)
+
+  region_selector = _compose_region_selector(nations, states)
+  if not region_selector:
+    cloudlog.warning("iq_maps: no region selected for offline map download")
+    return
+  if _region_sync_worker is not None and _region_sync_worker.is_alive():
+    cloudlog.warning("iq_maps: vendor map download already in progress")
+    return
+
+  requested_regions = {"nations": nations, "states": states or [], "paths": region_selector}
+  cloudlog.info(f"iq_maps: starting vendor map download for {region_selector}")
+  _region_sync_worker = threading.Thread(
+    target=_drive_vendor_fetch,
+    args=(region_selector, requested_regions),
+    daemon=True,
+  )
+  _region_sync_worker.start()
+
+
+def normalize_region_selection(nations: list[str], states: list[str] | None = None) -> tuple[list[str], list[str]]:
+  normalized_nations = list(nations)
+  normalized_states = list(states or [])
+  lowered_states = {entry.lower() for entry in normalized_states}
+
+  if "US" in normalized_nations and normalized_states and "all" not in lowered_states:
+    normalized_nations = [entry for entry in normalized_nations if entry != "US"]
+  elif normalized_states:
+    normalized_states = [entry for entry in normalized_states if entry.lower() != "all"]
+
+  return normalized_nations, normalized_states
+
+
+_AUTO_RESTORE_INTERVAL_S = 1800.0
+_last_auto_restore_t = 0.0
+
+
+def region_data_missing() -> bool:
+  # a media wipe (reflash/format) can delete the downloaded region while the params
+  # that configure offline maps survive; mapd then retries the missing files forever
+  # and nothing re-downloads (stale_region_artifacts only sees leftover files)
+  if not params.get_bool("OsmLocal"):
+    return False
+  if not params.get("OsmDownloadedDate"):
+    return False
+  if glob.glob(f"{Paths.mapd_root()}/db") or glob.glob(f"{Paths.mapd_root()}/v*"):
+    return False
+  # mapd v2 stores region tiles under offline/<evenLat>/<evenLon>.tar.gz — without this
+  # check a v2 install looks perpetually wiped and re-downloads every backoff interval
+  if glob.glob(f"{Paths.mapd_root()}/offline/*/*"):
+    return False
+  country = params.get("OsmLocationName", return_default=True)
+  return bool(country)
+
+
+def configured_states() -> list[str]:
+  """Selected US states: OsmStateNames (JSON list, multi-state) wins; the legacy
+  single OsmStateName remains the fallback for pre-list configs."""
+  try:
+    states = params.get("OsmStateNames")
+    if isinstance(states, bytes):
+      import json as _json
+      states = _json.loads(states.decode("utf-8"))
+    if isinstance(states, str):
+      import json as _json
+      states = _json.loads(states)
+    if isinstance(states, list) and states:
+      return [str(s).strip().upper() for s in states if str(s).strip()]
+  except Exception:
+    pass
+  state = params.get("OsmStateName", return_default=True)
+  return [state] if state else []
+
+
+def maybe_auto_restore_region() -> None:
+  global _last_auto_restore_t
+  if not region_data_missing():
+    return
+  if _region_sync_worker is not None and _region_sync_worker.is_alive():
+    return
+  now = time.monotonic()
+  if now - _last_auto_restore_t < _AUTO_RESTORE_INTERVAL_S:
+    return
+  _last_auto_restore_t = now
+  country = params.get("OsmLocationName", return_default=True)
+  states = configured_states()
+  nations, states_filtered = normalize_region_selection([country], states)
+  cloudlog.warning(f"iq_maps: configured offline region {country}/{states} has no data on disk; auto-restoring")
+  queue_region_refresh(nations, states_filtered)
+
+
+_TILE_RESTORE_INTERVAL_S = 1800.0
+_last_tile_restore_t = 0.0
+_tile_only_worker: threading.Thread | None = None
+
+
+def _configured_region_selector() -> str:
+  country = params.get("OsmLocationName", return_default=True)
+  states = configured_states()
+  nations, states_filtered = normalize_region_selection([country] if country else [], states)
+  return _compose_region_selector(nations, states_filtered)
+
+
+def tile_bundles_missing() -> bool:
+  # covers a media wipe AND the user enabling OfflineOSMaps after the region download
+  # already ran (the vendor fetch only pulls tile bundles when the toggle is on)
+  if not params.get_bool("OfflineOSMaps"):
+    return False
+  selector = _configured_region_selector()
+  if not selector:
+    return False
+  return any(not region_bundle_installed(part) for part in selector.split(",") if part)
+
+
+def maybe_restore_tile_bundles() -> None:
+  """Tile-only download: don't re-run the whole mapd vendor fetch when only the display
+  tiles are missing."""
+  global _last_tile_restore_t, _tile_only_worker
+  if not tile_bundles_missing():
+    return
+  if _region_sync_worker is not None and _region_sync_worker.is_alive():
+    return
+  if _tile_only_worker is not None and _tile_only_worker.is_alive():
+    return
+  now = time.monotonic()
+  if now - _last_tile_restore_t < _TILE_RESTORE_INTERVAL_S:
+    return
+  _last_tile_restore_t = now
+  selector = _configured_region_selector()
+  cloudlog.warning(f"iq_maps: offline map tile bundles missing for {selector}; downloading")
+  _tile_only_worker = threading.Thread(
+    target=_fetch_tile_bundles,
+    args=(selector,),
+    kwargs={"abort_check": _shutdown.is_set},
+    daemon=True,
+  )
+  _tile_only_worker.start()
+
+
+def sync_osm_request_flags() -> None:
+  maybe_auto_restore_region()
+  maybe_restore_tile_bundles()
+  if params.get_bool("OsmDbUpdatesCheck"):
+    if _region_sync_worker is not None and _region_sync_worker.is_alive():
+      # A download is already writing into Paths.mapd_root() - deleting/rewriting
+      # files under it right now would race the writer (and any onroad mapd
+      # reader) the same way the orphaned-subprocess bug did. Wait for it to finish.
+      return
+    purge_stale_region_artifacts(stale_region_artifacts())
+    country = params.get("OsmLocationName", return_default=True)
+    states = configured_states()
+    filtered_nations, filtered_states = normalize_region_selection([country], states)
+    queue_region_refresh(filtered_nations, filtered_states)
+
+  if not mem_params.get("OSMDownloadBounds"):
+    mem_params.put("OSMDownloadBounds", "")
+
+  if not mem_params.get("LastGPSPosition"):
+    mem_params.put("LastGPSPosition", "{}")
+
+
+def run_loop():
+  ensure_vendor_runtime()
+  config_realtime_process([0, 1, 2, 3], 5)
+
+  rk = Ratekeeper(1, print_delay_threshold=None)
+
+  try:
+    os.mkdir(Paths.mapd_root())
+  except FileExistsError:
+    pass
+  except PermissionError:
+    cloudlog.exception(f"iq_maps: failed to make {Paths.mapd_root()}")
+
+  # A prior run that got SIGKILLed (or crashed) may have left its vendor-fetch
+  # mapd subprocess running and still writing into Paths.mapd_root(); clear it
+  # before anything (including the onroad mapd, once `started` flips) reads
+  # from that directory. Signal handlers cover the graceful-shutdown path.
+  _reap_orphaned_vendor_fetch()
+  _install_signal_handlers()
+
+  while not _shutdown.is_set():
+    show_alert = stale_region_artifacts() and params.get_bool("OsmLocal")
+    set_offroad_alert("Offroad_OSMUpdateRequired", show_alert, "This alert will be cleared when new maps are downloaded.")
+
+    sync_osm_request_flags()
+    rk.keep_time()
+
+
+def main():
+  run_loop()
+
+
+if __name__ == "__main__":
+  main()

iqpilot/iq_maps/tile_bundle_downloader.py

@@ -0,0 +1,355 @@
+#!/usr/bin/env python3
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
+
+Downloads per-region raster display-tile bundles (.mbtiles) for the offline on-screen map.
+
+These are a separate asset from mapd's routing/speed-limit data: mapd pulls OSM way tiles
+into Paths.mapd_root(), while the on-screen map (OsmOfflineProvider) reads raster .mbtiles
+from offline_map_root()/regions/<selector>/tiles/offline.mbtiles. Bundles are built per
+state/nation by scripts/iqpilot/build_state_tile_bundles.py and hosted behind a static base
+URL that serves:
+
+  <base>/index.json                       {"version": 1, "regions": {<selector>: entry}}
+  <base>/<entry["path"]>                  the raster .mbtiles for that region
+
+Entry fields: path, bytes, sha256, bounds ("minLon,minLat,maxLon,maxLat"), minzoom, maxzoom.
+Selectors match the mapd region menu naming: us_state.CA, nation.US.
+"""
+import hashlib
+import json
+import platform
+import threading
+import time
+from pathlib import Path
+
+import requests
+
+from iqpilot.common.params import Params
+from iqpilot.common.swaglog import cloudlog
+from iqpilot.ui.onroad.offline_tiles import offline_map_root
+
+# Proprietary auth + hosted endpoints (gitea raw with an embedded read-only PAT, same
+# pattern as the model selector). Optional: without the private bundle the downloader
+# still works anonymously against OfflineTilesBaseUrl (e.g. a public R2 bucket).
+try:
+  from iqpilot.iq_maps.tiles_auth import get_base_urls as _private_base_urls, get_requests_auth as _private_auth
+except Exception:  # ProprietaryModuleMissing or import errors in stripped builds
+  _private_base_urls = None
+  _private_auth = None
+
+# R2 bucket iqnav behind the public custom domain (see scripts/iqpilot/tile_factory/r2_sync_watch.py)
+DEFAULT_TILE_BUNDLE_BASE_URL = "https://maps.konn3kt.com/iqosmd/v1"
+BASE_URL_PARAM = "OfflineTilesBaseUrl"
+PROGRESS_PARAM = "OfflineTilesDownloadProgress"
+REQUEST_PARAM = "OfflineTilesDownloadRequest"
+CHUNK_BYTES = 1 << 20
+HTTP_TIMEOUT_S = 30.0
+STREAM_RETRIES = 8
+
+
+def candidate_base_urls(params: Params) -> list[str]:
+  """Hosts to try in order: user/param override first, then the embedded private
+  endpoints (gitea raw), then the public default."""
+  override = params.get(BASE_URL_PARAM)
+  if isinstance(override, bytes):
+    override = override.decode("utf-8", errors="ignore")
+  override = (override or "").strip()
+  if override:
+    return [override.rstrip("/")]
+  urls: list[str] = []
+  if _private_base_urls is not None:
+    try:
+      urls.extend(url.rstrip("/") for url in _private_base_urls())
+    except Exception:
+      pass
+  urls.append(DEFAULT_TILE_BUNDLE_BASE_URL)
+  return urls
+
+
+def request_auth() -> tuple[str, str] | None:
+  if _private_auth is None:
+    return None
+  try:
+    return _private_auth()
+  except Exception:
+    return None
+
+
+def fetch_index(base_url: str, session: requests.Session) -> dict:
+  response = session.get(f"{base_url}/index.json", timeout=HTTP_TIMEOUT_S, auth=request_auth())
+  response.raise_for_status()
+  index = response.json()
+  regions = index.get("regions")
+  if not isinstance(regions, dict):
+    raise ValueError("tile bundle index has no regions")
+  return regions
+
+
+def region_bundle_dir(selector: str) -> Path:
+  return offline_map_root() / "regions" / selector
+
+
+def region_bundle_path(selector: str) -> Path:
+  return region_bundle_dir(selector) / "tiles" / "offline.mbtiles"
+
+
+def region_bundle_installed(selector: str) -> bool:
+  return region_bundle_path(selector).exists()
+
+
+def installed_region_selectors() -> list[str]:
+  regions_root = offline_map_root() / "regions"
+  if not regions_root.exists():
+    return []
+  return sorted(
+    child.name for child in regions_root.iterdir()
+    if child.is_dir() and (child / "tiles" / "offline.mbtiles").exists()
+  )
+
+
+def _hash_existing(path: Path) -> tuple["hashlib._Hash", int]:
+  digest = hashlib.sha256()
+  size = 0
+  with open(path, "rb") as f:
+    while True:
+      chunk = f.read(CHUNK_BYTES)
+      if not chunk:
+        break
+      digest.update(chunk)
+      size += len(chunk)
+  return digest, size
+
+
+def _write_manifest(selector: str, entry: dict) -> None:
+  manifest = {
+    "region": selector,
+    "version": entry.get("version", ""),
+    "mbtiles": {
+      "bounds": entry.get("bounds", ""),
+      "minzoom": entry.get("minzoom"),
+      "maxzoom": entry.get("maxzoom"),
+      "bytes": entry.get("bytes"),
+      "sha256": entry.get("sha256", ""),
+    },
+  }
+  if entry.get("day_path"):
+    manifest["mbtiles_day"] = {
+      "bytes": entry.get("day_bytes"),
+      "sha256": entry.get("day_sha256", ""),
+    }
+  manifest_path = region_bundle_dir(selector) / "manifest.json"
+  manifest_path.parent.mkdir(parents=True, exist_ok=True)
+  manifest_path.write_text(json.dumps(manifest, indent=2))
+
+
+class TileBundleDownloader:
+  """Streams region bundles to disk with resume + sha256 verify + atomic install.
+
+  Cancellation matches the mapd flow: the caller sets REQUEST_PARAM in mem params while a
+  download runs; removing it (konn3kt cancel RPC or settings) aborts between chunks. The
+  partial .part file is kept so a retry resumes instead of restarting.
+  """
+
+  def __init__(self, params: Params | None = None, mem_params: Params | None = None,
+               abort_check=None):
+    self.params = params if params is not None else Params()
+    if mem_params is not None:
+      self.mem_params = mem_params
+    else:
+      self.mem_params = Params("/dev/shm/params") if platform.system() != "Darwin" else self.params
+    self.session = requests.Session()
+    self._cancelled = threading.Event()
+    # optional external cancel signal, e.g. the orchestrator's OSMDownloadLocations removal
+    self._abort_check = abort_check
+
+  def cancel(self) -> None:
+    self._cancelled.set()
+
+  def _should_abort(self) -> bool:
+    if self._cancelled.is_set():
+      return True
+    if not self.mem_params.get(REQUEST_PARAM):
+      # request flag was removed out from under us -> user cancelled
+      self._cancelled.set()
+      return True
+    if self._abort_check is not None and self._abort_check():
+      self._cancelled.set()
+      return True
+    return False
+
+  def _publish_progress(self, region: str, downloaded: int, total: int, active: bool) -> None:
+    self.mem_params.put(PROGRESS_PARAM, {
+      "active": active,
+      "region": region,
+      "downloaded_bytes": int(downloaded),
+      "total_bytes": int(total),
+    })
+
+  def _download_one(self, selector: str, entry: dict, base_url: str,
+                    progress_offset: int, progress_total: int) -> bool:
+    """Download a region: the night bundle, plus the optional day-style variant."""
+    night_path = region_bundle_path(selector)
+    ok = self._download_file(
+      selector, base_url, entry["path"], int(entry.get("bytes", 0)),
+      str(entry.get("sha256", "")).strip().lower(), night_path,
+      progress_offset, progress_total,
+    )
+    if not ok:
+      return False
+    if entry.get("day_path"):
+      day_ok = self._download_file(
+        selector, base_url, entry["day_path"], int(entry.get("day_bytes", 0)),
+        str(entry.get("day_sha256", "")).strip().lower(),
+        night_path.with_name("offline_day.mbtiles"),
+        progress_offset + int(entry.get("bytes", 0)), progress_total,
+      )
+      if not day_ok:
+        # the night set is complete and usable; a failed day variant retries next pass
+        cloudlog.warning(f"iq_maps: day-style bundle failed for {selector}; night set installed")
+    # manifest last: bounds drive region matching, so it must describe installed files
+    _write_manifest(selector, entry)
+    cloudlog.info(f"iq_maps: installed tile bundle {selector}")
+    return True
+
+  def _download_file(self, selector: str, base_url: str, remote_path: str, expected_bytes: int,
+                     expected_sha: str, final_path: Path,
+                     progress_offset: int, progress_total: int) -> bool:
+    url = f"{base_url}/{remote_path.lstrip('/')}"
+    part_path = final_path.with_name(final_path.name + ".part")
+    part_path.parent.mkdir(parents=True, exist_ok=True)
+
+    # A cellular/hotspot link routinely kills a multi-hundred-MB stream mid-flight; retry
+    # each interruption from the current .part offset instead of failing the whole region.
+    downloaded = 0
+    digest = hashlib.sha256()
+    last_error: Exception | None = None
+    for attempt in range(STREAM_RETRIES):
+      if self._should_abort():
+        cloudlog.warning(f"iq_maps: tile bundle download cancelled for {selector}")
+        return False
+      if attempt:
+        time.sleep(min(30.0, 2.0 * attempt))
+      try:
+        digest = hashlib.sha256()
+        resume_from = 0
+        if part_path.exists():
+          digest, resume_from = _hash_existing(part_path)
+          if expected_bytes and resume_from > expected_bytes:
+            part_path.unlink()
+            digest = hashlib.sha256()
+            resume_from = 0
+
+        headers = {"Range": f"bytes={resume_from}-"} if resume_from else {}
+        auth = request_auth()
+        response = self.session.get(url, headers=headers, stream=True, timeout=HTTP_TIMEOUT_S, auth=auth)
+        if resume_from and response.status_code != 206:
+          # server ignored the Range request -> restart from scratch
+          digest = hashlib.sha256()
+          resume_from = 0
+          part_path.unlink(missing_ok=True)
+          if response.status_code == 416:
+            response = self.session.get(url, stream=True, timeout=HTTP_TIMEOUT_S, auth=auth)
+        response.raise_for_status()
+
+        downloaded = resume_from
+        mode = "ab" if resume_from else "wb"
+        with open(part_path, mode) as f:
+          for chunk in response.iter_content(chunk_size=CHUNK_BYTES):
+            if self._should_abort():
+              cloudlog.warning(f"iq_maps: tile bundle download cancelled for {selector}")
+              return False
+            f.write(chunk)
+            digest.update(chunk)
+            downloaded += len(chunk)
+            self._publish_progress(selector, progress_offset + downloaded, progress_total, active=True)
+        break
+      except requests.RequestException as exc:
+        last_error = exc
+        cloudlog.warning(f"iq_maps: tile bundle stream interrupted for {selector} "
+                         + f"(attempt {attempt + 1}/{STREAM_RETRIES}): {exc}")
+    else:
+      raise requests.RequestException(f"stream failed after {STREAM_RETRIES} attempts") from last_error
+
+    if expected_bytes and downloaded != expected_bytes:
+      cloudlog.error(f"iq_maps: tile bundle size mismatch for {selector}: {downloaded} != {expected_bytes}")
+      part_path.unlink(missing_ok=True)
+      return False
+    if expected_sha and digest.hexdigest() != expected_sha:
+      cloudlog.error(f"iq_maps: tile bundle sha256 mismatch for {selector}")
+      part_path.unlink(missing_ok=True)
+      return False
+
+    part_path.replace(final_path)
+    return True
+
+  def download_regions(self, selectors: list[str]) -> bool:
+    """Download the display-tile bundles for the given region selectors. Returns True if all
+    requested bundles are installed and current when done."""
+    self._cancelled.clear()
+    ok = True
+    try:
+      self.mem_params.put(REQUEST_PARAM, {"regions": list(selectors)})
+      regions = None
+      base_url = ""
+      for candidate in candidate_base_urls(self.params):
+        try:
+          regions = fetch_index(candidate, self.session)
+          base_url = candidate
+          break
+        except (requests.RequestException, ValueError, json.JSONDecodeError):
+          cloudlog.warning(f"iq_maps: tile bundle index unavailable at {candidate}")
+      if regions is None:
+        cloudlog.error("iq_maps: no tile bundle host reachable")
+        return False
+
+      wanted: list[tuple[str, dict]] = []
+      for selector in selectors:
+        entry = regions.get(selector)
+        if entry is None:
+          cloudlog.warning(f"iq_maps: no tile bundle published for {selector}")
+          ok = False
+          continue
+        if region_bundle_installed(selector) and self._installed_matches(selector, entry):
+          continue
+        wanted.append((selector, entry))
+
+      progress_total = sum(int(entry.get("bytes", 0)) + int(entry.get("day_bytes", 0)) for _, entry in wanted)
+      progress_offset = 0
+      for selector, entry in wanted:
+        if self._should_abort():
+          return False
+        try:
+          if not self._download_one(selector, entry, base_url, progress_offset, progress_total):
+            ok = False
+        except (requests.RequestException, OSError):
+          cloudlog.exception(f"iq_maps: tile bundle download failed for {selector}")
+          ok = False
+        progress_offset += int(entry.get("bytes", 0)) + int(entry.get("day_bytes", 0))
+      return ok
+    finally:
+      self._publish_progress("", 0, 0, active=False)
+      try:
+        self.mem_params.remove(REQUEST_PARAM)
+      except Exception:
+        pass
+
+  @staticmethod
+  def _installed_matches(selector: str, entry: dict) -> bool:
+    manifest_path = region_bundle_dir(selector) / "manifest.json"
+    try:
+      manifest = json.loads(manifest_path.read_text())
+    except (OSError, json.JSONDecodeError):
+      return False
+    installed_sha = str(manifest.get("mbtiles", {}).get("sha256", "")).strip().lower()
+    expected_sha = str(entry.get("sha256", "")).strip().lower()
+    if not expected_sha or installed_sha != expected_sha:
+      return False
+    if entry.get("day_path"):
+      # a published day variant must be installed and current too
+      day_file = region_bundle_dir(selector) / "tiles" / "offline_day.mbtiles"
+      installed_day = str(manifest.get("mbtiles_day", {}).get("sha256", "")).strip().lower()
+      expected_day = str(entry.get("day_sha256", "")).strip().lower()
+      if not day_file.exists() or installed_day != expected_day:
+        return False
+    return True

iqpilot/iq_maps/tiles_auth.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+"""
+Copyright (c) IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
+"""
+from iqpilot._proprietary_loader import ProprietaryModuleMissing, load_private_module
+
+try:
+  load_private_module(__name__, "iqpilot_private.maps.git_auth")
+except ProprietaryModuleMissing:
+  from iqpilot.maps_private_src.git_auth import *  # noqa: F403

iqpilot/iq_maps/update_vendor_version.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
+
+Maintainer utility: pin a new mapd release tag and refresh the checked-in binary
+hash. Not used at runtime. Binaries come from the gitlvb teal/mapd CI (built
+against teal/gomsgq) — drop the artifact at third_party/mapd_pfeiferj/mapd, then
+run this so the hash pin moves in the same commit.
+"""
+import argparse
+import os
+import re
+import sys
+
+from iqpilot.common.basedir import BASEDIR
+from iqpilot.iq_maps import VENDOR_MAPD_PATH
+from iqpilot.iq_maps.vendor_mapd_installer import (
+  VENDOR_RELEASE_TAG,
+  sha256_of_file,
+)
+
+_RELEASE_SYMBOL = "VENDOR_RELEASE_TAG"
+_INSTALLER_SRC = os.path.join(BASEDIR, "iqpilot", "iq_maps", "vendor_mapd_installer.py")
+# public: the checked-in hash the version test compares the installed binary against
+HASH_FILE = os.path.join(BASEDIR, "iqpilot", "iq_maps", "tests", "mapd_hash")
+_HASH_FILE = HASH_FILE
+_TAG_ASSIGN = re.compile(rf'^{_RELEASE_SYMBOL}\s*=\s*["\'][^"\']*["\']', re.MULTILINE)
+
+
+def rewrite_pinned_tag(new_tag: str) -> bool:
+  with open(_INSTALLER_SRC) as f:
+    src = f.read()
+
+  patched, count = _TAG_ASSIGN.subn(f'{_RELEASE_SYMBOL} = "{new_tag}"', src, count=1)
+  if count != 1:
+    print(f"could not locate the {_RELEASE_SYMBOL} assignment in {_INSTALLER_SRC}; nothing written")
+    return False
+
+  with open(_INSTALLER_SRC, "w") as f:
+    f.write(patched)
+  print(f"pinned {_RELEASE_SYMBOL} -> {new_tag}")
+  return True
+
+
+def refresh_hash_file() -> None:
+  digest = sha256_of_file(VENDOR_MAPD_PATH)
+  with open(_HASH_FILE, "w") as f:
+    f.write(digest)
+  print(f"wrote binary hash {digest} -> {_HASH_FILE}")
+
+
+def main() -> int:
+  parser = argparse.ArgumentParser(description="Pin a new mapd release tag and refresh its hash")
+  parser.add_argument("--new_ver", type=str, help='e.g. --new_ver "v2.1.0"')
+  args = parser.parse_args()
+
+  if not args.new_ver:
+    parser.print_help()
+    print(f'\ncurrently pinned: {VENDOR_RELEASE_TAG} (unchanged)')
+    return 0
+
+  target = args.new_ver.strip()
+  if target == VENDOR_RELEASE_TAG:
+    reply = input(f"{target} is already the pinned tag — re-run anyway? (y/N): ").strip().lower()
+    if reply != "y":
+      print("aborted; nothing changed")
+      return 0
+
+  if not rewrite_pinned_tag(target):
+    return 1
+  refresh_hash_file()
+  return 0
+
+
+if __name__ == "__main__":
+  sys.exit(main())

iqpilot/iq_maps/vendor_mapd_installer.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
+
+Verifies the vendored `mapd` routing binary authored by Jacob Pfeifer
+(github.com/pfeiferj/mapd), built from the gitlvb teal/mapd fork against
+teal/gomsgq. The only accepted binary is the checked-in one matching the pinned
+hash; nothing is ever downloaded at runtime. Jacob's stock release build embeds
+a 15-reader msgq header layout — on this fork (NUM_READERS=32) its registration
+writes land inside other processes' reader slots, so a wrong binary is
+quarantined rather than left where manager could start it.
+"""
+import hashlib
+import os
+import sys
+
+from iqpilot.common.basedir import BASEDIR
+from iqpilot.common.params import Params
+from iqpilot.common.spinner import Spinner
+from iqpilot.common.swaglog import cloudlog
+from iqpilot.iq_maps import VENDOR_MAPD_PATH
+import iqpilot.system.sentry as sentry
+
+VENDOR_RELEASE_TAG = "v2.0.6-iq1"
+
+_VERSION_PARAM = "MapdVersion"
+_HASH_FILE = os.path.join(BASEDIR, "iqpilot", "iq_maps", "tests", "mapd_hash")
+QUARANTINE_PATH = VENDOR_MAPD_PATH + ".quarantined"
+
+
+def sha256_of_file(path: str) -> str:
+  """Hex SHA-256 digest of a file on disk."""
+  digest = hashlib.sha256()
+  with open(path, "rb") as handle:
+    for block in iter(lambda: handle.read(1 << 20), b""):
+      digest.update(block)
+  return digest.hexdigest()
+
+
+def stamp_vendor_version(version: str, params: Params | None = None) -> None:
+  (params or Params()).put(_VERSION_PARAM, version)
+
+
+class VendorMapdInstaller:
+  def __init__(self, spinner_ref: Spinner | None = None, params: Params | None = None):
+    self._spinner = spinner_ref
+    self._params = params if params is not None else Params()
+
+  def get_installed_version(self) -> str:
+    return str(self._params.get(_VERSION_PARAM) or "")
+
+  def verify(self) -> bool:
+    """True iff the on-disk binary matches the pinned hash; quarantines a wrong one."""
+    expected = self._expected_hash()
+    if not expected:
+      cloudlog.error("iq_maps: pinned mapd hash missing, vendor binary cannot be verified")
+      return False
+
+    if not os.path.isfile(VENDOR_MAPD_PATH):
+      # the binary is a tracked file: the updater/bundle restores it
+      self._say("Offline maps engine missing; it will be restored by the next update.")
+      self._params.remove(_VERSION_PARAM)
+      return False
+
+    try:
+      current = sha256_of_file(VENDOR_MAPD_PATH)
+    except OSError:
+      cloudlog.exception("iq_maps: vendor mapd unreadable")
+      return False
+
+    if current == expected:
+      stamp_vendor_version(VENDOR_RELEASE_TAG, self._params)
+      try:
+        os.remove(QUARANTINE_PATH)
+      except OSError:
+        pass
+      self._say(f"Offline maps engine verified [{VENDOR_RELEASE_TAG}]")
+      return True
+
+    # a foreign binary — e.g. a stock release download from the retired fetch
+    # path — must never run: quarantine it where manager can't start it
+    cloudlog.error(f"iq_maps: vendor mapd hash {current[:12]} != pinned {expected[:12]}, quarantining")
+    self._say("Offline maps engine failed verification; quarantined until the next update.")
+    try:
+      os.replace(VENDOR_MAPD_PATH, QUARANTINE_PATH)
+    except OSError:
+      cloudlog.exception("iq_maps: vendor mapd quarantine failed")
+      return False
+    self._params.remove(_VERSION_PARAM)
+    try:
+      raise RuntimeError(f"vendor mapd hash mismatch quarantined: {current}")
+    except RuntimeError as exc:
+      sentry.init(sentry.SentryProject.SELFDRIVE)
+      sentry.capture_exception(exc)
+    return False
+
+  def _expected_hash(self) -> str:
+    try:
+      with open(_HASH_FILE) as f:
+        return f.read().strip()
+    except OSError:
+      return ""
+
+  def _say(self, text: str) -> None:
+    if self._spinner is not None:
+      self._spinner.update(text)
+
+
+if __name__ == "__main__":
+  spinner = Spinner()
+  ok = VendorMapdInstaller(spinner).verify()
+  spinner.close()
+  sys.exit(0 if ok else 1)