1
0
forked from IQ.Lvbs/IQ.Pilot

IQ.Pilot Prebuilt Release @ ab07000

This commit is contained in:
IQ.Lvbs history cleanup
2026-08-22 23:42:42 -05:00
commit 9f9c9a70cc
3729 changed files with 778697 additions and 0 deletions

View File

@@ -0,0 +1,5 @@
import os
from openpilot.common.basedir import BASEDIR
VENDOR_MAPD_BIN_DIR = os.path.join(BASEDIR, "third_party/mapd_pfeiferj")
VENDOR_MAPD_PATH = os.path.join(VENDOR_MAPD_BIN_DIR, "mapd")

463
iqpilot/iq_maps/orchestrator.py Executable file
View File

@@ -0,0 +1,463 @@
#!/usr/bin/env python3
"""
Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
"""
import platform
import os
import glob
import shutil
import signal
import subprocess
import threading
import time
from datetime import datetime
import cereal.messaging as messaging
from cereal import custom
from openpilot.common.params import Params
from openpilot.common.realtime import Ratekeeper, config_realtime_process
from openpilot.common.swaglog import cloudlog
from openpilot.selfdrive.selfdrived.alertmanager import set_offroad_alert
from openpilot.system.hardware.hw import Paths
from openpilot.iqpilot.iq_maps import VENDOR_MAPD_BIN_DIR, VENDOR_MAPD_PATH
from openpilot.iqpilot.iq_maps.tile_bundle_downloader import TileBundleDownloader, region_bundle_installed
from openpilot.iqpilot.iq_maps.vendor_mapd_installer import VendorMapdInstaller
OfflineMapAction = custom.MapdInputType
_region_sync_worker: threading.Thread | None = None
# mapd_manager only runs offroad (process_config.only_offroad) and the onroad
# NativeProcess("mapd", ...) is started the instant `started` flips True. If a
# vendor-map download is in flight at that exact moment, the two `mapd`
# binaries end up pointed at the same Paths.mapd_root() tile directory at the
# same time: this one still downloading/writing, the onroad one already
# mmap-reading. Manager only sends SIGINT/SIGTERM to stop mapd_manager, which
# by default only interrupts the main thread — the background download thread
# and the vendor `mapd` subprocess it spawned are otherwise orphaned and keep
# writing into the tile directory the onroad reader just opened, which is what
# was segfaulting (-12) the onroad process in a tight restart loop. The lock +
# pidfile below make sure that subprocess is always killed (on clean shutdown
# via the signal handlers, and on the next boot if this process itself got
# SIGKILLed) before anything else is allowed to read the tile directory.
_active_proc_lock = threading.Lock()
_active_proc: subprocess.Popen | None = None
_shutdown = threading.Event()
# Display-tile bundles for the offline on-screen map (separate asset from mapd's routing
# data). Downloaded after the mapd fetch in the same worker so a region selection installs
# both, and independently restorable when only the tile bundle is missing.
_tile_downloader: TileBundleDownloader | None = None
def _vendor_fetch_pidfile() -> str:
return os.path.join(Paths.mapd_root(), ".vendor_fetch.pid")
def _pid_is_vendor_fetch(pid: int) -> bool:
try:
with open(f"/proc/{pid}/cmdline", "rb") as f:
cmdline = f.read()
except OSError:
return False
return VENDOR_MAPD_PATH.encode() in cmdline
def _reap_orphaned_vendor_fetch() -> None:
"""Kill any vendor-fetch mapd subprocess left running from a prior, uncleanly-terminated run."""
pidfile = _vendor_fetch_pidfile()
try:
with open(pidfile) as f:
pid = int(f.read().strip())
except (OSError, ValueError):
return
try:
if _pid_is_vendor_fetch(pid):
cloudlog.warning(f"iq_maps: reaping orphaned vendor-fetch mapd pid={pid} from a prior run")
os.kill(pid, signal.SIGTERM)
for _ in range(20):
time.sleep(0.1)
if not _pid_is_vendor_fetch(pid):
break
else:
os.kill(pid, signal.SIGKILL)
except ProcessLookupError:
pass
finally:
try:
os.remove(pidfile)
except OSError:
pass
def _kill_active_proc() -> None:
with _active_proc_lock:
proc = _active_proc
if proc is None or proc.poll() is not None:
return
proc.terminate()
try:
proc.wait(timeout=3)
except Exception:
proc.kill()
try:
proc.wait(timeout=2)
except Exception:
pass
def _handle_shutdown_signal(signum, _frame) -> None:
cloudlog.warning(f"iq_maps: mapd_manager received signal {signum}, cleaning up vendor-fetch subprocess")
_shutdown.set()
_kill_active_proc()
if _tile_downloader is not None:
_tile_downloader.cancel()
worker = _region_sync_worker
if worker is not None and worker.is_alive():
worker.join(timeout=3)
raise SystemExit(0)
def _install_signal_handlers() -> None:
signal.signal(signal.SIGINT, _handle_shutdown_signal)
signal.signal(signal.SIGTERM, _handle_shutdown_signal)
def ensure_vendor_runtime() -> None:
# verify-only: a hash-mismatched binary is quarantined, never replaced from
# the network — the updater restores the checked-in one
try:
VendorMapdInstaller().verify()
except Exception:
cloudlog.exception("iq_maps: vendor runtime verification failed")
params = Params()
mem_params = Params("/dev/shm/params") if platform.system() != "Darwin" else params
def stale_region_artifacts() -> list[str]:
patterns = [
f"{Paths.mapd_root()}/db",
f"{Paths.mapd_root()}/v*"
]
stale_paths: list[str] = []
for pattern in patterns:
for match in glob.glob(pattern):
stale_paths.append(match)
if os.path.isdir(match):
stale_paths.extend(glob.glob(match + '/**', recursive=True))
if not os.path.isfile(VENDOR_MAPD_PATH):
stale_paths.append(VENDOR_MAPD_PATH)
return stale_paths
def purge_stale_region_artifacts(stale_paths: list[str]) -> None:
for candidate in stale_paths:
if candidate.endswith('/') and os.path.isfile(candidate[:-1]):
candidate = candidate[:-1]
if os.path.islink(candidate) or os.path.isfile(candidate):
os.remove(candidate)
elif os.path.isdir(candidate):
shutil.rmtree(candidate, ignore_errors=False)
def _compose_region_selector(nations: list[str], states: list[str] | None = None) -> str:
requested_paths: list[str] = []
for state_code in (states or []):
code = str(state_code).strip().upper()
if code and code != "ALL":
requested_paths.append(f"us_state.{code}")
for nation_code in (nations or []):
code = str(nation_code).strip().upper()
if code:
requested_paths.append(f"nation.{code}")
return ",".join(requested_paths)
def _fetch_tile_bundles(region_selector: str, abort_check=None) -> None:
"""Download the offline on-screen map display tiles for the selected regions.
Separate asset from mapd's routing data: the on-screen map's OsmOfflineProvider reads
raster .mbtiles bundles, so a region selection installs both when OfflineOSMaps is on."""
global _tile_downloader
if not params.get_bool("OfflineOSMaps"):
return
selectors = [part for part in region_selector.split(",") if part]
if not selectors:
return
try:
_tile_downloader = TileBundleDownloader(params=params, mem_params=mem_params, abort_check=abort_check)
_tile_downloader.download_regions(selectors)
except Exception:
cloudlog.exception("iq_maps: tile bundle download failed")
finally:
_tile_downloader = None
def _drive_vendor_fetch(region_selector: str, requested_regions: dict) -> None:
global _active_proc
proc = None
cancelled = False
try:
mem_params.put("OSMDownloadLocations", requested_regions)
proc = subprocess.Popen([VENDOR_MAPD_PATH], cwd=VENDOR_MAPD_BIN_DIR,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
start_new_session=True)
with _active_proc_lock:
_active_proc = proc
with open(_vendor_fetch_pidfile(), "w") as f:
f.write(str(proc.pid))
pm = messaging.PubMaster(["mapdIn"])
sm = messaging.SubMaster(["mapdExtendedOut"])
time.sleep(4.0)
for _ in range(10):
msg = messaging.new_message("mapdIn")
msg.mapdIn.type = OfflineMapAction.download
msg.mapdIn.str = region_selector
pm.send("mapdIn", msg)
time.sleep(0.2)
started = False
deadline = time.monotonic() + 3600.0
while time.monotonic() < deadline and not _shutdown.is_set():
sm.update(500)
dp = sm["mapdExtendedOut"].downloadProgress
mem_params.put("OSMDownloadProgress", {
"active": bool(dp.active),
"total_files": int(dp.totalFiles),
"downloaded_files": int(dp.downloadedFiles),
})
if dp.active:
started = True
elif started:
break
if not mem_params.get("OSMDownloadLocations"):
cancelled = True
cancel = messaging.new_message("mapdIn")
cancel.mapdIn.type = OfflineMapAction.cancelDownload
pm.send("mapdIn", cancel)
break
cloudlog.info(f"iq_maps: vendor map download finished for {region_selector}")
if not cancelled and not _shutdown.is_set():
# OSMDownloadLocations stays set until the finally below, so the konn3kt cancel RPC
# (which removes it) aborts the tile phase exactly like it cancels the mapd phase.
_fetch_tile_bundles(region_selector, abort_check=lambda: _shutdown.is_set() or not mem_params.get("OSMDownloadLocations"))
except Exception:
cloudlog.exception("iq_maps: vendor map download failed")
finally:
try:
mem_params.remove("OSMDownloadLocations")
except Exception:
pass
if proc is not None:
proc.terminate()
try:
proc.wait(timeout=5)
except Exception:
proc.kill()
with _active_proc_lock:
_active_proc = None
try:
os.remove(_vendor_fetch_pidfile())
except OSError:
pass
def queue_region_refresh(nations: list[str], states: list[str] | None = None) -> None:
global _region_sync_worker
params.put("OsmDownloadedDate", str(datetime.now().timestamp()))
params.put_bool("OsmDbUpdatesCheck", False)
region_selector = _compose_region_selector(nations, states)
if not region_selector:
cloudlog.warning("iq_maps: no region selected for offline map download")
return
if _region_sync_worker is not None and _region_sync_worker.is_alive():
cloudlog.warning("iq_maps: vendor map download already in progress")
return
requested_regions = {"nations": nations, "states": states or [], "paths": region_selector}
cloudlog.info(f"iq_maps: starting vendor map download for {region_selector}")
_region_sync_worker = threading.Thread(
target=_drive_vendor_fetch,
args=(region_selector, requested_regions),
daemon=True,
)
_region_sync_worker.start()
def normalize_region_selection(nations: list[str], states: list[str] | None = None) -> tuple[list[str], list[str]]:
normalized_nations = list(nations)
normalized_states = list(states or [])
lowered_states = {entry.lower() for entry in normalized_states}
if "US" in normalized_nations and normalized_states and "all" not in lowered_states:
normalized_nations = [entry for entry in normalized_nations if entry != "US"]
elif normalized_states:
normalized_states = [entry for entry in normalized_states if entry.lower() != "all"]
return normalized_nations, normalized_states
_AUTO_RESTORE_INTERVAL_S = 1800.0
_last_auto_restore_t = 0.0
def region_data_missing() -> bool:
# a media wipe (reflash/format) can delete the downloaded region while the params
# that configure offline maps survive; mapd then retries the missing files forever
# and nothing re-downloads (stale_region_artifacts only sees leftover files)
if not params.get_bool("OsmLocal"):
return False
if not params.get("OsmDownloadedDate"):
return False
if glob.glob(f"{Paths.mapd_root()}/db") or glob.glob(f"{Paths.mapd_root()}/v*"):
return False
# mapd v2 stores region tiles under offline/<evenLat>/<evenLon>.tar.gz — without this
# check a v2 install looks perpetually wiped and re-downloads every backoff interval
if glob.glob(f"{Paths.mapd_root()}/offline/*/*"):
return False
country = params.get("OsmLocationName", return_default=True)
return bool(country)
def configured_states() -> list[str]:
"""Selected US states: OsmStateNames (JSON list, multi-state) wins; the legacy
single OsmStateName remains the fallback for pre-list configs."""
try:
states = params.get("OsmStateNames")
if isinstance(states, bytes):
import json as _json
states = _json.loads(states.decode("utf-8"))
if isinstance(states, str):
import json as _json
states = _json.loads(states)
if isinstance(states, list) and states:
return [str(s).strip().upper() for s in states if str(s).strip()]
except Exception:
pass
state = params.get("OsmStateName", return_default=True)
return [state] if state else []
def maybe_auto_restore_region() -> None:
global _last_auto_restore_t
if not region_data_missing():
return
if _region_sync_worker is not None and _region_sync_worker.is_alive():
return
now = time.monotonic()
if now - _last_auto_restore_t < _AUTO_RESTORE_INTERVAL_S:
return
_last_auto_restore_t = now
country = params.get("OsmLocationName", return_default=True)
states = configured_states()
nations, states_filtered = normalize_region_selection([country], states)
cloudlog.warning(f"iq_maps: configured offline region {country}/{states} has no data on disk; auto-restoring")
queue_region_refresh(nations, states_filtered)
_TILE_RESTORE_INTERVAL_S = 1800.0
_last_tile_restore_t = 0.0
_tile_only_worker: threading.Thread | None = None
def _configured_region_selector() -> str:
country = params.get("OsmLocationName", return_default=True)
states = configured_states()
nations, states_filtered = normalize_region_selection([country] if country else [], states)
return _compose_region_selector(nations, states_filtered)
def tile_bundles_missing() -> bool:
# covers a media wipe AND the user enabling OfflineOSMaps after the region download
# already ran (the vendor fetch only pulls tile bundles when the toggle is on)
if not params.get_bool("OfflineOSMaps"):
return False
selector = _configured_region_selector()
if not selector:
return False
return any(not region_bundle_installed(part) for part in selector.split(",") if part)
def maybe_restore_tile_bundles() -> None:
"""Tile-only download: don't re-run the whole mapd vendor fetch when only the display
tiles are missing."""
global _last_tile_restore_t, _tile_only_worker
if not tile_bundles_missing():
return
if _region_sync_worker is not None and _region_sync_worker.is_alive():
return
if _tile_only_worker is not None and _tile_only_worker.is_alive():
return
now = time.monotonic()
if now - _last_tile_restore_t < _TILE_RESTORE_INTERVAL_S:
return
_last_tile_restore_t = now
selector = _configured_region_selector()
cloudlog.warning(f"iq_maps: offline map tile bundles missing for {selector}; downloading")
_tile_only_worker = threading.Thread(
target=_fetch_tile_bundles,
args=(selector,),
kwargs={"abort_check": _shutdown.is_set},
daemon=True,
)
_tile_only_worker.start()
def sync_osm_request_flags() -> None:
maybe_auto_restore_region()
maybe_restore_tile_bundles()
if params.get_bool("OsmDbUpdatesCheck"):
if _region_sync_worker is not None and _region_sync_worker.is_alive():
# A download is already writing into Paths.mapd_root() - deleting/rewriting
# files under it right now would race the writer (and any onroad mapd
# reader) the same way the orphaned-subprocess bug did. Wait for it to finish.
return
purge_stale_region_artifacts(stale_region_artifacts())
country = params.get("OsmLocationName", return_default=True)
states = configured_states()
filtered_nations, filtered_states = normalize_region_selection([country], states)
queue_region_refresh(filtered_nations, filtered_states)
if not mem_params.get("OSMDownloadBounds"):
mem_params.put("OSMDownloadBounds", "")
if not mem_params.get("LastGPSPosition"):
mem_params.put("LastGPSPosition", "{}")
def run_loop():
ensure_vendor_runtime()
config_realtime_process([0, 1, 2, 3], 5)
rk = Ratekeeper(1, print_delay_threshold=None)
try:
os.mkdir(Paths.mapd_root())
except FileExistsError:
pass
except PermissionError:
cloudlog.exception(f"iq_maps: failed to make {Paths.mapd_root()}")
# A prior run that got SIGKILLed (or crashed) may have left its vendor-fetch
# mapd subprocess running and still writing into Paths.mapd_root(); clear it
# before anything (including the onroad mapd, once `started` flips) reads
# from that directory. Signal handlers cover the graceful-shutdown path.
_reap_orphaned_vendor_fetch()
_install_signal_handlers()
while not _shutdown.is_set():
show_alert = stale_region_artifacts() and params.get_bool("OsmLocal")
set_offroad_alert("Offroad_OSMUpdateRequired", show_alert, "This alert will be cleared when new maps are downloaded.")
sync_osm_request_flags()
rk.keep_time()
def main():
run_loop()
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,25 @@
"""
Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
Shared tunables and a small debug logger for the offline road-name / turn-speed path.
"""
from openpilot.common.swaglog import cloudlog
# seconds of road ahead we scan for upcoming turn-speed zones published on iqLiveData
LOOK_AHEAD_HORIZON_TIME = 15.0
# clear the on-screen road name once it has gone this long without a refresh (s)
ROAD_NAME_TIMEOUT = 30
R = 6373000.0 # mean Earth radius in metres (great-circle distance math)
QUERY_RADIUS = 3000 # online OSM query reach, metres
QUERY_RADIUS_OFFLINE = 2250 # offline-tile OSM query reach, metres
_DEBUG = False
_CLOUDLOG_DEBUG = False
def debug_road_data(msg, log_to_cloud=True):
if _CLOUDLOG_DEBUG and log_to_cloud:
cloudlog.debug(msg)
if _DEBUG:
print(msg)

View File

@@ -0,0 +1,67 @@
"""
Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
"""
import json
import math
import platform
from cereal import custom
from openpilot.common.params import Params
from openpilot.iqpilot.iq_maps.road_data.signal_bridge import RoadSignalBridge
from openpilot.iqpilot.navd.helpers import Coordinate
class IQRoadLayer(RoadSignalBridge):
def __init__(self):
super().__init__()
self.mem_params = Params("/dev/shm/params") if platform.system() != "Darwin" else self.params
def refresh_position(self) -> None:
location = self.location_sub['iqLiveLocation']
self.fix_ready = (
location.solutionState == custom.IQLiveLocation.SolutionState.ready
and location.geodeticPosition.isValid
)
if self.fix_ready:
self.heading_deg = math.degrees(location.alignedOrientationNed.values[2])
self.last_coordinate = Coordinate(location.geodeticPosition.values[0], location.geodeticPosition.values[1])
if self.last_coordinate is None:
return
payload = {
"latitude": self.last_coordinate.latitude,
"longitude": self.last_coordinate.longitude,
}
if self.heading_deg is not None:
payload["bearing"] = self.heading_deg
self.mem_params.put("LastGPSPosition", json.dumps(payload))
def read_current_limit(self) -> float:
return float(self.mem_params.get("MapSpeedLimit") or 0.0)
def read_current_road(self) -> str:
return str(self.mem_params.get("RoadName") or "")
def read_upcoming_limit(self) -> tuple[float, float]:
raw_segment = self.mem_params.get("NextMapSpeedLimit")
if isinstance(raw_segment, bytes):
raw_segment = raw_segment.decode("utf-8")
try:
upcoming_segment = json.loads(raw_segment) if isinstance(raw_segment, str) and raw_segment else (raw_segment or {})
except json.JSONDecodeError:
upcoming_segment = {}
next_limit = float(upcoming_segment.get("speedlimit", 0.0) or 0.0)
target_lat = upcoming_segment.get("latitude")
target_lon = upcoming_segment.get("longitude")
distance_to_limit = 0.0
if target_lat is not None and target_lon is not None:
limit_coordinate = Coordinate(float(target_lat), float(target_lon))
distance_to_limit = (self.last_coordinate or Coordinate(0, 0)).distance_to(limit_coordinate)
return next_limit, distance_to_limit

View File

@@ -0,0 +1,35 @@
"""
Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
"""
import threading
import traceback
from openpilot.common.realtime import Ratekeeper, config_realtime_process
from openpilot.iqpilot.iq_maps.road_data import debug_road_data
from openpilot.iqpilot.iq_maps.road_data.iq_road_layer import IQRoadLayer
ROAD_LAYER_HZ = 1
ROAD_LAYER_CORES = [0, 1, 2, 3]
def _log_thread_exception(args) -> None:
debug_road_data(f"IQ maps threading exception:\n{args}")
traceback.print_exception(args.exc_type, args.exc_value, args.exc_traceback)
def run() -> None:
config_realtime_process(ROAD_LAYER_CORES, 5)
layer = IQRoadLayer()
rk = Ratekeeper(ROAD_LAYER_HZ, print_delay_threshold=None)
while True:
layer.step()
rk.keep_time()
def main() -> None:
threading.excepthook = _log_thread_exception
run()
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,62 @@
"""
Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
"""
from abc import abstractmethod, ABC
import cereal.messaging as messaging
from openpilot.common.params import Params
from openpilot.common.constants import CV
from openpilot.selfdrive.car.cruise import V_CRUISE_UNSET
from openpilot.iqpilot.navd.helpers import coordinate_from_param
ROAD_SPEED_CEILING = V_CRUISE_UNSET * CV.KPH_TO_MS
class RoadSignalBridge(ABC):
def __init__(self):
self.params = Params()
self.location_sub = messaging.SubMaster(['iqLiveLocation'])
self.output_pub = messaging.PubMaster(['iqLiveData'])
self.fix_ready = False
self.heading_deg = None
self.last_coordinate = coordinate_from_param("LastGPSPositionIQLoc", self.params)
@abstractmethod
def refresh_position(self) -> None:
pass
@abstractmethod
def read_current_limit(self) -> float:
pass
@abstractmethod
def read_upcoming_limit(self) -> tuple[float, float]:
pass
@abstractmethod
def read_current_road(self) -> str:
pass
def publish_snapshot(self) -> None:
active_limit = self.read_current_limit()
next_limit, next_limit_distance = self.read_upcoming_limit()
outbound = messaging.new_message('iqLiveData')
outbound.valid = self.location_sub['iqLiveLocation'].gpsHealthy
live_data = outbound.iqLiveData
live_data.speedLimitValid = bool(ROAD_SPEED_CEILING > active_limit > 0)
live_data.speedLimit = active_limit
live_data.speedLimitAheadValid = bool(ROAD_SPEED_CEILING > next_limit > 0)
live_data.speedLimitAhead = next_limit
live_data.speedLimitAheadDistance = next_limit_distance
live_data.roadName = self.read_current_road()
self.output_pub.send('iqLiveData', outbound)
def step(self) -> None:
self.location_sub.update(0)
self.refresh_position()
self.publish_snapshot()

View File

View File

@@ -0,0 +1 @@
4de87a77eb698200acf0cf734aaaa0222c29b64ab389430107f912ae201c762e

View File

@@ -0,0 +1,235 @@
import hashlib
import json
import sqlite3
import threading
from functools import partial
from http.server import HTTPServer, SimpleHTTPRequestHandler
import pytest
from openpilot.iqpilot.iq_maps import tile_bundle_downloader as tbd
from openpilot.iqpilot.ui.onroad import offline_tiles
PNG_1X1 = (
b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01" +
b"\x08\x06\x00\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\rIDATx\x9cc\xf8\xcf" +
b"\xc0\xf0\x1f\x00\x05\x00\x01\xff\x89\x99=\x1d\x00\x00\x00\x00IEND\xaeB`\x82"
)
class FakeParams:
def __init__(self):
self.store: dict[str, object] = {}
def get(self, key, return_default=False):
return self.store.get(key)
def get_bool(self, key):
return bool(self.store.get(key))
def put(self, key, value):
self.store[key] = value
def put_bool(self, key, value):
self.store[key] = bool(value)
def remove(self, key):
self.store.pop(key, None)
def _make_mbtiles(path):
conn = sqlite3.connect(path)
conn.execute("CREATE TABLE metadata (name text, value text)")
conn.execute("CREATE TABLE tiles (zoom_level integer, tile_column integer, tile_row integer, tile_data blob)")
conn.executemany("INSERT INTO metadata (name, value) VALUES (?, ?)",
[("format", "png"), ("minzoom", "10"), ("maxzoom", "16"),
("bounds", "-124.5,32.4,-114.1,42.1")])
conn.execute("INSERT INTO tiles VALUES (?, ?, ?, ?)", (10, 163, 396, PNG_1X1))
conn.commit()
conn.close()
@pytest.fixture
def hosting(tmp_path, monkeypatch):
"""Local static host serving index.json + a us_state.CA bundle; offline root redirected."""
serve_root = tmp_path / "serve"
serve_root.mkdir()
bundle = serve_root / "us_state.CA.mbtiles"
_make_mbtiles(bundle)
payload = bundle.read_bytes()
index = {
"version": 1,
"regions": {
"us_state.CA": {
"path": "us_state.CA.mbtiles",
"bytes": len(payload),
"sha256": hashlib.sha256(payload).hexdigest(),
"bounds": "-124.5,32.4,-114.1,42.1",
"minzoom": 10,
"maxzoom": 16,
"version": "20260709",
}
},
}
(serve_root / "index.json").write_text(json.dumps(index))
handler = partial(SimpleHTTPRequestHandler, directory=str(serve_root))
server = HTTPServer(("127.0.0.1", 0), handler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
offline_root = tmp_path / "offline_maps"
monkeypatch.setenv(offline_tiles.OFFLINE_TILE_ROOT_ENV, str(offline_root / "tiles"))
offline_tiles._region_roots_cache = None
offline_tiles._region_bounds_cache.clear()
params = FakeParams()
params.put(tbd.BASE_URL_PARAM, f"http://127.0.0.1:{server.server_address[1]}")
sessions: list = []
real_ctor = tbd.TileBundleDownloader.__init__
def tracking_ctor(self, *args, **kwargs):
real_ctor(self, *args, **kwargs)
sessions.append(self.session)
monkeypatch.setattr(tbd.TileBundleDownloader, "__init__", tracking_ctor)
try:
yield params, index, offline_root
finally:
for session in sessions:
session.close()
server.shutdown()
server.server_close()
def test_download_installs_bundle_and_manifest(hosting):
params, index, offline_root = hosting
dl = tbd.TileBundleDownloader(params=params, mem_params=params)
assert dl.download_regions(["us_state.CA"]) is True
installed = offline_root / "regions" / "us_state.CA" / "tiles" / "offline.mbtiles"
assert installed.exists()
manifest = json.loads((installed.parent.parent / "manifest.json").read_text())
assert manifest["mbtiles"]["bounds"] == "-124.5,32.4,-114.1,42.1"
assert manifest["mbtiles"]["sha256"] == index["regions"]["us_state.CA"]["sha256"]
# request/progress params cleaned up
assert params.get(tbd.REQUEST_PARAM) is None
assert params.get(tbd.PROGRESS_PARAM)["active"] is False
# and the on-screen map provider can find + read it
assert offline_tiles.find_offline_mbtiles_path(37.0, -120.0) == installed
conn = offline_tiles.open_mbtiles(installed)
try:
assert offline_tiles.mbtiles_is_raster(conn)
assert offline_tiles.load_raster_tile_blob(conn, 10, 163, 2 ** 10 - 1 - 396) == PNG_1X1
finally:
conn.close()
def test_skips_already_installed_matching_sha(hosting):
params, _, offline_root = hosting
dl = tbd.TileBundleDownloader(params=params, mem_params=params)
assert dl.download_regions(["us_state.CA"]) is True
installed = offline_root / "regions" / "us_state.CA" / "tiles" / "offline.mbtiles"
first_mtime = installed.stat().st_mtime_ns
assert dl.download_regions(["us_state.CA"]) is True
assert installed.stat().st_mtime_ns == first_mtime
def test_unknown_region_fails_cleanly(hosting):
params, _, offline_root = hosting
dl = tbd.TileBundleDownloader(params=params, mem_params=params)
assert dl.download_regions(["us_state.ZZ"]) is False
assert not (offline_root / "regions" / "us_state.ZZ").exists()
def test_resume_from_partial(hosting):
params, _, offline_root = hosting
part = offline_root / "regions" / "us_state.CA" / "tiles" / "offline.mbtiles.part"
part.parent.mkdir(parents=True)
# pre-seed the first half as an interrupted download
full = (offline_root / ".." / "serve" / "us_state.CA.mbtiles").resolve().read_bytes()
part.write_bytes(full[: len(full) // 2])
dl = tbd.TileBundleDownloader(params=params, mem_params=params)
assert dl.download_regions(["us_state.CA"]) is True
installed = part.parent / "offline.mbtiles"
assert installed.read_bytes() == full
assert not part.exists()
def test_cancel_aborts_before_install(hosting):
params, _, offline_root = hosting
dl = tbd.TileBundleDownloader(params=params, mem_params=params, abort_check=lambda: True)
assert dl.download_regions(["us_state.CA"]) is False
assert not (offline_root / "regions" / "us_state.CA" / "tiles" / "offline.mbtiles").exists()
# request param cleaned up so the UI doesn't show a stuck download
assert params.get(tbd.REQUEST_PARAM) is None
def test_sha_mismatch_rejected(hosting):
params, index, offline_root = hosting
index["regions"]["us_state.CA"]["sha256"] = "0" * 64
serve_root = (offline_root / ".." / "serve").resolve()
(serve_root / "index.json").write_text(json.dumps(index))
dl = tbd.TileBundleDownloader(params=params, mem_params=params)
assert dl.download_regions(["us_state.CA"]) is False
assert not (offline_root / "regions" / "us_state.CA" / "tiles" / "offline.mbtiles").exists()
def test_new_region_visible_without_process_restart(hosting):
"""Regression: lru_cache on _candidate_region_roots hid freshly downloaded regions."""
params, _, offline_root = hosting
# UI already scanned (and found nothing)
offline_tiles._region_roots_cache = None
assert offline_tiles.find_offline_mbtiles_path(37.0, -120.0) is None
dl = tbd.TileBundleDownloader(params=params, mem_params=params)
assert dl.download_regions(["us_state.CA"]) is True
# TTL cache: expire it and the new region shows up in the same process
offline_tiles._region_roots_cache = None
found = offline_tiles.find_offline_mbtiles_path(37.0, -120.0)
assert found is not None and found.exists()
def test_candidate_base_urls_param_override_wins():
params = FakeParams()
params.put(tbd.BASE_URL_PARAM, "https://my-r2.example.com/v1/")
assert tbd.candidate_base_urls(params) == ["https://my-r2.example.com/v1"]
def test_candidate_base_urls_private_endpoints_first():
params = FakeParams()
urls = tbd.candidate_base_urls(params)
# embedded private endpoints (gitea) come before the public default
assert urls[-1] == tbd.DEFAULT_TILE_BUNDLE_BASE_URL
if tbd._private_base_urls is not None:
assert any("git.konn3kt.com" in url for url in urls[:-1])
assert tbd.request_auth() is not None
def test_day_variant_downloaded_and_manifested(hosting, tmp_path):
params, index, offline_root = hosting
serve_root = (offline_root / ".." / "serve").resolve()
day_bundle = serve_root / "us_state.CA_day.mbtiles"
_make_mbtiles(day_bundle)
day_payload = day_bundle.read_bytes()
entry = index["regions"]["us_state.CA"]
entry["day_path"] = "us_state.CA_day.mbtiles"
entry["day_bytes"] = len(day_payload)
entry["day_sha256"] = hashlib.sha256(day_payload).hexdigest()
(serve_root / "index.json").write_text(json.dumps(index))
dl = tbd.TileBundleDownloader(params=params, mem_params=params)
assert dl.download_regions(["us_state.CA"]) is True
tiles = offline_root / "regions" / "us_state.CA" / "tiles"
assert (tiles / "offline.mbtiles").exists()
assert (tiles / "offline_day.mbtiles").exists()
manifest = json.loads((tiles.parent / "manifest.json").read_text())
assert manifest["mbtiles_day"]["sha256"] == entry["day_sha256"]
# installed-and-current check must account for the day file
assert dl._installed_matches("us_state.CA", entry) is True
(tiles / "offline_day.mbtiles").unlink()
assert dl._installed_matches("us_state.CA", entry) is False

View File

@@ -0,0 +1,86 @@
import hashlib
import pytest
from openpilot.iqpilot.iq_maps import vendor_mapd_installer as vmi
class FakeParams:
def __init__(self):
self.store: dict[str, object] = {}
def get(self, key, return_default=False):
return self.store.get(key)
def put(self, key, value):
self.store[key] = value
def remove(self, key):
self.store.pop(key, None)
@pytest.fixture
def env(tmp_path, monkeypatch):
binary = tmp_path / "mapd"
hash_file = tmp_path / "mapd_hash"
monkeypatch.setattr(vmi, "VENDOR_MAPD_PATH", str(binary))
monkeypatch.setattr(vmi, "QUARANTINE_PATH", str(binary) + ".quarantined")
monkeypatch.setattr(vmi, "_HASH_FILE", str(hash_file))
monkeypatch.setattr(vmi.sentry, "init", lambda *a, **k: None)
monkeypatch.setattr(vmi.sentry, "capture_exception", lambda *a, **k: None)
return binary, hash_file
def test_verified_binary_stamps_version(env):
binary, hash_file = env
binary.write_bytes(b"vetted")
hash_file.write_text(hashlib.sha256(b"vetted").hexdigest())
params = FakeParams()
assert vmi.VendorMapdInstaller(params=params).verify()
assert params.store["MapdVersion"] == vmi.VENDOR_RELEASE_TAG
assert binary.exists()
def test_foreign_binary_quarantined(env):
# the fresh-install poison scenario: a stock release build on disk while the
# pin points at the vetted build
binary, hash_file = env
binary.write_bytes(b"stock release build")
hash_file.write_text(hashlib.sha256(b"vetted").hexdigest())
params = FakeParams()
params.store["MapdVersion"] = vmi.VENDOR_RELEASE_TAG
assert not vmi.VendorMapdInstaller(params=params).verify()
assert not binary.exists()
assert (binary.parent / "mapd.quarantined").read_bytes() == b"stock release build"
assert "MapdVersion" not in params.store
def test_verify_clears_stale_quarantine(env):
binary, hash_file = env
binary.write_bytes(b"vetted")
hash_file.write_text(hashlib.sha256(b"vetted").hexdigest())
quarantine = binary.parent / "mapd.quarantined"
quarantine.write_bytes(b"old poison")
assert vmi.VendorMapdInstaller(params=FakeParams()).verify()
assert not quarantine.exists()
def test_missing_hash_pin_leaves_binary_alone(env):
binary, _ = env
binary.write_bytes(b"anything")
assert not vmi.VendorMapdInstaller(params=FakeParams()).verify()
assert binary.exists()
def test_missing_binary(env):
_, hash_file = env
hash_file.write_text(hashlib.sha256(b"vetted").hexdigest())
params = FakeParams()
params.store["MapdVersion"] = vmi.VENDOR_RELEASE_TAG
assert not vmi.VendorMapdInstaller(params=params).verify()
assert "MapdVersion" not in params.store

View File

@@ -0,0 +1,16 @@
"""
Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
"""
from openpilot.iqpilot.iq_maps.vendor_mapd_installer import sha256_of_file
from openpilot.iqpilot.iq_maps import VENDOR_MAPD_PATH
from openpilot.iqpilot.iq_maps.update_vendor_version import HASH_FILE
class TestMapdVersion:
def test_compare_versions(self):
mapd_hash = sha256_of_file(VENDOR_MAPD_PATH)
with open(HASH_FILE) as f:
current_hash = f.read().strip()
assert current_hash == mapd_hash, "Run iqpilot/iq_maps/update_vendor_version.py to update the current mapd version and hash"

View File

@@ -0,0 +1,355 @@
#!/usr/bin/env python3
"""
Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
Downloads per-region raster display-tile bundles (.mbtiles) for the offline on-screen map.
These are a separate asset from mapd's routing/speed-limit data: mapd pulls OSM way tiles
into Paths.mapd_root(), while the on-screen map (OsmOfflineProvider) reads raster .mbtiles
from offline_map_root()/regions/<selector>/tiles/offline.mbtiles. Bundles are built per
state/nation by scripts/iqpilot/build_state_tile_bundles.py and hosted behind a static base
URL that serves:
<base>/index.json {"version": 1, "regions": {<selector>: entry}}
<base>/<entry["path"]> the raster .mbtiles for that region
Entry fields: path, bytes, sha256, bounds ("minLon,minLat,maxLon,maxLat"), minzoom, maxzoom.
Selectors match the mapd region menu naming: us_state.CA, nation.US.
"""
import hashlib
import json
import platform
import threading
import time
from pathlib import Path
import requests
from openpilot.common.params import Params
from openpilot.common.swaglog import cloudlog
from openpilot.iqpilot.ui.onroad.offline_tiles import offline_map_root
# Proprietary auth + hosted endpoints (gitea raw with an embedded read-only PAT, same
# pattern as the model selector). Optional: without the private bundle the downloader
# still works anonymously against OfflineTilesBaseUrl (e.g. a public R2 bucket).
try:
from openpilot.iqpilot.iq_maps.tiles_auth import get_base_urls as _private_base_urls, get_requests_auth as _private_auth
except Exception: # ProprietaryModuleMissing or import errors in stripped builds
_private_base_urls = None
_private_auth = None
# R2 bucket iqnav behind the public custom domain (see scripts/iqpilot/tile_factory/r2_sync_watch.py)
DEFAULT_TILE_BUNDLE_BASE_URL = "https://maps.konn3kt.com/iqosmd/v1"
BASE_URL_PARAM = "OfflineTilesBaseUrl"
PROGRESS_PARAM = "OfflineTilesDownloadProgress"
REQUEST_PARAM = "OfflineTilesDownloadRequest"
CHUNK_BYTES = 1 << 20
HTTP_TIMEOUT_S = 30.0
STREAM_RETRIES = 8
def candidate_base_urls(params: Params) -> list[str]:
"""Hosts to try in order: user/param override first, then the embedded private
endpoints (gitea raw), then the public default."""
override = params.get(BASE_URL_PARAM)
if isinstance(override, bytes):
override = override.decode("utf-8", errors="ignore")
override = (override or "").strip()
if override:
return [override.rstrip("/")]
urls: list[str] = []
if _private_base_urls is not None:
try:
urls.extend(url.rstrip("/") for url in _private_base_urls())
except Exception:
pass
urls.append(DEFAULT_TILE_BUNDLE_BASE_URL)
return urls
def request_auth() -> tuple[str, str] | None:
if _private_auth is None:
return None
try:
return _private_auth()
except Exception:
return None
def fetch_index(base_url: str, session: requests.Session) -> dict:
response = session.get(f"{base_url}/index.json", timeout=HTTP_TIMEOUT_S, auth=request_auth())
response.raise_for_status()
index = response.json()
regions = index.get("regions")
if not isinstance(regions, dict):
raise ValueError("tile bundle index has no regions")
return regions
def region_bundle_dir(selector: str) -> Path:
return offline_map_root() / "regions" / selector
def region_bundle_path(selector: str) -> Path:
return region_bundle_dir(selector) / "tiles" / "offline.mbtiles"
def region_bundle_installed(selector: str) -> bool:
return region_bundle_path(selector).exists()
def installed_region_selectors() -> list[str]:
regions_root = offline_map_root() / "regions"
if not regions_root.exists():
return []
return sorted(
child.name for child in regions_root.iterdir()
if child.is_dir() and (child / "tiles" / "offline.mbtiles").exists()
)
def _hash_existing(path: Path) -> tuple["hashlib._Hash", int]:
digest = hashlib.sha256()
size = 0
with open(path, "rb") as f:
while True:
chunk = f.read(CHUNK_BYTES)
if not chunk:
break
digest.update(chunk)
size += len(chunk)
return digest, size
def _write_manifest(selector: str, entry: dict) -> None:
manifest = {
"region": selector,
"version": entry.get("version", ""),
"mbtiles": {
"bounds": entry.get("bounds", ""),
"minzoom": entry.get("minzoom"),
"maxzoom": entry.get("maxzoom"),
"bytes": entry.get("bytes"),
"sha256": entry.get("sha256", ""),
},
}
if entry.get("day_path"):
manifest["mbtiles_day"] = {
"bytes": entry.get("day_bytes"),
"sha256": entry.get("day_sha256", ""),
}
manifest_path = region_bundle_dir(selector) / "manifest.json"
manifest_path.parent.mkdir(parents=True, exist_ok=True)
manifest_path.write_text(json.dumps(manifest, indent=2))
class TileBundleDownloader:
"""Streams region bundles to disk with resume + sha256 verify + atomic install.
Cancellation matches the mapd flow: the caller sets REQUEST_PARAM in mem params while a
download runs; removing it (konn3kt cancel RPC or settings) aborts between chunks. The
partial .part file is kept so a retry resumes instead of restarting.
"""
def __init__(self, params: Params | None = None, mem_params: Params | None = None,
abort_check=None):
self.params = params if params is not None else Params()
if mem_params is not None:
self.mem_params = mem_params
else:
self.mem_params = Params("/dev/shm/params") if platform.system() != "Darwin" else self.params
self.session = requests.Session()
self._cancelled = threading.Event()
# optional external cancel signal, e.g. the orchestrator's OSMDownloadLocations removal
self._abort_check = abort_check
def cancel(self) -> None:
self._cancelled.set()
def _should_abort(self) -> bool:
if self._cancelled.is_set():
return True
if not self.mem_params.get(REQUEST_PARAM):
# request flag was removed out from under us -> user cancelled
self._cancelled.set()
return True
if self._abort_check is not None and self._abort_check():
self._cancelled.set()
return True
return False
def _publish_progress(self, region: str, downloaded: int, total: int, active: bool) -> None:
self.mem_params.put(PROGRESS_PARAM, {
"active": active,
"region": region,
"downloaded_bytes": int(downloaded),
"total_bytes": int(total),
})
def _download_one(self, selector: str, entry: dict, base_url: str,
progress_offset: int, progress_total: int) -> bool:
"""Download a region: the night bundle, plus the optional day-style variant."""
night_path = region_bundle_path(selector)
ok = self._download_file(
selector, base_url, entry["path"], int(entry.get("bytes", 0)),
str(entry.get("sha256", "")).strip().lower(), night_path,
progress_offset, progress_total,
)
if not ok:
return False
if entry.get("day_path"):
day_ok = self._download_file(
selector, base_url, entry["day_path"], int(entry.get("day_bytes", 0)),
str(entry.get("day_sha256", "")).strip().lower(),
night_path.with_name("offline_day.mbtiles"),
progress_offset + int(entry.get("bytes", 0)), progress_total,
)
if not day_ok:
# the night set is complete and usable; a failed day variant retries next pass
cloudlog.warning(f"iq_maps: day-style bundle failed for {selector}; night set installed")
# manifest last: bounds drive region matching, so it must describe installed files
_write_manifest(selector, entry)
cloudlog.info(f"iq_maps: installed tile bundle {selector}")
return True
def _download_file(self, selector: str, base_url: str, remote_path: str, expected_bytes: int,
expected_sha: str, final_path: Path,
progress_offset: int, progress_total: int) -> bool:
url = f"{base_url}/{remote_path.lstrip('/')}"
part_path = final_path.with_name(final_path.name + ".part")
part_path.parent.mkdir(parents=True, exist_ok=True)
# A cellular/hotspot link routinely kills a multi-hundred-MB stream mid-flight; retry
# each interruption from the current .part offset instead of failing the whole region.
downloaded = 0
digest = hashlib.sha256()
last_error: Exception | None = None
for attempt in range(STREAM_RETRIES):
if self._should_abort():
cloudlog.warning(f"iq_maps: tile bundle download cancelled for {selector}")
return False
if attempt:
time.sleep(min(30.0, 2.0 * attempt))
try:
digest = hashlib.sha256()
resume_from = 0
if part_path.exists():
digest, resume_from = _hash_existing(part_path)
if expected_bytes and resume_from > expected_bytes:
part_path.unlink()
digest = hashlib.sha256()
resume_from = 0
headers = {"Range": f"bytes={resume_from}-"} if resume_from else {}
auth = request_auth()
response = self.session.get(url, headers=headers, stream=True, timeout=HTTP_TIMEOUT_S, auth=auth)
if resume_from and response.status_code != 206:
# server ignored the Range request -> restart from scratch
digest = hashlib.sha256()
resume_from = 0
part_path.unlink(missing_ok=True)
if response.status_code == 416:
response = self.session.get(url, stream=True, timeout=HTTP_TIMEOUT_S, auth=auth)
response.raise_for_status()
downloaded = resume_from
mode = "ab" if resume_from else "wb"
with open(part_path, mode) as f:
for chunk in response.iter_content(chunk_size=CHUNK_BYTES):
if self._should_abort():
cloudlog.warning(f"iq_maps: tile bundle download cancelled for {selector}")
return False
f.write(chunk)
digest.update(chunk)
downloaded += len(chunk)
self._publish_progress(selector, progress_offset + downloaded, progress_total, active=True)
break
except requests.RequestException as exc:
last_error = exc
cloudlog.warning(f"iq_maps: tile bundle stream interrupted for {selector} "
+ f"(attempt {attempt + 1}/{STREAM_RETRIES}): {exc}")
else:
raise requests.RequestException(f"stream failed after {STREAM_RETRIES} attempts") from last_error
if expected_bytes and downloaded != expected_bytes:
cloudlog.error(f"iq_maps: tile bundle size mismatch for {selector}: {downloaded} != {expected_bytes}")
part_path.unlink(missing_ok=True)
return False
if expected_sha and digest.hexdigest() != expected_sha:
cloudlog.error(f"iq_maps: tile bundle sha256 mismatch for {selector}")
part_path.unlink(missing_ok=True)
return False
part_path.replace(final_path)
return True
def download_regions(self, selectors: list[str]) -> bool:
"""Download the display-tile bundles for the given region selectors. Returns True if all
requested bundles are installed and current when done."""
self._cancelled.clear()
ok = True
try:
self.mem_params.put(REQUEST_PARAM, {"regions": list(selectors)})
regions = None
base_url = ""
for candidate in candidate_base_urls(self.params):
try:
regions = fetch_index(candidate, self.session)
base_url = candidate
break
except (requests.RequestException, ValueError, json.JSONDecodeError):
cloudlog.warning(f"iq_maps: tile bundle index unavailable at {candidate}")
if regions is None:
cloudlog.error("iq_maps: no tile bundle host reachable")
return False
wanted: list[tuple[str, dict]] = []
for selector in selectors:
entry = regions.get(selector)
if entry is None:
cloudlog.warning(f"iq_maps: no tile bundle published for {selector}")
ok = False
continue
if region_bundle_installed(selector) and self._installed_matches(selector, entry):
continue
wanted.append((selector, entry))
progress_total = sum(int(entry.get("bytes", 0)) + int(entry.get("day_bytes", 0)) for _, entry in wanted)
progress_offset = 0
for selector, entry in wanted:
if self._should_abort():
return False
try:
if not self._download_one(selector, entry, base_url, progress_offset, progress_total):
ok = False
except (requests.RequestException, OSError):
cloudlog.exception(f"iq_maps: tile bundle download failed for {selector}")
ok = False
progress_offset += int(entry.get("bytes", 0)) + int(entry.get("day_bytes", 0))
return ok
finally:
self._publish_progress("", 0, 0, active=False)
try:
self.mem_params.remove(REQUEST_PARAM)
except Exception:
pass
@staticmethod
def _installed_matches(selector: str, entry: dict) -> bool:
manifest_path = region_bundle_dir(selector) / "manifest.json"
try:
manifest = json.loads(manifest_path.read_text())
except (OSError, json.JSONDecodeError):
return False
installed_sha = str(manifest.get("mbtiles", {}).get("sha256", "")).strip().lower()
expected_sha = str(entry.get("sha256", "")).strip().lower()
if not expected_sha or installed_sha != expected_sha:
return False
if entry.get("day_path"):
# a published day variant must be installed and current too
day_file = region_bundle_dir(selector) / "tiles" / "offline_day.mbtiles"
installed_day = str(manifest.get("mbtiles_day", {}).get("sha256", "")).strip().lower()
expected_day = str(entry.get("day_sha256", "")).strip().lower()
if not day_file.exists() or installed_day != expected_day:
return False
return True

View File

@@ -0,0 +1,10 @@
#!/usr/bin/env python3
"""
Copyright (c) IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
"""
from openpilot.iqpilot._proprietary_loader import ProprietaryModuleMissing, load_private_module
try:
load_private_module(__name__, "iqpilot_private.maps.git_auth")
except ProprietaryModuleMissing:
from iqpilot.maps_private_src.git_auth import * # noqa: F403

View File

@@ -0,0 +1,76 @@
#!/usr/bin/env python3
"""
Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
Maintainer utility: pin a new mapd release tag and refresh the checked-in binary
hash. Not used at runtime. Binaries come from the gitlvb teal/mapd CI (built
against teal/gomsgq) — drop the artifact at third_party/mapd_pfeiferj/mapd, then
run this so the hash pin moves in the same commit.
"""
import argparse
import os
import re
import sys
from openpilot.common.basedir import BASEDIR
from openpilot.iqpilot.iq_maps import VENDOR_MAPD_PATH
from openpilot.iqpilot.iq_maps.vendor_mapd_installer import (
VENDOR_RELEASE_TAG,
sha256_of_file,
)
_RELEASE_SYMBOL = "VENDOR_RELEASE_TAG"
_INSTALLER_SRC = os.path.join(BASEDIR, "iqpilot", "iq_maps", "vendor_mapd_installer.py")
# public: the checked-in hash the version test compares the installed binary against
HASH_FILE = os.path.join(BASEDIR, "iqpilot", "iq_maps", "tests", "mapd_hash")
_HASH_FILE = HASH_FILE
_TAG_ASSIGN = re.compile(rf'^{_RELEASE_SYMBOL}\s*=\s*["\'][^"\']*["\']', re.MULTILINE)
def rewrite_pinned_tag(new_tag: str) -> bool:
with open(_INSTALLER_SRC) as f:
src = f.read()
patched, count = _TAG_ASSIGN.subn(f'{_RELEASE_SYMBOL} = "{new_tag}"', src, count=1)
if count != 1:
print(f"could not locate the {_RELEASE_SYMBOL} assignment in {_INSTALLER_SRC}; nothing written")
return False
with open(_INSTALLER_SRC, "w") as f:
f.write(patched)
print(f"pinned {_RELEASE_SYMBOL} -> {new_tag}")
return True
def refresh_hash_file() -> None:
digest = sha256_of_file(VENDOR_MAPD_PATH)
with open(_HASH_FILE, "w") as f:
f.write(digest)
print(f"wrote binary hash {digest} -> {_HASH_FILE}")
def main() -> int:
parser = argparse.ArgumentParser(description="Pin a new mapd release tag and refresh its hash")
parser.add_argument("--new_ver", type=str, help='e.g. --new_ver "v2.1.0"')
args = parser.parse_args()
if not args.new_ver:
parser.print_help()
print(f'\ncurrently pinned: {VENDOR_RELEASE_TAG} (unchanged)')
return 0
target = args.new_ver.strip()
if target == VENDOR_RELEASE_TAG:
reply = input(f"{target} is already the pinned tag — re-run anyway? (y/N): ").strip().lower()
if reply != "y":
print("aborted; nothing changed")
return 0
if not rewrite_pinned_tag(target):
return 1
refresh_hash_file()
return 0
if __name__ == "__main__":
sys.exit(main())

View File

@@ -0,0 +1,113 @@
#!/usr/bin/env python3
"""
Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
Verifies the vendored `mapd` routing binary authored by Jacob Pfeifer
(github.com/pfeiferj/mapd), built from the gitlvb teal/mapd fork against
teal/gomsgq. The only accepted binary is the checked-in one matching the pinned
hash; nothing is ever downloaded at runtime. Jacob's stock release build embeds
a 15-reader msgq header layout — on this fork (NUM_READERS=32) its registration
writes land inside other processes' reader slots, so a wrong binary is
quarantined rather than left where manager could start it.
"""
import hashlib
import os
import sys
from openpilot.common.basedir import BASEDIR
from openpilot.common.params import Params
from openpilot.common.spinner import Spinner
from openpilot.common.swaglog import cloudlog
from openpilot.iqpilot.iq_maps import VENDOR_MAPD_PATH
import openpilot.system.sentry as sentry
VENDOR_RELEASE_TAG = "v2.0.6-iq1"
_VERSION_PARAM = "MapdVersion"
_HASH_FILE = os.path.join(BASEDIR, "iqpilot", "iq_maps", "tests", "mapd_hash")
QUARANTINE_PATH = VENDOR_MAPD_PATH + ".quarantined"
def sha256_of_file(path: str) -> str:
"""Hex SHA-256 digest of a file on disk."""
digest = hashlib.sha256()
with open(path, "rb") as handle:
for block in iter(lambda: handle.read(1 << 20), b""):
digest.update(block)
return digest.hexdigest()
def stamp_vendor_version(version: str, params: Params | None = None) -> None:
(params or Params()).put(_VERSION_PARAM, version)
class VendorMapdInstaller:
def __init__(self, spinner_ref: Spinner | None = None, params: Params | None = None):
self._spinner = spinner_ref
self._params = params if params is not None else Params()
def get_installed_version(self) -> str:
return str(self._params.get(_VERSION_PARAM) or "")
def verify(self) -> bool:
"""True iff the on-disk binary matches the pinned hash; quarantines a wrong one."""
expected = self._expected_hash()
if not expected:
cloudlog.error("iq_maps: pinned mapd hash missing, vendor binary cannot be verified")
return False
if not os.path.isfile(VENDOR_MAPD_PATH):
# the binary is a tracked file: the updater/bundle restores it
self._say("Offline maps engine missing; it will be restored by the next update.")
self._params.remove(_VERSION_PARAM)
return False
try:
current = sha256_of_file(VENDOR_MAPD_PATH)
except OSError:
cloudlog.exception("iq_maps: vendor mapd unreadable")
return False
if current == expected:
stamp_vendor_version(VENDOR_RELEASE_TAG, self._params)
try:
os.remove(QUARANTINE_PATH)
except OSError:
pass
self._say(f"Offline maps engine verified [{VENDOR_RELEASE_TAG}]")
return True
# a foreign binary — e.g. a stock release download from the retired fetch
# path — must never run: quarantine it where manager can't start it
cloudlog.error(f"iq_maps: vendor mapd hash {current[:12]} != pinned {expected[:12]}, quarantining")
self._say("Offline maps engine failed verification; quarantined until the next update.")
try:
os.replace(VENDOR_MAPD_PATH, QUARANTINE_PATH)
except OSError:
cloudlog.exception("iq_maps: vendor mapd quarantine failed")
return False
self._params.remove(_VERSION_PARAM)
try:
raise RuntimeError(f"vendor mapd hash mismatch quarantined: {current}")
except RuntimeError as exc:
sentry.init(sentry.SentryProject.SELFDRIVE)
sentry.capture_exception(exc)
return False
def _expected_hash(self) -> str:
try:
with open(_HASH_FILE) as f:
return f.read().strip()
except OSError:
return ""
def _say(self, text: str) -> None:
if self._spinner is not None:
self._spinner.update(text)
if __name__ == "__main__":
spinner = Spinner()
ok = VendorMapdInstaller(spinner).verify()
spinner.close()
sys.exit(0 if ok else 1)

View File