forked from IQ.Lvbs/IQ.Pilot
IQ.Pilot Release Commit @ f2a861c
This commit is contained in:
IQ.Lvbs CI [bot]
@@ -1,5 +1,5 @@
|
||||
import os
|
||||
from openpilot.common.basedir import BASEDIR
|
||||
from iqpilot.common.basedir import BASEDIR
|
||||
|
||||
VENDOR_MAPD_BIN_DIR = os.path.join(BASEDIR, "third_party/mapd_pfeiferj")
|
||||
VENDOR_MAPD_BIN_DIR = os.path.join(BASEDIR, "iqpilot/third_party/mapd_pfeiferj")
|
||||
VENDOR_MAPD_PATH = os.path.join(VENDOR_MAPD_BIN_DIR, "mapd")
|
||||
|
||||
@@ -12,39 +12,23 @@ import threading
|
||||
import time
|
||||
from datetime import datetime
|
||||
|
||||
import cereal.messaging as messaging
|
||||
from cereal import custom
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.common.realtime import Ratekeeper, config_realtime_process
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
from openpilot.selfdrive.selfdrived.alertmanager import set_offroad_alert
|
||||
from openpilot.system.hardware.hw import Paths
|
||||
from openpilot.iqpilot.iq_maps import VENDOR_MAPD_BIN_DIR, VENDOR_MAPD_PATH
|
||||
from openpilot.iqpilot.iq_maps.tile_bundle_downloader import TileBundleDownloader, region_bundle_installed
|
||||
from openpilot.iqpilot.iq_maps.vendor_mapd_installer import VendorMapdInstaller
|
||||
import iqpilot.cereal.messaging as messaging
|
||||
from iqpilot.cereal import custom
|
||||
from iqpilot.common.params import Params
|
||||
from iqpilot.common.realtime import Ratekeeper, config_realtime_process
|
||||
from iqpilot.common.swaglog import cloudlog
|
||||
from iqpilot.selfdrive.selfdrived.alertmanager import set_offroad_alert
|
||||
from iqpilot.system.hardware.hw import Paths
|
||||
from iqpilot.iq_maps import VENDOR_MAPD_BIN_DIR, VENDOR_MAPD_PATH
|
||||
from iqpilot.iq_maps.tile_bundle_downloader import TileBundleDownloader, region_bundle_installed
|
||||
from iqpilot.iq_maps.vendor_mapd_installer import VendorMapdInstaller
|
||||
|
||||
OfflineMapAction = custom.MapdInputType
|
||||
_region_sync_worker: threading.Thread | None = None
|
||||
|
||||
# mapd_manager only runs offroad (process_config.only_offroad) and the onroad
|
||||
# NativeProcess("mapd", ...) is started the instant `started` flips True. If a
|
||||
# vendor-map download is in flight at that exact moment, the two `mapd`
|
||||
# binaries end up pointed at the same Paths.mapd_root() tile directory at the
|
||||
# same time: this one still downloading/writing, the onroad one already
|
||||
# mmap-reading. Manager only sends SIGINT/SIGTERM to stop mapd_manager, which
|
||||
# by default only interrupts the main thread — the background download thread
|
||||
# and the vendor `mapd` subprocess it spawned are otherwise orphaned and keep
|
||||
# writing into the tile directory the onroad reader just opened, which is what
|
||||
# was segfaulting (-12) the onroad process in a tight restart loop. The lock +
|
||||
# pidfile below make sure that subprocess is always killed (on clean shutdown
|
||||
# via the signal handlers, and on the next boot if this process itself got
|
||||
# SIGKILLed) before anything else is allowed to read the tile directory.
|
||||
_active_proc_lock = threading.Lock()
|
||||
_active_proc: subprocess.Popen | None = None
|
||||
_shutdown = threading.Event()
|
||||
# Display-tile bundles for the offline on-screen map (separate asset from mapd's routing
|
||||
# data). Downloaded after the mapd fetch in the same worker so a region selection installs
|
||||
# both, and independently restorable when only the tile bundle is missing.
|
||||
_tile_downloader: TileBundleDownloader | None = None
|
||||
|
||||
|
||||
@@ -62,7 +46,6 @@ def _pid_is_vendor_fetch(pid: int) -> bool:
|
||||
|
||||
|
||||
def _reap_orphaned_vendor_fetch() -> None:
|
||||
"""Kill any vendor-fetch mapd subprocess left running from a prior, uncleanly-terminated run."""
|
||||
pidfile = _vendor_fetch_pidfile()
|
||||
try:
|
||||
with open(pidfile) as f:
|
||||
@@ -121,19 +104,11 @@ def _install_signal_handlers() -> None:
|
||||
signal.signal(signal.SIGTERM, _handle_shutdown_signal)
|
||||
|
||||
|
||||
class _QuietSpinner:
|
||||
def update(self, *args, **kwargs) -> None:
|
||||
pass
|
||||
|
||||
def close(self, *args, **kwargs) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def ensure_vendor_runtime() -> None:
|
||||
try:
|
||||
VendorMapdInstaller(_QuietSpinner()).check_and_download()
|
||||
VendorMapdInstaller().verify()
|
||||
except Exception:
|
||||
cloudlog.exception("iq_maps: vendor runtime install/download failed")
|
||||
cloudlog.exception("iq_maps: vendor runtime verification failed")
|
||||
|
||||
params = Params()
|
||||
mem_params = Params("/dev/shm/params") if platform.system() != "Darwin" else params
|
||||
@@ -179,10 +154,6 @@ def _compose_region_selector(nations: list[str], states: list[str] | None = None
|
||||
|
||||
|
||||
def _fetch_tile_bundles(region_selector: str, abort_check=None) -> None:
|
||||
"""Download the offline on-screen map display tiles for the selected regions.
|
||||
|
||||
Separate asset from mapd's routing data: the on-screen map's OsmOfflineProvider reads
|
||||
raster .mbtiles bundles, so a region selection installs both when OfflineOSMaps is on."""
|
||||
global _tile_downloader
|
||||
if not params.get_bool("OfflineOSMaps"):
|
||||
return
|
||||
@@ -245,8 +216,6 @@ def _drive_vendor_fetch(region_selector: str, requested_regions: dict) -> None:
|
||||
break
|
||||
cloudlog.info(f"iq_maps: vendor map download finished for {region_selector}")
|
||||
if not cancelled and not _shutdown.is_set():
|
||||
# OSMDownloadLocations stays set until the finally below, so the konn3kt cancel RPC
|
||||
# (which removes it) aborts the tile phase exactly like it cancels the mapd phase.
|
||||
_fetch_tile_bundles(region_selector, abort_check=lambda: _shutdown.is_set() or not mem_params.get("OSMDownloadLocations"))
|
||||
except Exception:
|
||||
cloudlog.exception("iq_maps: vendor map download failed")
|
||||
@@ -310,17 +279,12 @@ _last_auto_restore_t = 0.0
|
||||
|
||||
|
||||
def region_data_missing() -> bool:
|
||||
# a media wipe (reflash/format) can delete the downloaded region while the params
|
||||
# that configure offline maps survive; mapd then retries the missing files forever
|
||||
# and nothing re-downloads (stale_region_artifacts only sees leftover files)
|
||||
if not params.get_bool("OsmLocal"):
|
||||
return False
|
||||
if not params.get("OsmDownloadedDate"):
|
||||
return False
|
||||
if glob.glob(f"{Paths.mapd_root()}/db") or glob.glob(f"{Paths.mapd_root()}/v*"):
|
||||
return False
|
||||
# mapd v2 stores region tiles under offline/<evenLat>/<evenLon>.tar.gz — without this
|
||||
# check a v2 install looks perpetually wiped and re-downloads every backoff interval
|
||||
if glob.glob(f"{Paths.mapd_root()}/offline/*/*"):
|
||||
return False
|
||||
country = params.get("OsmLocationName", return_default=True)
|
||||
@@ -328,8 +292,6 @@ def region_data_missing() -> bool:
|
||||
|
||||
|
||||
def configured_states() -> list[str]:
|
||||
"""Selected US states: OsmStateNames (JSON list, multi-state) wins; the legacy
|
||||
single OsmStateName remains the fallback for pre-list configs."""
|
||||
try:
|
||||
states = params.get("OsmStateNames")
|
||||
if isinstance(states, bytes):
|
||||
@@ -376,8 +338,6 @@ def _configured_region_selector() -> str:
|
||||
|
||||
|
||||
def tile_bundles_missing() -> bool:
|
||||
# covers a media wipe AND the user enabling OfflineOSMaps after the region download
|
||||
# already ran (the vendor fetch only pulls tile bundles when the toggle is on)
|
||||
if not params.get_bool("OfflineOSMaps"):
|
||||
return False
|
||||
selector = _configured_region_selector()
|
||||
@@ -387,8 +347,6 @@ def tile_bundles_missing() -> bool:
|
||||
|
||||
|
||||
def maybe_restore_tile_bundles() -> None:
|
||||
"""Tile-only download: don't re-run the whole mapd vendor fetch when only the display
|
||||
tiles are missing."""
|
||||
global _last_tile_restore_t, _tile_only_worker
|
||||
if not tile_bundles_missing():
|
||||
return
|
||||
@@ -416,9 +374,6 @@ def sync_osm_request_flags() -> None:
|
||||
maybe_restore_tile_bundles()
|
||||
if params.get_bool("OsmDbUpdatesCheck"):
|
||||
if _region_sync_worker is not None and _region_sync_worker.is_alive():
|
||||
# A download is already writing into Paths.mapd_root() - deleting/rewriting
|
||||
# files under it right now would race the writer (and any onroad mapd
|
||||
# reader) the same way the orphaned-subprocess bug did. Wait for it to finish.
|
||||
return
|
||||
purge_stale_region_artifacts(stale_region_artifacts())
|
||||
country = params.get("OsmLocationName", return_default=True)
|
||||
@@ -445,11 +400,6 @@ def run_loop():
|
||||
pass
|
||||
except PermissionError:
|
||||
cloudlog.exception(f"iq_maps: failed to make {Paths.mapd_root()}")
|
||||
|
||||
# A prior run that got SIGKILLed (or crashed) may have left its vendor-fetch
|
||||
# mapd subprocess running and still writing into Paths.mapd_root(); clear it
|
||||
# before anything (including the onroad mapd, once `started` flips) reads
|
||||
# from that directory. Signal handlers cover the graceful-shutdown path.
|
||||
_reap_orphaned_vendor_fetch()
|
||||
_install_signal_handlers()
|
||||
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
"""
|
||||
Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
|
||||
|
||||
Shared tunables and a small debug logger for the offline road-name / turn-speed path.
|
||||
"""
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
|
||||
# seconds of road ahead we scan for upcoming turn-speed zones published on iqLiveData
|
||||
LOOK_AHEAD_HORIZON_TIME = 15.0
|
||||
# clear the on-screen road name once it has gone this long without a refresh (s)
|
||||
ROAD_NAME_TIMEOUT = 30
|
||||
|
||||
R = 6373000.0 # mean Earth radius in metres (great-circle distance math)
|
||||
QUERY_RADIUS = 3000 # online OSM query reach, metres
|
||||
QUERY_RADIUS_OFFLINE = 2250 # offline-tile OSM query reach, metres
|
||||
|
||||
_DEBUG = False
|
||||
_CLOUDLOG_DEBUG = False
|
||||
|
||||
|
||||
def debug_road_data(msg, log_to_cloud=True):
|
||||
if _CLOUDLOG_DEBUG and log_to_cloud:
|
||||
cloudlog.debug(msg)
|
||||
if _DEBUG:
|
||||
print(msg)
|
||||
@@ -1,67 +0,0 @@
|
||||
"""
|
||||
Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
|
||||
"""
|
||||
import json
|
||||
import math
|
||||
import platform
|
||||
|
||||
from cereal import custom
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.iqpilot.iq_maps.road_data.signal_bridge import RoadSignalBridge
|
||||
from openpilot.iqpilot.navd.helpers import Coordinate
|
||||
|
||||
|
||||
class IQRoadLayer(RoadSignalBridge):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.mem_params = Params("/dev/shm/params") if platform.system() != "Darwin" else self.params
|
||||
|
||||
def refresh_position(self) -> None:
|
||||
location = self.location_sub['iqLiveLocation']
|
||||
self.fix_ready = (
|
||||
location.solutionState == custom.IQLiveLocation.SolutionState.ready
|
||||
and location.geodeticPosition.isValid
|
||||
)
|
||||
|
||||
if self.fix_ready:
|
||||
self.heading_deg = math.degrees(location.alignedOrientationNed.values[2])
|
||||
self.last_coordinate = Coordinate(location.geodeticPosition.values[0], location.geodeticPosition.values[1])
|
||||
|
||||
if self.last_coordinate is None:
|
||||
return
|
||||
|
||||
payload = {
|
||||
"latitude": self.last_coordinate.latitude,
|
||||
"longitude": self.last_coordinate.longitude,
|
||||
}
|
||||
|
||||
if self.heading_deg is not None:
|
||||
payload["bearing"] = self.heading_deg
|
||||
|
||||
self.mem_params.put("LastGPSPosition", json.dumps(payload))
|
||||
|
||||
def read_current_limit(self) -> float:
|
||||
return float(self.mem_params.get("MapSpeedLimit") or 0.0)
|
||||
|
||||
def read_current_road(self) -> str:
|
||||
return str(self.mem_params.get("RoadName") or "")
|
||||
|
||||
def read_upcoming_limit(self) -> tuple[float, float]:
|
||||
raw_segment = self.mem_params.get("NextMapSpeedLimit")
|
||||
if isinstance(raw_segment, bytes):
|
||||
raw_segment = raw_segment.decode("utf-8")
|
||||
try:
|
||||
upcoming_segment = json.loads(raw_segment) if isinstance(raw_segment, str) and raw_segment else (raw_segment or {})
|
||||
except json.JSONDecodeError:
|
||||
upcoming_segment = {}
|
||||
|
||||
next_limit = float(upcoming_segment.get("speedlimit", 0.0) or 0.0)
|
||||
target_lat = upcoming_segment.get("latitude")
|
||||
target_lon = upcoming_segment.get("longitude")
|
||||
distance_to_limit = 0.0
|
||||
|
||||
if target_lat is not None and target_lon is not None:
|
||||
limit_coordinate = Coordinate(float(target_lat), float(target_lon))
|
||||
distance_to_limit = (self.last_coordinate or Coordinate(0, 0)).distance_to(limit_coordinate)
|
||||
|
||||
return next_limit, distance_to_limit
|
||||
@@ -1,35 +0,0 @@
|
||||
"""
|
||||
Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
|
||||
"""
|
||||
import threading
|
||||
import traceback
|
||||
|
||||
from openpilot.common.realtime import Ratekeeper, config_realtime_process
|
||||
from openpilot.iqpilot.iq_maps.road_data import debug_road_data
|
||||
from openpilot.iqpilot.iq_maps.road_data.iq_road_layer import IQRoadLayer
|
||||
|
||||
ROAD_LAYER_HZ = 1
|
||||
ROAD_LAYER_CORES = [0, 1, 2, 3]
|
||||
|
||||
|
||||
def _log_thread_exception(args) -> None:
|
||||
debug_road_data(f"IQ maps threading exception:\n{args}")
|
||||
traceback.print_exception(args.exc_type, args.exc_value, args.exc_traceback)
|
||||
|
||||
|
||||
def run() -> None:
|
||||
config_realtime_process(ROAD_LAYER_CORES, 5)
|
||||
layer = IQRoadLayer()
|
||||
rk = Ratekeeper(ROAD_LAYER_HZ, print_delay_threshold=None)
|
||||
while True:
|
||||
layer.step()
|
||||
rk.keep_time()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
threading.excepthook = _log_thread_exception
|
||||
run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,62 +0,0 @@
|
||||
"""
|
||||
Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
|
||||
"""
|
||||
from abc import abstractmethod, ABC
|
||||
|
||||
import cereal.messaging as messaging
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.common.constants import CV
|
||||
from openpilot.selfdrive.car.cruise import V_CRUISE_UNSET
|
||||
from openpilot.iqpilot.navd.helpers import coordinate_from_param
|
||||
|
||||
ROAD_SPEED_CEILING = V_CRUISE_UNSET * CV.KPH_TO_MS
|
||||
|
||||
|
||||
class RoadSignalBridge(ABC):
|
||||
def __init__(self):
|
||||
self.params = Params()
|
||||
|
||||
self.location_sub = messaging.SubMaster(['iqLiveLocation'])
|
||||
self.output_pub = messaging.PubMaster(['iqLiveData'])
|
||||
|
||||
self.fix_ready = False
|
||||
self.heading_deg = None
|
||||
self.last_coordinate = coordinate_from_param("LastGPSPositionIQLoc", self.params)
|
||||
|
||||
@abstractmethod
|
||||
def refresh_position(self) -> None:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def read_current_limit(self) -> float:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def read_upcoming_limit(self) -> tuple[float, float]:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def read_current_road(self) -> str:
|
||||
pass
|
||||
|
||||
def publish_snapshot(self) -> None:
|
||||
active_limit = self.read_current_limit()
|
||||
next_limit, next_limit_distance = self.read_upcoming_limit()
|
||||
|
||||
outbound = messaging.new_message('iqLiveData')
|
||||
outbound.valid = self.location_sub['iqLiveLocation'].gpsHealthy
|
||||
live_data = outbound.iqLiveData
|
||||
|
||||
live_data.speedLimitValid = bool(ROAD_SPEED_CEILING > active_limit > 0)
|
||||
live_data.speedLimit = active_limit
|
||||
live_data.speedLimitAheadValid = bool(ROAD_SPEED_CEILING > next_limit > 0)
|
||||
live_data.speedLimitAhead = next_limit
|
||||
live_data.speedLimitAheadDistance = next_limit_distance
|
||||
live_data.roadName = self.read_current_road()
|
||||
|
||||
self.output_pub.send('iqLiveData', outbound)
|
||||
|
||||
def step(self) -> None:
|
||||
self.location_sub.update(0)
|
||||
self.refresh_position()
|
||||
self.publish_snapshot()
|
||||
@@ -1,20 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
|
||||
|
||||
Downloads per-region raster display-tile bundles (.mbtiles) for the offline on-screen map.
|
||||
|
||||
These are a separate asset from mapd's routing/speed-limit data: mapd pulls OSM way tiles
|
||||
into Paths.mapd_root(), while the on-screen map (OsmOfflineProvider) reads raster .mbtiles
|
||||
from offline_map_root()/regions/<selector>/tiles/offline.mbtiles. Bundles are built per
|
||||
state/nation by scripts/iqpilot/build_state_tile_bundles.py and hosted behind a static base
|
||||
URL that serves:
|
||||
|
||||
<base>/index.json {"version": 1, "regions": {<selector>: entry}}
|
||||
<base>/<entry["path"]> the raster .mbtiles for that region
|
||||
|
||||
Entry fields: path, bytes, sha256, bounds ("minLon,minLat,maxLon,maxLat"), minzoom, maxzoom.
|
||||
Selectors match the mapd region menu naming: us_state.CA, nation.US.
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
@@ -25,46 +11,114 @@ from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
from openpilot.iqpilot.ui.onroad.offline_tiles import offline_map_root
|
||||
from iqpilot.common.params import Params
|
||||
from iqpilot.common.swaglog import cloudlog
|
||||
from iqpilot.ui.onroad.offline_tiles import offline_map_root
|
||||
|
||||
# Proprietary auth + hosted endpoints (gitea raw with an embedded read-only PAT, same
|
||||
# pattern as the model selector). Optional: without the private bundle the downloader
|
||||
# still works anonymously against OfflineTilesBaseUrl (e.g. a public R2 bucket).
|
||||
try:
|
||||
from openpilot.iqpilot.iq_maps.tiles_auth import get_base_urls as _private_base_urls, get_requests_auth as _private_auth
|
||||
from iqpilot.iq_maps.tiles_auth import get_base_urls as _private_base_urls, get_requests_auth as _private_auth
|
||||
except Exception: # ProprietaryModuleMissing or import errors in stripped builds
|
||||
_private_base_urls = None
|
||||
_private_auth = None
|
||||
|
||||
# R2 bucket iqnav behind the public custom domain (see scripts/iqpilot/tile_factory/r2_sync_watch.py)
|
||||
DEFAULT_TILE_BUNDLE_BASE_URL = "https://maps.konn3kt.com/iqosmd/v1"
|
||||
# Tile bundles live as LFS objects in the PRIVATE repo IQ.Lvbs/iqmaps (R2 is gone).
|
||||
# Anonymous access 404s by design; devices authenticate with the embedded read-only PAT
|
||||
# carried by the closed-source updater bundle (same fetch account as the OS images).
|
||||
# Hugging Face is primary: it is CDN-served, so device downloads no longer come off the
|
||||
# gitea box's home uplink. The gitea copies stay as failover -- if HF ever suspends the
|
||||
# repo the fleet silently falls back instead of losing maps entirely.
|
||||
HF_TILE_BUNDLE_BASE_URL = "https://huggingface.co/datasets/T3vl/iqmaps/resolve/main"
|
||||
DEFAULT_TILE_BUNDLE_BASE_URL = "https://git.konn3kt.com/IQ.Lvbs/iqmaps/raw/branch/master"
|
||||
FALLBACK_TILE_BUNDLE_BASE_URL = "https://gitlvb.teallvbs.xyz/IQ.Lvbs/iqmaps/raw/branch/master"
|
||||
|
||||
# Gitea /raw NEVER returns LFS content -- it returns this pointer, and the real bytes come
|
||||
# from the LFS batch API (see _resolve_object_url).
|
||||
LFS_POINTER_MAGIC = b"version https://git-lfs"
|
||||
BASE_URL_PARAM = "OfflineTilesBaseUrl"
|
||||
PROGRESS_PARAM = "OfflineTilesDownloadProgress"
|
||||
REQUEST_PARAM = "OfflineTilesDownloadRequest"
|
||||
CHUNK_BYTES = 1 << 20
|
||||
# must match scripts/iqpilot/tile_factory/upload_bundles_lfs.py
|
||||
PART_BYTES = 90 * 1024 * 1024
|
||||
HTTP_TIMEOUT_S = 30.0
|
||||
STREAM_RETRIES = 8
|
||||
|
||||
|
||||
def candidate_base_urls(params: Params) -> list[str]:
|
||||
"""Hosts to try in order: user/param override first, then the embedded private
|
||||
endpoints (gitea raw), then the public default."""
|
||||
override = params.get(BASE_URL_PARAM)
|
||||
if isinstance(override, bytes):
|
||||
override = override.decode("utf-8", errors="ignore")
|
||||
override = (override or "").strip()
|
||||
if override:
|
||||
return [override.rstrip("/")]
|
||||
urls: list[str] = []
|
||||
# HF first (CDN, and it keeps device traffic off the gitea box's uplink); the bundle's
|
||||
# own endpoints and the self-hosted defaults follow as failover.
|
||||
urls: list[str] = [HF_TILE_BUNDLE_BASE_URL]
|
||||
if _private_base_urls is not None:
|
||||
try:
|
||||
urls.extend(url.rstrip("/") for url in _private_base_urls())
|
||||
except Exception:
|
||||
pass
|
||||
urls.append(DEFAULT_TILE_BUNDLE_BASE_URL)
|
||||
return urls
|
||||
urls.append(FALLBACK_TILE_BUNDLE_BASE_URL)
|
||||
seen: set[str] = set()
|
||||
return [u for u in urls if not (u in seen or seen.add(u))]
|
||||
|
||||
|
||||
def _is_hf(url: str) -> bool:
|
||||
return "huggingface.co" in url.lower()
|
||||
|
||||
|
||||
def _maps_auth_module():
|
||||
"""The read PAT lives in the compiled updater bundle (never in this open file)."""
|
||||
try:
|
||||
from iqpilot.system.proprietary_runtime._verified_import import import_verified_module
|
||||
return import_verified_module("iqpilot_updater_private", "iqpilot_private.updater.git_remote")
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
import importlib
|
||||
import os
|
||||
import sys
|
||||
root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
||||
bundle_python = os.path.join(root, "artifacts", "iqpilot_updater_private", "python")
|
||||
if os.path.isdir(bundle_python):
|
||||
if bundle_python not in sys.path:
|
||||
sys.path.insert(0, bundle_python)
|
||||
return importlib.import_module("iqpilot_private.updater.git_remote")
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def request_headers(url: str) -> dict:
|
||||
mod = _maps_auth_module()
|
||||
if mod is not None:
|
||||
if _is_hf(url):
|
||||
# HF wants a bearer token, not basic auth; a build whose bundle predates HF
|
||||
# hosting simply gets nothing here and falls through to the gitea mirrors.
|
||||
try:
|
||||
token = mod.map_tiles_hf_token()
|
||||
if token:
|
||||
return {"Authorization": f"Bearer {token}"}
|
||||
except Exception:
|
||||
pass
|
||||
return {}
|
||||
try:
|
||||
headers = mod.map_tiles_headers(url)
|
||||
if headers:
|
||||
return headers
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
from iqpilot.common.git_creds import get_credentials
|
||||
creds = get_credentials()
|
||||
if creds and all(creds) and "/iq.lvbs/iqmaps" in url.lower():
|
||||
import base64
|
||||
return {"Authorization": "Basic " + base64.b64encode(f"{creds[0]}:{creds[1]}".encode()).decode()}
|
||||
except Exception:
|
||||
pass
|
||||
return {}
|
||||
|
||||
|
||||
def request_auth() -> tuple[str, str] | None:
|
||||
@@ -76,8 +130,69 @@ def request_auth() -> tuple[str, str] | None:
|
||||
return None
|
||||
|
||||
|
||||
def _lfs_endpoint(base_url: str) -> str:
|
||||
"""<host>/<owner>/<repo>/raw/branch/<b> -> <host>/<owner>/<repo>.git/info/lfs"""
|
||||
return base_url.split("/raw/", 1)[0] + ".git/info/lfs"
|
||||
|
||||
|
||||
def _resolve_oid_url(session: requests.Session, base_url: str, oid: str, size: int,
|
||||
headers: dict) -> tuple[str, dict]:
|
||||
"""Bundles are stored as bare LFS objects addressed by oid from the index -- no pointer
|
||||
files, because committing one per part meant hundreds of concurrent commits per branch."""
|
||||
batch = session.post(f"{_lfs_endpoint(base_url)}/objects/batch",
|
||||
data=json.dumps({"operation": "download", "transfers": ["basic"],
|
||||
"objects": [{"oid": oid, "size": size}]}),
|
||||
headers={"Content-Type": "application/vnd.git-lfs+json",
|
||||
"Accept": "application/vnd.git-lfs+json", **headers},
|
||||
timeout=HTTP_TIMEOUT_S)
|
||||
batch.raise_for_status()
|
||||
entry = batch.json()["objects"][0]
|
||||
if "actions" not in entry:
|
||||
raise requests.RequestException(f"LFS object unavailable: {entry.get('error', oid)}")
|
||||
action = entry["actions"]["download"]
|
||||
return action["href"], action.get("header", {})
|
||||
|
||||
|
||||
def _resolve_object_url(session: requests.Session, url: str, headers: dict) -> tuple[str, dict]:
|
||||
"""Follow a Gitea LFS pointer to the real (pre-signed) object URL.
|
||||
|
||||
Returns the URL to stream plus any extra headers it needs. A plain host that serves the
|
||||
bytes directly (local test server, static mirror) resolves to itself unchanged."""
|
||||
probe = session.get(url, headers={**headers, "Accept-Encoding": None}, stream=True,
|
||||
timeout=HTTP_TIMEOUT_S)
|
||||
probe.raise_for_status()
|
||||
# A host that serves the bytes itself still needs the caller's auth on the real GET --
|
||||
# returning {} here sends the download out anonymous and a private host answers 401.
|
||||
if int(probe.headers.get("content-length") or 0) >= 1024:
|
||||
probe.close()
|
||||
return url, dict(headers)
|
||||
body = probe.content
|
||||
probe.close()
|
||||
if not body.startswith(LFS_POINTER_MAGIC):
|
||||
return url, dict(headers)
|
||||
|
||||
meta = dict(line.split(" ", 1) for line in body.decode().strip().splitlines() if " " in line)
|
||||
oid = meta["oid"].split(":", 1)[1]
|
||||
size = int(meta["size"])
|
||||
lfs_base = url.split("/raw/", 1)[0] + ".git/info/lfs"
|
||||
batch = session.post(f"{lfs_base}/objects/batch",
|
||||
data=json.dumps({"operation": "download", "transfers": ["basic"],
|
||||
"objects": [{"oid": oid, "size": size}]}),
|
||||
headers={"Content-Type": "application/vnd.git-lfs+json",
|
||||
"Accept": "application/vnd.git-lfs+json", **headers},
|
||||
timeout=HTTP_TIMEOUT_S)
|
||||
batch.raise_for_status()
|
||||
action = batch.json()["objects"][0]["actions"]["download"]
|
||||
return action["href"], action.get("header", {})
|
||||
|
||||
|
||||
def fetch_index(base_url: str, session: requests.Session) -> dict:
|
||||
response = session.get(f"{base_url}/index.json", timeout=HTTP_TIMEOUT_S, auth=request_auth())
|
||||
index_url = f"{base_url}/index.json"
|
||||
headers = request_headers(index_url)
|
||||
# requests' auth= rewrites the Authorization header, so only fall back to it when the
|
||||
# closed-source bundle gave us nothing.
|
||||
response = session.get(index_url, timeout=HTTP_TIMEOUT_S, headers=headers,
|
||||
auth=None if headers else request_auth())
|
||||
response.raise_for_status()
|
||||
index = response.json()
|
||||
regions = index.get("regions")
|
||||
@@ -94,6 +209,25 @@ def region_bundle_path(selector: str) -> Path:
|
||||
return region_bundle_dir(selector) / "tiles" / "offline.mbtiles"
|
||||
|
||||
|
||||
def region_valhalla_path(selector: str) -> Path:
|
||||
# valhalla mmaps this tar in place, so it stays uncompressed on disk
|
||||
return region_bundle_dir(selector) / "valhalla" / "tiles.tar"
|
||||
|
||||
|
||||
def region_valhalla_installed(selector: str) -> bool:
|
||||
return region_valhalla_path(selector).exists()
|
||||
|
||||
|
||||
def installed_valhalla_selectors() -> list[str]:
|
||||
regions_root = offline_map_root() / "regions"
|
||||
if not regions_root.exists():
|
||||
return []
|
||||
return sorted(
|
||||
child.name for child in regions_root.iterdir()
|
||||
if child.is_dir() and (child / "valhalla" / "tiles.tar").exists()
|
||||
)
|
||||
|
||||
|
||||
def region_bundle_installed(selector: str) -> bool:
|
||||
return region_bundle_path(selector).exists()
|
||||
|
||||
@@ -144,12 +278,6 @@ def _write_manifest(selector: str, entry: dict) -> None:
|
||||
|
||||
|
||||
class TileBundleDownloader:
|
||||
"""Streams region bundles to disk with resume + sha256 verify + atomic install.
|
||||
|
||||
Cancellation matches the mapd flow: the caller sets REQUEST_PARAM in mem params while a
|
||||
download runs; removing it (konn3kt cancel RPC or settings) aborts between chunks. The
|
||||
partial .part file is kept so a retry resumes instead of restarting.
|
||||
"""
|
||||
|
||||
def __init__(self, params: Params | None = None, mem_params: Params | None = None,
|
||||
abort_check=None):
|
||||
@@ -160,7 +288,6 @@ class TileBundleDownloader:
|
||||
self.mem_params = Params("/dev/shm/params") if platform.system() != "Darwin" else self.params
|
||||
self.session = requests.Session()
|
||||
self._cancelled = threading.Event()
|
||||
# optional external cancel signal, e.g. the orchestrator's OSMDownloadLocations removal
|
||||
self._abort_check = abort_check
|
||||
|
||||
def cancel(self) -> None:
|
||||
@@ -170,7 +297,6 @@ class TileBundleDownloader:
|
||||
if self._cancelled.is_set():
|
||||
return True
|
||||
if not self.mem_params.get(REQUEST_PARAM):
|
||||
# request flag was removed out from under us -> user cancelled
|
||||
self._cancelled.set()
|
||||
return True
|
||||
if self._abort_check is not None and self._abort_check():
|
||||
@@ -188,12 +314,11 @@ class TileBundleDownloader:
|
||||
|
||||
def _download_one(self, selector: str, entry: dict, base_url: str,
|
||||
progress_offset: int, progress_total: int) -> bool:
|
||||
"""Download a region: the night bundle, plus the optional day-style variant."""
|
||||
night_path = region_bundle_path(selector)
|
||||
ok = self._download_file(
|
||||
selector, base_url, entry["path"], int(entry.get("bytes", 0)),
|
||||
str(entry.get("sha256", "")).strip().lower(), night_path,
|
||||
progress_offset, progress_total,
|
||||
progress_offset, progress_total, int(entry.get("parts", 1)), entry.get("objects"),
|
||||
)
|
||||
if not ok:
|
||||
return False
|
||||
@@ -203,24 +328,47 @@ class TileBundleDownloader:
|
||||
str(entry.get("day_sha256", "")).strip().lower(),
|
||||
night_path.with_name("offline_day.mbtiles"),
|
||||
progress_offset + int(entry.get("bytes", 0)), progress_total,
|
||||
int(entry.get("day_parts", 1)), entry.get("day_objects"),
|
||||
)
|
||||
if not day_ok:
|
||||
# the night set is complete and usable; a failed day variant retries next pass
|
||||
cloudlog.warning(f"iq_maps: day-style bundle failed for {selector}; night set installed")
|
||||
# manifest last: bounds drive region matching, so it must describe installed files
|
||||
if entry.get("valhalla_path"):
|
||||
# routing is additive: a region whose extract is missing or corrupt must still end up
|
||||
# with a usable map rather than failing the whole download
|
||||
try:
|
||||
nav_ok = self._download_file(
|
||||
selector, base_url, entry["valhalla_path"], int(entry.get("valhalla_bytes", 0)),
|
||||
str(entry.get("valhalla_sha256", "")).strip().lower(),
|
||||
region_valhalla_path(selector),
|
||||
progress_offset + int(entry.get("bytes", 0)) + int(entry.get("day_bytes", 0)),
|
||||
progress_total, 1, entry.get("valhalla_objects"),
|
||||
)
|
||||
except Exception as exc:
|
||||
nav_ok = False
|
||||
cloudlog.warning(f"iq_maps: routing extract errored for {selector}: {exc}")
|
||||
if not nav_ok:
|
||||
cloudlog.warning(f"iq_maps: routing extract failed for {selector}; map tiles installed")
|
||||
_write_manifest(selector, entry)
|
||||
cloudlog.info(f"iq_maps: installed tile bundle {selector}")
|
||||
return True
|
||||
|
||||
def _download_file(self, selector: str, base_url: str, remote_path: str, expected_bytes: int,
|
||||
expected_sha: str, final_path: Path,
|
||||
progress_offset: int, progress_total: int) -> bool:
|
||||
url = f"{base_url}/{remote_path.lstrip('/')}"
|
||||
progress_offset: int, progress_total: int, parts: int = 1,
|
||||
objects: list | None = None) -> bool:
|
||||
# Bundles are published as <name>.pNN because Cloudflare caps proxied bodies at ~100MB.
|
||||
# They stream back-to-back into ONE .part file: concatenating afterwards would need
|
||||
# double the free space, which devices do not have.
|
||||
base = f"{base_url}/{remote_path.lstrip('/')}"
|
||||
count = len(objects) if objects else parts
|
||||
if objects and not _is_hf(base_url):
|
||||
urls = [None] * count # gitea: resolved per-attempt from the oid
|
||||
else:
|
||||
# HF (and plain mirrors) serve the same chunks as ordinary .pNN files
|
||||
urls = [base] if count <= 1 else [f"{base}.p{i:02d}" for i in range(count)]
|
||||
part_path = final_path.with_name(final_path.name + ".part")
|
||||
part_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# A cellular/hotspot link routinely kills a multi-hundred-MB stream mid-flight; retry
|
||||
# each interruption from the current .part offset instead of failing the whole region.
|
||||
downloaded = 0
|
||||
digest = hashlib.sha256()
|
||||
last_error: Exception | None = None
|
||||
@@ -240,29 +388,45 @@ class TileBundleDownloader:
|
||||
digest = hashlib.sha256()
|
||||
resume_from = 0
|
||||
|
||||
headers = {"Range": f"bytes={resume_from}-"} if resume_from else {}
|
||||
auth = request_auth()
|
||||
response = self.session.get(url, headers=headers, stream=True, timeout=HTTP_TIMEOUT_S, auth=auth)
|
||||
if resume_from and response.status_code != 206:
|
||||
# server ignored the Range request -> restart from scratch
|
||||
digest = hashlib.sha256()
|
||||
resume_from = 0
|
||||
part_path.unlink(missing_ok=True)
|
||||
if response.status_code == 416:
|
||||
response = self.session.get(url, stream=True, timeout=HTTP_TIMEOUT_S, auth=auth)
|
||||
response.raise_for_status()
|
||||
|
||||
# every part but the last is exactly PART_BYTES, so a byte offset maps to a part index
|
||||
first_part = resume_from // PART_BYTES if len(urls) > 1 else 0
|
||||
skip_in_part = resume_from - first_part * PART_BYTES if len(urls) > 1 else resume_from
|
||||
downloaded = resume_from
|
||||
mode = "ab" if resume_from else "wb"
|
||||
with open(part_path, mode) as f:
|
||||
for chunk in response.iter_content(chunk_size=CHUNK_BYTES):
|
||||
if self._should_abort():
|
||||
cloudlog.warning(f"iq_maps: tile bundle download cancelled for {selector}")
|
||||
return False
|
||||
f.write(chunk)
|
||||
digest.update(chunk)
|
||||
downloaded += len(chunk)
|
||||
self._publish_progress(selector, progress_offset + downloaded, progress_total, active=True)
|
||||
for index in range(first_part, len(urls)):
|
||||
if objects and not _is_hf(base_url):
|
||||
url_headers = request_headers(base_url)
|
||||
auth = None if url_headers else request_auth()
|
||||
object_url, object_headers = _resolve_oid_url(
|
||||
self.session, base_url, objects[index]["oid"], int(objects[index]["size"]),
|
||||
url_headers)
|
||||
else:
|
||||
url = urls[index]
|
||||
url_headers = request_headers(url)
|
||||
auth = None if url_headers else request_auth()
|
||||
# Re-resolve per part: a pre-signed LFS object URL can expire mid-download.
|
||||
object_url, object_headers = _resolve_object_url(self.session, url, url_headers)
|
||||
headers = dict(object_headers)
|
||||
offset = skip_in_part if index == first_part else 0
|
||||
if offset:
|
||||
headers["Range"] = f"bytes={offset}-"
|
||||
response = self.session.get(object_url, headers=headers, stream=True,
|
||||
timeout=HTTP_TIMEOUT_S, auth=auth)
|
||||
if offset and response.status_code != 206:
|
||||
# server ignored the range: restart this whole file cleanly
|
||||
f.close()
|
||||
part_path.unlink(missing_ok=True)
|
||||
raise requests.RequestException(f"range not honoured for part {index}")
|
||||
response.raise_for_status()
|
||||
for chunk in response.iter_content(chunk_size=CHUNK_BYTES):
|
||||
if self._should_abort():
|
||||
cloudlog.warning(f"iq_maps: tile bundle download cancelled for {selector}")
|
||||
return False
|
||||
f.write(chunk)
|
||||
digest.update(chunk)
|
||||
downloaded += len(chunk)
|
||||
self._publish_progress(selector, progress_offset + downloaded, progress_total, active=True)
|
||||
break
|
||||
except requests.RequestException as exc:
|
||||
last_error = exc
|
||||
@@ -284,8 +448,6 @@ class TileBundleDownloader:
|
||||
return True
|
||||
|
||||
def download_regions(self, selectors: list[str]) -> bool:
|
||||
"""Download the display-tile bundles for the given region selectors. Returns True if all
|
||||
requested bundles are installed and current when done."""
|
||||
self._cancelled.clear()
|
||||
ok = True
|
||||
try:
|
||||
@@ -346,7 +508,6 @@ class TileBundleDownloader:
|
||||
if not expected_sha or installed_sha != expected_sha:
|
||||
return False
|
||||
if entry.get("day_path"):
|
||||
# a published day variant must be installed and current too
|
||||
day_file = region_bundle_dir(selector) / "tiles" / "offline_day.mbtiles"
|
||||
installed_day = str(manifest.get("mbtiles_day", {}).get("sha256", "")).strip().lower()
|
||||
expected_day = str(entry.get("day_sha256", "")).strip().lower()
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
"""
|
||||
Copyright (c) IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
|
||||
"""
|
||||
from openpilot.iqpilot._proprietary_loader import ProprietaryModuleMissing, load_private_module
|
||||
from iqpilot._proprietary_loader import ProprietaryModuleMissing, load_private_module
|
||||
|
||||
try:
|
||||
load_private_module(__name__, "iqpilot_private.maps.git_auth")
|
||||
except ProprietaryModuleMissing:
|
||||
from iqpilot.maps_private_src.git_auth import * # noqa: F403
|
||||
from iqpilot.maps_private_src.git_auth import *
|
||||
|
||||
@@ -1,25 +1,21 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
|
||||
|
||||
Maintainer utility: pin a new pfeiferj/mapd release tag and refresh the checked-in
|
||||
binary hash. Not used at runtime.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from openpilot.common.basedir import BASEDIR
|
||||
from openpilot.iqpilot.iq_maps import VENDOR_MAPD_PATH
|
||||
from openpilot.iqpilot.iq_maps.vendor_mapd_installer import (
|
||||
from iqpilot.common.basedir import BASEDIR
|
||||
from iqpilot.iq_maps import VENDOR_MAPD_PATH
|
||||
from iqpilot.iq_maps.vendor_mapd_installer import (
|
||||
VENDOR_RELEASE_TAG,
|
||||
sha256_of_file,
|
||||
)
|
||||
|
||||
_RELEASE_SYMBOL = "VENDOR_RELEASE_TAG"
|
||||
_INSTALLER_SRC = os.path.join(BASEDIR, "iqpilot", "iq_maps", "vendor_mapd_installer.py")
|
||||
# public: the checked-in hash the version test compares the installed binary against
|
||||
HASH_FILE = os.path.join(BASEDIR, "iqpilot", "iq_maps", "tests", "mapd_hash")
|
||||
_HASH_FILE = HASH_FILE
|
||||
_TAG_ASSIGN = re.compile(rf'^{_RELEASE_SYMBOL}\s*=\s*["\'][^"\']*["\']', re.MULTILINE)
|
||||
|
||||
@@ -1,41 +1,26 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
|
||||
|
||||
Provisions the `mapd` routing binary authored by Jacob Pfeifer (github.com/pfeiferj/mapd).
|
||||
The binary itself is his work; this module only fetches, verifies and stages it on-device.
|
||||
"""
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import stat
|
||||
import time
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
import requests
|
||||
from iqpilot.common.basedir import BASEDIR
|
||||
from iqpilot.common.params import Params
|
||||
from iqpilot.common.spinner import Spinner
|
||||
from iqpilot.common.swaglog import cloudlog
|
||||
from iqpilot.iq_maps import VENDOR_MAPD_PATH
|
||||
import iqpilot.system.sentry as sentry
|
||||
|
||||
from cereal import messaging
|
||||
from openpilot.common.basedir import BASEDIR
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.common.spinner import Spinner
|
||||
from openpilot.system.hardware.hw import Paths
|
||||
from openpilot.system.version import is_prebuilt
|
||||
from openpilot.iqpilot.iq_maps import VENDOR_MAPD_BIN_DIR, VENDOR_MAPD_PATH
|
||||
import openpilot.system.sentry as sentry
|
||||
|
||||
VENDOR_RELEASE_TAG = "v2.0.6"
|
||||
VENDOR_RELEASE_URL = f"https://github.com/pfeiferj/mapd/releases/download/{VENDOR_RELEASE_TAG}/mapd"
|
||||
VENDOR_RELEASE_TAG = "v2.0.6-iq1"
|
||||
|
||||
_VERSION_PARAM = "MapdVersion"
|
||||
_HASH_FILE = os.path.join(BASEDIR, "iqpilot", "iq_maps", "tests", "mapd_hash")
|
||||
_HTTP_TIMEOUT_S = 60
|
||||
_FETCH_ATTEMPTS = 5
|
||||
_NET_PROBE_ATTEMPTS = 10
|
||||
_NET_PROBE_INTERVAL_S = 2
|
||||
QUARANTINE_PATH = VENDOR_MAPD_PATH + ".quarantined"
|
||||
|
||||
|
||||
def sha256_of_file(path: str) -> str:
|
||||
"""Hex SHA-256 digest of a file on disk."""
|
||||
digest = hashlib.sha256()
|
||||
with open(path, "rb") as handle:
|
||||
for block in iter(lambda: handle.read(1 << 20), b""):
|
||||
@@ -48,45 +33,54 @@ def stamp_vendor_version(version: str, params: Params | None = None) -> None:
|
||||
|
||||
|
||||
class VendorMapdInstaller:
|
||||
def __init__(self, spinner_ref: Spinner):
|
||||
def __init__(self, spinner_ref: Spinner | None = None, params: Params | None = None):
|
||||
self._spinner = spinner_ref
|
||||
self._params = Params()
|
||||
self._params = params if params is not None else Params()
|
||||
|
||||
# --- externally consumed surface -----------------------------------------
|
||||
def get_installed_version(self) -> str:
|
||||
return str(self._params.get(_VERSION_PARAM) or "")
|
||||
|
||||
@staticmethod
|
||||
def ensure_directories_exist() -> None:
|
||||
for directory in (Paths.mapd_root(), VENDOR_MAPD_BIN_DIR):
|
||||
os.makedirs(directory, exist_ok=True)
|
||||
def verify(self) -> bool:
|
||||
expected = self._expected_hash()
|
||||
if not expected:
|
||||
cloudlog.error("iq_maps: pinned mapd hash missing, vendor binary cannot be verified")
|
||||
return False
|
||||
|
||||
def check_and_download(self) -> None:
|
||||
if not self._binary_up_to_date():
|
||||
self._provision()
|
||||
|
||||
def non_prebuilt_install(self) -> None:
|
||||
if self._on_metered_link():
|
||||
self._say("Metered connection detected — offline maps engine will not download here.")
|
||||
time.sleep(5)
|
||||
return
|
||||
if not os.path.isfile(VENDOR_MAPD_PATH):
|
||||
self._say("Offline maps engine missing; it will be restored by the next update.")
|
||||
self._params.remove(_VERSION_PARAM)
|
||||
return False
|
||||
|
||||
try:
|
||||
self.ensure_directories_exist()
|
||||
if self._binary_up_to_date():
|
||||
self._say("Offline maps engine already present and current.")
|
||||
time.sleep(0.1)
|
||||
return
|
||||
current = sha256_of_file(VENDOR_MAPD_PATH)
|
||||
except OSError:
|
||||
cloudlog.exception("iq_maps: vendor mapd unreadable")
|
||||
return False
|
||||
|
||||
if self._block_until_online():
|
||||
self._say(f"Retrieving offline maps engine [{self.get_installed_version() or 'none'}] -> [{VENDOR_RELEASE_TAG}]")
|
||||
time.sleep(0.1)
|
||||
self._provision()
|
||||
self._spinner.close()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self._announce_failure(exc)
|
||||
if current == expected:
|
||||
stamp_vendor_version(VENDOR_RELEASE_TAG, self._params)
|
||||
try:
|
||||
os.remove(QUARANTINE_PATH)
|
||||
except OSError:
|
||||
pass
|
||||
self._say(f"Offline maps engine verified [{VENDOR_RELEASE_TAG}]")
|
||||
return True
|
||||
|
||||
cloudlog.error(f"iq_maps: vendor mapd hash {current[:12]} != pinned {expected[:12]}, quarantining")
|
||||
self._say("Offline maps engine failed verification; quarantined until the next update.")
|
||||
try:
|
||||
os.replace(VENDOR_MAPD_PATH, QUARANTINE_PATH)
|
||||
except OSError:
|
||||
cloudlog.exception("iq_maps: vendor mapd quarantine failed")
|
||||
return False
|
||||
self._params.remove(_VERSION_PARAM)
|
||||
try:
|
||||
raise RuntimeError(f"vendor mapd hash mismatch quarantined: {current}")
|
||||
except RuntimeError as exc:
|
||||
sentry.init(sentry.SentryProject.SELFDRIVE)
|
||||
sentry.capture_exception(exc)
|
||||
return False
|
||||
|
||||
# --- internal ------------------------------------------------------------
|
||||
def _expected_hash(self) -> str:
|
||||
try:
|
||||
with open(_HASH_FILE) as f:
|
||||
@@ -94,88 +88,13 @@ class VendorMapdInstaller:
|
||||
except OSError:
|
||||
return ""
|
||||
|
||||
def _binary_up_to_date(self) -> bool:
|
||||
if not os.path.exists(VENDOR_MAPD_PATH):
|
||||
return False
|
||||
if self.get_installed_version() != VENDOR_RELEASE_TAG:
|
||||
return False
|
||||
reference = self._expected_hash()
|
||||
if not reference:
|
||||
return True
|
||||
try:
|
||||
return sha256_of_file(VENDOR_MAPD_PATH) == reference
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
def _provision(self) -> None:
|
||||
self.ensure_directories_exist()
|
||||
if self._retrieve_binary():
|
||||
stamp_vendor_version(VENDOR_RELEASE_TAG, self._params)
|
||||
|
||||
def _retrieve_binary(self) -> bool:
|
||||
staging = Path(f"{VENDOR_MAPD_PATH}.part")
|
||||
last_error: Exception | None = None
|
||||
for attempt in range(1, _FETCH_ATTEMPTS + 1):
|
||||
try:
|
||||
with requests.get(VENDOR_RELEASE_URL, stream=True, timeout=_HTTP_TIMEOUT_S) as resp:
|
||||
resp.raise_for_status()
|
||||
with open(staging, "wb") as out:
|
||||
for chunk in resp.iter_content(chunk_size=1 << 16):
|
||||
out.write(chunk)
|
||||
out.flush()
|
||||
os.fsync(out.fileno())
|
||||
os.chmod(staging, os.lstat(staging).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
||||
staging.replace(VENDOR_MAPD_PATH)
|
||||
return True
|
||||
except requests.exceptions.RequestException as exc:
|
||||
last_error = exc
|
||||
self._say(f"offline maps fetch attempt {attempt}/{_FETCH_ATTEMPTS} did not complete ({exc})")
|
||||
time.sleep(0.5)
|
||||
staging.unlink(missing_ok=True)
|
||||
logging.error("offline maps engine could not be fetched after %d attempts: %s", _FETCH_ATTEMPTS, last_error)
|
||||
return False
|
||||
|
||||
def _on_metered_link(self) -> bool:
|
||||
sm = messaging.SubMaster(["deviceState"])
|
||||
return bool(sm["deviceState"].networkMetered)
|
||||
|
||||
def _block_until_online(self) -> bool:
|
||||
for i in range(1, _NET_PROBE_ATTEMPTS + 1):
|
||||
self._say(f"Waiting for a usable network connection... [{i}/{_NET_PROBE_ATTEMPTS}]")
|
||||
if self._link_reachable():
|
||||
return True
|
||||
time.sleep(_NET_PROBE_INTERVAL_S)
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _link_reachable() -> bool:
|
||||
try:
|
||||
requests.head(VENDOR_RELEASE_URL, timeout=10, allow_redirects=True)
|
||||
return True
|
||||
except requests.exceptions.RequestException as exc:
|
||||
logging.debug("network probe failed: %s", exc)
|
||||
return False
|
||||
|
||||
def _announce_failure(self, exc: Exception) -> None:
|
||||
for remaining in range(5, 0, -1):
|
||||
self._say(f"Offline maps engine unavailable; navigation stays online-only. Boot continues in {remaining}s...")
|
||||
time.sleep(1)
|
||||
logging.exception("vendor mapd install failed")
|
||||
sentry.init(sentry.SentryProject.SELFDRIVE)
|
||||
sentry.capture_exception(exc)
|
||||
|
||||
def _say(self, text: str) -> None:
|
||||
self._spinner.update(text)
|
||||
if self._spinner is not None:
|
||||
self._spinner.update(text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
spinner = Spinner()
|
||||
installer = VendorMapdInstaller(spinner)
|
||||
installer.ensure_directories_exist()
|
||||
if is_prebuilt():
|
||||
spinner.update(f"[DEBUG] Prebuilt build; vendor mapd install skipped. "
|
||||
f"target [{VENDOR_RELEASE_TAG}], param [{installer.get_installed_version()}]")
|
||||
stamp_vendor_version(VENDOR_RELEASE_TAG)
|
||||
else:
|
||||
spinner.update(f"Verifying vendor mapd install. prebuilt [{is_prebuilt()}]")
|
||||
installer.non_prebuilt_install()
|
||||
ok = VendorMapdInstaller(spinner).verify()
|
||||
spinner.close()
|
||||
sys.exit(0 if ok else 1)
|
||||
|
||||
Reference in New Issue
Block a user