IQ.Lvbs/IQ.Pilot
8
7
Fork 3
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

IQ.Pilot Release Commit @ bec7652

Browse Source
This commit is contained in:
IQ.Lvbs CI [bot]
2026-08-22 21:29:55 -05:00
parent 11cefcb266
commit 7786d94c33
58 changed files with 578 additions and 267 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
iqpilot/iq_maps/orchestrator.py
View File

@@ -26,25 +26,9 @@ from iqpilot.iq_maps.vendor_mapd_installer import VendorMapdInstaller
OfflineMapAction = custom.MapdInputType
_region_sync_worker: threading.Thread | None = None
# mapd_manager only runs offroad (process_config.only_offroad) and the onroad
# NativeProcess("mapd", ...) is started the instant `started` flips True. If a
# vendor-map download is in flight at that exact moment, the two `mapd`
# binaries end up pointed at the same Paths.mapd_root() tile directory at the
# same time: this one still downloading/writing, the onroad one already
# mmap-reading. Manager only sends SIGINT/SIGTERM to stop mapd_manager, which
# by default only interrupts the main thread — the background download thread
# and the vendor `mapd` subprocess it spawned are otherwise orphaned and keep
# writing into the tile directory the onroad reader just opened, which is what
# was segfaulting (-12) the onroad process in a tight restart loop. The lock +
# pidfile below make sure that subprocess is always killed (on clean shutdown
# via the signal handlers, and on the next boot if this process itself got
# SIGKILLed) before anything else is allowed to read the tile directory.
_active_proc_lock = threading.Lock()
_active_proc: subprocess.Popen | None = None
_shutdown = threading.Event()
# Display-tile bundles for the offline on-screen map (separate asset from mapd's routing
# data). Downloaded after the mapd fetch in the same worker so a region selection installs
# both, and independently restorable when only the tile bundle is missing.
_tile_downloader: TileBundleDownloader | None = None
@@ -62,7 +46,6 @@ def _pid_is_vendor_fetch(pid: int) -> bool:
def _reap_orphaned_vendor_fetch() -> None:
"""Kill any vendor-fetch mapd subprocess left running from a prior, uncleanly-terminated run."""
pidfile = _vendor_fetch_pidfile()
try:
with open(pidfile) as f:
@@ -122,8 +105,6 @@ def _install_signal_handlers() -> None:
def ensure_vendor_runtime() -> None:
# verify-only: a hash-mismatched binary is quarantined, never replaced from
# the network — the updater restores the checked-in one
try:
VendorMapdInstaller().verify()
except Exception:
@@ -173,10 +154,6 @@ def _compose_region_selector(nations: list[str], states: list[str] | None = None
def _fetch_tile_bundles(region_selector: str, abort_check=None) -> None:
"""Download the offline on-screen map display tiles for the selected regions.
Separate asset from mapd's routing data: the on-screen map's OsmOfflineProvider reads
raster .mbtiles bundles, so a region selection installs both when OfflineOSMaps is on."""
global _tile_downloader
if not params.get_bool("OfflineOSMaps"):
return
@@ -239,8 +216,6 @@ def _drive_vendor_fetch(region_selector: str, requested_regions: dict) -> None:
break
cloudlog.info(f"iq_maps: vendor map download finished for {region_selector}")
if not cancelled and not _shutdown.is_set():
# OSMDownloadLocations stays set until the finally below, so the konn3kt cancel RPC
# (which removes it) aborts the tile phase exactly like it cancels the mapd phase.
_fetch_tile_bundles(region_selector, abort_check=lambda: _shutdown.is_set() or not mem_params.get("OSMDownloadLocations"))
except Exception:
cloudlog.exception("iq_maps: vendor map download failed")
@@ -304,17 +279,12 @@ _last_auto_restore_t = 0.0
def region_data_missing() -> bool:
# a media wipe (reflash/format) can delete the downloaded region while the params
# that configure offline maps survive; mapd then retries the missing files forever
# and nothing re-downloads (stale_region_artifacts only sees leftover files)
if not params.get_bool("OsmLocal"):
return False
if not params.get("OsmDownloadedDate"):
return False
if glob.glob(f"{Paths.mapd_root()}/db") or glob.glob(f"{Paths.mapd_root()}/v*"):
return False
# mapd v2 stores region tiles under offline/<evenLat>/<evenLon>.tar.gz — without this
# check a v2 install looks perpetually wiped and re-downloads every backoff interval
if glob.glob(f"{Paths.mapd_root()}/offline/*/*"):
return False
country = params.get("OsmLocationName", return_default=True)
@@ -322,8 +292,6 @@ def region_data_missing() -> bool:
def configured_states() -> list[str]:
"""Selected US states: OsmStateNames (JSON list, multi-state) wins; the legacy
single OsmStateName remains the fallback for pre-list configs."""
try:
states = params.get("OsmStateNames")
if isinstance(states, bytes):
@@ -370,8 +338,6 @@ def _configured_region_selector() -> str:
def tile_bundles_missing() -> bool:
# covers a media wipe AND the user enabling OfflineOSMaps after the region download
# already ran (the vendor fetch only pulls tile bundles when the toggle is on)
if not params.get_bool("OfflineOSMaps"):
return False
selector = _configured_region_selector()
@@ -381,8 +347,6 @@ def tile_bundles_missing() -> bool:
def maybe_restore_tile_bundles() -> None:
"""Tile-only download: don't re-run the whole mapd vendor fetch when only the display
tiles are missing."""
global _last_tile_restore_t, _tile_only_worker
if not tile_bundles_missing():
return
@@ -410,9 +374,6 @@ def sync_osm_request_flags() -> None:
maybe_restore_tile_bundles()
if params.get_bool("OsmDbUpdatesCheck"):
if _region_sync_worker is not None and _region_sync_worker.is_alive():
# A download is already writing into Paths.mapd_root() - deleting/rewriting
# files under it right now would race the writer (and any onroad mapd
# reader) the same way the orphaned-subprocess bug did. Wait for it to finish.
return
purge_stale_region_artifacts(stale_region_artifacts())
country = params.get("OsmLocationName", return_default=True)
@@ -439,11 +400,6 @@ def run_loop():
pass
except PermissionError:
cloudlog.exception(f"iq_maps: failed to make {Paths.mapd_root()}")
# A prior run that got SIGKILLed (or crashed) may have left its vendor-fetch
# mapd subprocess running and still writing into Paths.mapd_root(); clear it
# before anything (including the onroad mapd, once `started` flips) reads
# from that directory. Signal handlers cover the graceful-shutdown path.
_reap_orphaned_vendor_fetch()
_install_signal_handlers()

37
iqpilot/iq_maps/tile_bundle_downloader.py
View File

@@ -1,20 +1,6 @@
#!/usr/bin/env python3
"""
Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
Downloads per-region raster display-tile bundles (.mbtiles) for the offline on-screen map.
These are a separate asset from mapd's routing/speed-limit data: mapd pulls OSM way tiles
into Paths.mapd_root(), while the on-screen map (OsmOfflineProvider) reads raster .mbtiles
from offline_map_root()/regions/<selector>/tiles/offline.mbtiles. Bundles are built per
state/nation by scripts/iqpilot/build_state_tile_bundles.py and hosted behind a static base
URL that serves:
<base>/index.json {"version": 1, "regions": {<selector>: entry}}
<base>/<entry["path"]> the raster .mbtiles for that region
Entry fields: path, bytes, sha256, bounds ("minLon,minLat,maxLon,maxLat"), minzoom, maxzoom.
Selectors match the mapd region menu naming: us_state.CA, nation.US.
"""
import hashlib
import json
@@ -29,16 +15,12 @@ from iqpilot.common.params import Params
from iqpilot.common.swaglog import cloudlog
from iqpilot.ui.onroad.offline_tiles import offline_map_root
# Proprietary auth + hosted endpoints (gitea raw with an embedded read-only PAT, same
# pattern as the model selector). Optional: without the private bundle the downloader
# still works anonymously against OfflineTilesBaseUrl (e.g. a public R2 bucket).
try:
from iqpilot.iq_maps.tiles_auth import get_base_urls as _private_base_urls, get_requests_auth as _private_auth
except Exception: # ProprietaryModuleMissing or import errors in stripped builds
_private_base_urls = None
_private_auth = None
# R2 bucket iqnav behind the public custom domain (see scripts/iqpilot/tile_factory/r2_sync_watch.py)
DEFAULT_TILE_BUNDLE_BASE_URL = "https://maps.konn3kt.com/iqosmd/v1"
BASE_URL_PARAM = "OfflineTilesBaseUrl"
PROGRESS_PARAM = "OfflineTilesDownloadProgress"
@@ -49,8 +31,6 @@ STREAM_RETRIES = 8
def candidate_base_urls(params: Params) -> list[str]:
"""Hosts to try in order: user/param override first, then the embedded private
endpoints (gitea raw), then the public default."""
override = params.get(BASE_URL_PARAM)
if isinstance(override, bytes):
override = override.decode("utf-8", errors="ignore")
@@ -144,12 +124,6 @@ def _write_manifest(selector: str, entry: dict) -> None:
class TileBundleDownloader:
"""Streams region bundles to disk with resume + sha256 verify + atomic install.
Cancellation matches the mapd flow: the caller sets REQUEST_PARAM in mem params while a
download runs; removing it (konn3kt cancel RPC or settings) aborts between chunks. The
partial .part file is kept so a retry resumes instead of restarting.
"""
def __init__(self, params: Params | None = None, mem_params: Params | None = None,
abort_check=None):
@@ -160,7 +134,6 @@ class TileBundleDownloader:
self.mem_params = Params("/dev/shm/params") if platform.system() != "Darwin" else self.params
self.session = requests.Session()
self._cancelled = threading.Event()
# optional external cancel signal, e.g. the orchestrator's OSMDownloadLocations removal
self._abort_check = abort_check
def cancel(self) -> None:
@@ -170,7 +143,6 @@ class TileBundleDownloader:
if self._cancelled.is_set():
return True
if not self.mem_params.get(REQUEST_PARAM):
# request flag was removed out from under us -> user cancelled
self._cancelled.set()
return True
if self._abort_check is not None and self._abort_check():
@@ -188,7 +160,6 @@ class TileBundleDownloader:
def _download_one(self, selector: str, entry: dict, base_url: str,
progress_offset: int, progress_total: int) -> bool:
"""Download a region: the night bundle, plus the optional day-style variant."""
night_path = region_bundle_path(selector)
ok = self._download_file(
selector, base_url, entry["path"], int(entry.get("bytes", 0)),
@@ -205,9 +176,7 @@ class TileBundleDownloader:
progress_offset + int(entry.get("bytes", 0)), progress_total,
)
if not day_ok:
# the night set is complete and usable; a failed day variant retries next pass
cloudlog.warning(f"iq_maps: day-style bundle failed for {selector}; night set installed")
# manifest last: bounds drive region matching, so it must describe installed files
_write_manifest(selector, entry)
cloudlog.info(f"iq_maps: installed tile bundle {selector}")
return True
@@ -219,8 +188,6 @@ class TileBundleDownloader:
part_path = final_path.with_name(final_path.name + ".part")
part_path.parent.mkdir(parents=True, exist_ok=True)
# A cellular/hotspot link routinely kills a multi-hundred-MB stream mid-flight; retry
# each interruption from the current .part offset instead of failing the whole region.
downloaded = 0
digest = hashlib.sha256()
last_error: Exception | None = None
@@ -244,7 +211,6 @@ class TileBundleDownloader:
auth = request_auth()
response = self.session.get(url, headers=headers, stream=True, timeout=HTTP_TIMEOUT_S, auth=auth)
if resume_from and response.status_code != 206:
# server ignored the Range request -> restart from scratch
digest = hashlib.sha256()
resume_from = 0
part_path.unlink(missing_ok=True)
@@ -284,8 +250,6 @@ class TileBundleDownloader:
return True
def download_regions(self, selectors: list[str]) -> bool:
"""Download the display-tile bundles for the given region selectors. Returns True if all
requested bundles are installed and current when done."""
self._cancelled.clear()
ok = True
try:
@@ -346,7 +310,6 @@ class TileBundleDownloader:
if not expected_sha or installed_sha != expected_sha:
return False
if entry.get("day_path"):
# a published day variant must be installed and current too
day_file = region_bundle_dir(selector) / "tiles" / "offline_day.mbtiles"
installed_day = str(manifest.get("mbtiles_day", {}).get("sha256", "")).strip().lower()
expected_day = str(entry.get("day_sha256", "")).strip().lower()

2
iqpilot/iq_maps/tiles_auth.py
View File

@@ -7,4 +7,4 @@ from iqpilot._proprietary_loader import ProprietaryModuleMissing, load_private_m
try:
load_private_module(__name__, "iqpilot_private.maps.git_auth")
except ProprietaryModuleMissing:
from iqpilot.maps_private_src.git_auth import * # noqa: F403
from iqpilot.maps_private_src.git_auth import *

6
iqpilot/iq_maps/update_vendor_version.py
View File

@@ -1,11 +1,6 @@
#!/usr/bin/env python3
"""
Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
Maintainer utility: pin a new mapd release tag and refresh the checked-in binary
hash. Not used at runtime. Binaries come from the gitlvb teal/mapd CI (built
against teal/gomsgq) — drop the artifact at third_party/mapd_pfeiferj/mapd, then
run this so the hash pin moves in the same commit.
"""
import argparse
import os
@@ -21,7 +16,6 @@ from iqpilot.iq_maps.vendor_mapd_installer import (
_RELEASE_SYMBOL = "VENDOR_RELEASE_TAG"
_INSTALLER_SRC = os.path.join(BASEDIR, "iqpilot", "iq_maps", "vendor_mapd_installer.py")
# public: the checked-in hash the version test compares the installed binary against
HASH_FILE = os.path.join(BASEDIR, "iqpilot", "iq_maps", "tests", "mapd_hash")
_HASH_FILE = HASH_FILE
_TAG_ASSIGN = re.compile(rf'^{_RELEASE_SYMBOL}\s*=\s*["\'][^"\']*["\']', re.MULTILINE)

13
iqpilot/iq_maps/vendor_mapd_installer.py
View File

@@ -1,14 +1,6 @@
#!/usr/bin/env python3
"""
Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
Verifies the vendored `mapd` routing binary authored by Jacob Pfeifer
(github.com/pfeiferj/mapd), built from the gitlvb teal/mapd fork against
teal/gomsgq. The only accepted binary is the checked-in one matching the pinned
hash; nothing is ever downloaded at runtime. Jacob's stock release build embeds
a 15-reader msgq header layout — on this fork (NUM_READERS=32) its registration
writes land inside other processes' reader slots, so a wrong binary is
quarantined rather than left where manager could start it.
"""
import hashlib
import os
@@ -29,7 +21,6 @@ QUARANTINE_PATH = VENDOR_MAPD_PATH + ".quarantined"
def sha256_of_file(path: str) -> str:
"""Hex SHA-256 digest of a file on disk."""
digest = hashlib.sha256()
with open(path, "rb") as handle:
for block in iter(lambda: handle.read(1 << 20), b""):
@@ -50,14 +41,12 @@ class VendorMapdInstaller:
return str(self._params.get(_VERSION_PARAM) or "")
def verify(self) -> bool:
"""True iff the on-disk binary matches the pinned hash; quarantines a wrong one."""
expected = self._expected_hash()
if not expected:
cloudlog.error("iq_maps: pinned mapd hash missing, vendor binary cannot be verified")
return False
if not os.path.isfile(VENDOR_MAPD_PATH):
# the binary is a tracked file: the updater/bundle restores it
self._say("Offline maps engine missing; it will be restored by the next update.")
self._params.remove(_VERSION_PARAM)
return False
@@ -77,8 +66,6 @@ class VendorMapdInstaller:
self._say(f"Offline maps engine verified [{VENDOR_RELEASE_TAG}]")
return True
# a foreign binary — e.g. a stock release download from the retired fetch
# path — must never run: quarantine it where manager can't start it
cloudlog.error(f"iq_maps: vendor mapd hash {current[:12]} != pinned {expected[:12]}, quarantining")
self._say("Offline maps engine failed verification; quarantined until the next update.")
try: