IQ.Pilot Prebuilt Release @ 67fd9c2

artifacts/package_runtime/iqdbc/lvbs/tools/fw_fingerprint/__init__.py

@@ -0,0 +1 @@
+

artifacts/package_runtime/iqdbc/lvbs/tools/fw_fingerprint/debug_fw_fingerprinting_offline.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+import argparse
+
+from iqdbc.car import uds
+from iqpilot.tools.lib.logreader import LogReader, ReadMode, live_logreader
+
+
+def main(route: str | None, addrs: list[int], rxoffset: int | None):
+  """
+  TODO:
+  - highlight TX vs RX clearly
+  - disambiguate sendcan and can (useful to know if something sent on sendcan made it to the bus on can->128)
+  - print as fixed width table, easier to read
+  """
+
+  if route is None:
+    lr = live_logreader()
+  else:
+    lr = LogReader(route, default_mode=ReadMode.RLOG, sort_by_time=True)
+
+  start_mono_time = None
+  prev_mono_time = 0
+
+  # include rx addresses
+  addrs = addrs + [uds.get_rx_addr_for_tx_addr(addr, rxoffset) for addr in addrs]
+
+  for msg in lr:
+    if msg.which() == 'can':
+      if start_mono_time is None:
+        start_mono_time = msg.logMonoTime
+
+    if msg.which() in ("can", 'sendcan'):
+      for can in getattr(msg, msg.which()):
+        if can.address in addrs or not len(addrs):
+          if msg.logMonoTime != prev_mono_time:
+            print()
+            prev_mono_time = msg.logMonoTime
+          print(f"{msg.which():>7}: rxaddr={can.address}, bus={str(can.src) + ',':<4} {round((msg.logMonoTime - start_mono_time) * 1e-6)} ms, " +
+                f"0x{can.dat.hex()}, {can.dat}, {len(can.dat)=}")
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(description='View back and forth ISO-TP communication between various ECUs given an address')
+  parser.add_argument('route', nargs='?', help='Route name, live if not specified')
+  parser.add_argument('--addrs', nargs='*', default=[], help='List of tx address to view (0x7e0 for engine)')
+  parser.add_argument('--rxoffset', default='')
+  args = parser.parse_args()
+
+  addrs = [int(addr, base=16) if addr.startswith('0x') else int(addr) for addr in args.addrs]
+  rxoffset = int(args.rxoffset, base=16) if args.rxoffset else None
+  main(args.route, addrs, rxoffset)

artifacts/package_runtime/iqdbc/lvbs/tools/fw_fingerprint/fingerprint_from_route.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+import sys
+from iqpilot.tools.lib.logreader import LogReader, ReadMode
+
+
+def get_fingerprint(lr):
+  # TODO: make this a nice tool for car ports. should also work with qlogs for FW
+
+  fw = None
+  vin = None
+  msgs = {}
+  for msg in lr:
+    if msg.which() == 'carParams':
+      fw = msg.carParams.carFw
+      vin = msg.carParams.carVin
+    elif msg.which() == 'can':
+      for c in msg.can:
+        # read also msgs sent by EON on CAN bus 0x80 and filter out the
+        # addr with more than 11 bits
+        if c.src % 0x80 == 0 and c.address < 0x800 and c.address not in (0x7df, 0x7e0, 0x7e8):
+          msgs[c.address] = len(c.dat)
+
+  # show CAN fingerprint
+  fingerprint = ', '.join(f"{v[0]}: {v[1]}" for v in sorted(msgs.items()))
+  print(f"\nfound {len(msgs)} messages. CAN fingerprint:\n")
+  print(fingerprint)
+
+  # TODO: also print the fw fingerprint merged with the existing ones
+  # show FW fingerprint
+  if fw:
+    print("\nFW fingerprint:\n")
+    for f in fw:
+      print(f"    (Ecu.{f.ecu}, {hex(f.address)}, {None if f.subAddress == 0 else f.subAddress}): [")
+      print(f"      {f.fwVersion},")
+      print("    ],")
+  print()
+  print(f"VIN: {vin}")
+
+
+if __name__ == "__main__":
+  if len(sys.argv) < 2:
+    print("Usage: ./fingerprint_from_route.py <route>")
+    sys.exit(1)
+
+  lr = LogReader(sys.argv[1], ReadMode.QLOG)
+  get_fingerprint(lr)

artifacts/package_runtime/iqdbc/lvbs/tools/fw_fingerprint/fuzz_fw_fingerprint.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+import random
+from collections import defaultdict
+
+from tqdm import tqdm
+
+from iqdbc.car.fw_versions import match_fw_to_car_fuzzy
+from iqdbc.car.toyota.values import FW_VERSIONS as TOYOTA_FW_VERSIONS
+from iqdbc.car.honda.values import FW_VERSIONS as HONDA_FW_VERSIONS
+from iqdbc.car.hyundai.values import FW_VERSIONS as HYUNDAI_FW_VERSIONS
+from iqdbc.car.volkswagen.values import FW_VERSIONS as VW_FW_VERSIONS
+
+
+FWS = {}
+FWS.update(TOYOTA_FW_VERSIONS)
+FWS.update(HONDA_FW_VERSIONS)
+FWS.update(HYUNDAI_FW_VERSIONS)
+FWS.update(VW_FW_VERSIONS)
+
+if __name__ == "__main__":
+  total = 0
+  match = 0
+  wrong_match = 0
+  confusions = defaultdict(set)
+
+  for _ in tqdm(range(1000)):
+    for candidate, fws in FWS.items():
+      fw_dict = {}
+      for (_, addr, subaddr), fw_list in fws.items():
+        fw_dict[(addr, subaddr)] = [random.choice(fw_list)]
+
+      matches = match_fw_to_car_fuzzy(fw_dict, log=False, exclude=candidate)
+
+      total += 1
+      if len(matches) == 1:
+        if list(matches)[0] == candidate:
+          match += 1
+        else:
+          confusions[candidate] |= matches
+          wrong_match += 1
+
+  print()
+  for candidate, wrong_matches in sorted(confusions.items()):
+    print(candidate, wrong_matches)
+
+  print()
+  print(f"Total fuzz cases: {total}")
+  print(f"Correct matches:  {match}")
+  print(f"Wrong matches:    {wrong_match}")

artifacts/package_runtime/iqdbc/lvbs/tools/fw_fingerprint/fw_query_on_routes.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+
+from collections import defaultdict
+import argparse
+import os
+import traceback
+from tqdm import tqdm
+from iqdbc.car.car_helpers import interface_names
+from iqdbc.car.fingerprints import MIGRATION
+from iqdbc.car.fw_versions import VERSIONS, match_fw_to_car
+from iqpilot.tools.lib.logreader import LogReader, ReadMode
+from iqpilot.tools.lib.route import SegmentRange
+
+
+NO_API = "NO_API" in os.environ
+SUPPORTED_BRANDS = VERSIONS.keys()
+SUPPORTED_CARS = [brand for brand in SUPPORTED_BRANDS for brand in interface_names[brand]]
+UNKNOWN_BRAND = "unknown"
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(description='Run FW fingerprint on Qlog of route or list of routes')
+  parser.add_argument('route', help='Route or file with list of routes')
+  parser.add_argument('--car', help='Force comparison fingerprint to known car')
+  args = parser.parse_args()
+
+  if os.path.exists(args.route):
+    routes = list(open(args.route))
+  else:
+    routes = [args.route]
+
+  mismatches = defaultdict(list)
+
+  not_fingerprinted = 0
+  solved_by_fuzzy = 0
+
+  good_exact = 0
+  wrong_fuzzy = 0
+  good_fuzzy = 0
+
+  dongles = []
+  for route in tqdm(routes):
+    sr = SegmentRange(route)
+    dongle_id = sr.dongle_id
+
+    if dongle_id in dongles:
+      continue
+
+    if sr.slice == '' and sr.selector is None:
+      route += '/0'
+
+    lr = LogReader(route, default_mode=ReadMode.QLOG)
+
+    try:
+      dongles.append(dongle_id)
+
+      CP = None
+      for msg in lr:
+        if msg.which() == "pandaStates":
+          if msg.pandaStates[0].pandaType in ('unknown', 'whitePanda', 'greyPanda', 'pedal'):
+            print("wrong panda type")
+            break
+
+        elif msg.which() == "carParams":
+          CP = msg.carParams
+          car_fw = [fw for fw in CP.carFw if not fw.logging]
+          if len(car_fw) == 0:
+            print("WARNING: no fw")
+
+          live_fingerprint = CP.carFingerprint
+          live_fingerprint = MIGRATION.get(live_fingerprint, live_fingerprint)
+
+          if args.car is not None:
+            live_fingerprint = args.car
+
+          if live_fingerprint not in SUPPORTED_CARS:
+            print("not in supported cars")
+            break
+
+          _, exact_matches = match_fw_to_car(car_fw, CP.carVin, allow_exact=True, allow_fuzzy=False)
+          _, fuzzy_matches = match_fw_to_car(car_fw, CP.carVin, allow_exact=False, allow_fuzzy=True)
+
+          if (len(exact_matches) == 1) and (list(exact_matches)[0] == live_fingerprint):
+            good_exact += 1
+            print(f"Correct! Live: {live_fingerprint} - Fuzzy: {fuzzy_matches}")
+
+            # Check if fuzzy match was correct
+            if len(fuzzy_matches) == 1:
+              if list(fuzzy_matches)[0] != live_fingerprint:
+                wrong_fuzzy += 1
+                print("Fuzzy match wrong! Fuzzy:", fuzzy_matches, "Live:", live_fingerprint)
+              else:
+                good_fuzzy += 1
+            break
+
+          print("Old style:", live_fingerprint, "Vin", CP.carVin)
+          print("New style (exact):", exact_matches)
+          print("New style (fuzzy):", fuzzy_matches)
+
+          padding = max([len(fw.brand or UNKNOWN_BRAND) for fw in car_fw] + [0])
+          for version in sorted(car_fw, key=lambda fw: fw.brand):
+            subaddr = None if version.subAddress == 0 else hex(version.subAddress)
+            print(f"  Brand: {version.brand or UNKNOWN_BRAND:{padding}}, bus: {version.bus} - " +
+                  f"(Ecu.{version.ecu}, {hex(version.address)}, {subaddr}): [{version.fwVersion}],")
+
+          print("Mismatches")
+          found = False
+          for brand in SUPPORTED_BRANDS:
+            car_fws = VERSIONS[brand]
+            if live_fingerprint in car_fws:
+              found = True
+              expected = car_fws[live_fingerprint]
+              for (_, expected_addr, expected_sub_addr), v in expected.items():
+                for version in car_fw:
+                  if version.brand != brand and len(version.brand):
+                    continue
+                  sub_addr = None if version.subAddress == 0 else version.subAddress
+                  addr = version.address
+
+                  if (addr, sub_addr) == (expected_addr, expected_sub_addr):
+                    if version.fwVersion not in v:
+                      print(f"({hex(addr)}, {'None' if sub_addr is None else hex(sub_addr)}) - {version.fwVersion}")
+
+                      # Add to global list of mismatches
+                      mismatch = (addr, sub_addr, version.fwVersion)
+                      if mismatch not in mismatches[live_fingerprint]:
+                        mismatches[live_fingerprint].append(mismatch)
+
+          # No FW versions for this car yet, add them all to mismatch list
+          if not found:
+            for version in car_fw:
+              sub_addr = None if version.subAddress == 0 else version.subAddress
+              addr = version.address
+              mismatch = (addr, sub_addr, version.fwVersion)
+              if mismatch not in mismatches[live_fingerprint]:
+                mismatches[live_fingerprint].append(mismatch)
+
+          print()
+          not_fingerprinted += 1
+
+          if len(fuzzy_matches) == 1:
+            if list(fuzzy_matches)[0] == live_fingerprint:
+              solved_by_fuzzy += 1
+            else:
+              wrong_fuzzy += 1
+              print("Fuzzy match wrong! Fuzzy:", fuzzy_matches, "Live:", live_fingerprint)
+
+          break
+
+      if CP is None:
+        print("no CarParams in logs")
+    except Exception:
+      traceback.print_exc()
+    except KeyboardInterrupt:
+      break
+
+  print()
+  # Print FW versions that need to be added separated out by car and address
+  for car, m in sorted(mismatches.items()):
+    print(car)
+    addrs = defaultdict(list)
+    for (addr, sub_addr, version) in m:
+      addrs[(addr, sub_addr)].append(version)
+
+    for (addr, sub_addr), versions in addrs.items():
+      print(f"  ({hex(addr)}, {'None' if sub_addr is None else hex(sub_addr)}): [")
+      for v in versions:
+        print(f"    {v},")
+      print("  ]")
+    print()
+
+  print()
+  print(f"Number of dongle ids checked: {len(dongles)}")
+  print(f"Fingerprinted:                {good_exact}")
+  print(f"Not fingerprinted:            {not_fingerprinted}")
+  print(f"  of which had a fuzzy match: {solved_by_fuzzy}")
+
+  print()
+  print(f"Correct fuzzy matches:        {good_fuzzy}")
+  print(f"Wrong fuzzy matches:          {wrong_fuzzy}")
+  print()

artifacts/package_runtime/iqdbc/lvbs/tools/fw_fingerprint/fw_versions.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+import time
+import argparse
+import iqpilot.cereal.messaging as messaging
+from iqpilot.cereal import car
+from iqdbc.car.carlog import carlog
+from iqdbc.car.fw_versions import get_fw_versions, match_fw_to_car
+from iqdbc.car.vin import get_vin
+from iqpilot.common.params import Params
+from iqpilot.selfdrive.car.card import can_comm_callbacks, obd_callback
+from typing import Any
+
+Ecu = car.CarParams.Ecu
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(description='Get firmware version of ECUs')
+  parser.add_argument('--scan', action='store_true')
+  parser.add_argument('--debug', action='store_true')
+  parser.add_argument('--brand', help='Only query addresses/with requests for this brand')
+  args = parser.parse_args()
+
+  if args.debug:
+    carlog.setLevel('DEBUG')
+
+  logcan = messaging.sub_sock('can')
+  pandaStates_sock = messaging.sub_sock('pandaStates')
+  sendcan = messaging.pub_sock('sendcan')
+  can_callbacks = can_comm_callbacks(logcan, sendcan)
+
+  # Set up params for pandad
+  params = Params()
+  params.remove("FirmwareQueryDone")
+  params.put_bool("IsOnroad", False)
+  time.sleep(0.2)  # thread is 10 Hz
+  params.put_bool("IsOnroad", True)
+  set_obd_multiplexing = obd_callback(params)
+
+  extra: Any = None
+  if args.scan:
+    extra = {}
+    # Honda
+    for i in range(256):
+      extra[(Ecu.unknown, 0x18da00f1 + (i << 8), None)] = []
+      extra[(Ecu.unknown, 0x700 + i, None)] = []
+      extra[(Ecu.unknown, 0x750, i)] = []
+    extra = {"any": {"debug": extra}}
+
+  num_pandas = len(messaging.recv_one_retry(pandaStates_sock).pandaStates)
+
+  t = time.monotonic()
+  print("Getting vin...")
+  set_obd_multiplexing(True)
+  vin_rx_addr, vin_rx_bus, vin = get_vin(*can_callbacks, (0, 1))
+  print(f'RX: {hex(vin_rx_addr)}, BUS: {vin_rx_bus}, VIN: {vin}')
+  print(f"Getting VIN took {time.monotonic() - t:.3f} s")
+  print()
+
+  t = time.monotonic()
+  fw_vers = get_fw_versions(*can_callbacks, set_obd_multiplexing, query_brand=args.brand, extra=extra, num_pandas=num_pandas, progress=True)
+  _, candidates = match_fw_to_car(fw_vers, vin)
+
+  print()
+  print("Found FW versions")
+  print("{")
+  padding = max([len(fw.brand) for fw in fw_vers] or [0])
+  for version in fw_vers:
+    subaddr = None if version.subAddress == 0 else hex(version.subAddress)
+    print(f"  Brand: {version.brand:{padding}}, bus: {version.bus}, OBD: {version.obdMultiplexing} - " +
+          f"(Ecu.{version.ecu}, {hex(version.address)}, {subaddr}): [{version.fwVersion!r}]")
+  print("}")
+
+  print()
+  print("Possible matches:", candidates)
+  print(f"Getting fw took {time.monotonic() - t:.3f} s")

artifacts/package_runtime/iqdbc/lvbs/tools/fw_fingerprint/get_fingerprint.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+
+# simple script to get a vehicle fingerprint.
+
+# Instructions:
+# - connect to a Panda
+# - run selfdrive/pandad/pandad
+# - launching this script
+#   Note: it's very important that the car is in stock mode, in order to collect a complete fingerprint
+# - since some messages are published at low frequency, keep this script running for at least 30s,
+#   until all messages are received at least once
+
+import iqpilot.cereal.messaging as messaging
+
+logcan = messaging.sub_sock('can')
+msgs = {}
+while True:
+  lc = messaging.recv_sock(logcan, True)
+  if lc is None:
+    continue
+
+  for c in lc.can:
+    # read also msgs sent by EON on CAN bus 0x80 and filter out the
+    # addr with more than 11 bits
+    if c.src % 0x80 == 0 and c.address < 0x800 and c.address not in (0x7df, 0x7e0, 0x7e8):
+      msgs[c.address] = len(c.dat)
+
+  fingerprint = ', '.join(f"{v[0]}: {v[1]}" for v in sorted(msgs.items()))
+
+  print(f"number of messages {len(msgs)}:")
+  print(f"fingerprint {fingerprint}")