IQ.Lvbs/IQ.Pilot
8
7
Fork 2
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

IQ.Pilot Release Commit @ bec7652

Browse Source
This commit is contained in:
IQ.Lvbs history cleanup
2026-08-22 23:42:41 -05:00
commit 58039e647c
4603 changed files with 1236178 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
iqpilot/cereal/README.md Normal file
View File

@@ -0,0 +1,95 @@
# What is cereal?
cereal is the messaging system for openpilot. It uses [msgq](https://github.com/commaai/msgq) as a pub/sub backend, and [Cap'n proto](https://capnproto.org/capnp-tool.html) for serialization of the structs.
## Messaging Spec
You'll find the message types in [log.capnp](log.capnp). It uses [Cap'n proto](https://capnproto.org/capnp-tool.html) and defines one struct called `Event`.
All `Events` have a `logMonoTime` and a `valid`. Then a big union defines the packet type.
### Best Practices
- **All fields must describe quantities in SI units**, unless otherwise specified in the field name.
- In the context of the message they are in, field names should be completely unambiguous.
- All values should be easy to plot and be human-readable with minimal parsing.
### Maintaining backwards-compatibility
When making changes to the messaging spec you want to maintain backwards-compatibility, such that old logs can
be parsed with a new version of cereal. Adding structs and adding members to structs is generally safe, most other
things are not. Read more details [here](https://capnproto.org/language.html).
### Custom forks
Forks of [openpilot](https://github.com/commaai/openpilot) might want to add things to the messaging
spec, however this could conflict with future changes made in mainline cereal/openpilot. Rebasing against mainline openpilot
then means breaking backwards-compatibility with all old logs of your fork. So we added reserved events in
[custom.capnp](custom.capnp) that we will leave empty in mainline cereal/openpilot. **If you only modify those, you can ensure your
fork will remain backwards-compatible with all versions of mainline openpilot and your fork.**
An example of compatible changes:
```diff
diff --git a/cereal/custom.capnp b/cereal/custom.capnp
index 3348e859e..3365c7b98 100644
--- a/cereal/custom.capnp
+++ b/cereal/custom.capnp
@@ -10,7 +10,11 @@ $Cxx.namespace("cereal");
# DO rename the structs
# DON'T change the identifier (e.g. @0x81c2f05a394cf4af)
-struct CustomReserved0 @0x81c2f05a394cf4af {
+struct SteeringInfo @0x81c2f05a394cf4af {
+ active @0 :Bool;
+ steeringAngleDeg @1 :Float32;
+ steeringRateDeg @2 :Float32;
+ steeringAccelDeg @3 :Float32;
}
struct CustomReserved1 @0xaedffd8f31e7b55d {
diff --git a/cereal/log.capnp b/cereal/log.capnp
index 1209f3fd9..b189f58b6 100644
--- a/cereal/log.capnp
+++ b/cereal/log.capnp
@@ -2558,14 +2558,14 @@ struct Event {
# DO change the name of the field
# DON'T change anything after the "@"
- customReservedRawData0 @124 :Data;
+ rawCanData @124 :Data;
customReservedRawData1 @125 :Data;
customReservedRawData2 @126 :Data;
# DO change the name of the field and struct
# DON'T change the ID (e.g. @107)
# DON'T change which struct it points to
- customReserved0 @107 :Custom.CustomReserved0;
+ steeringInfo @107 :Custom.SteeringInfo;
customReserved1 @108 :Custom.CustomReserved1;
customReserved2 @109 :Custom.CustomReserved2;
customReserved3 @110 :Custom.CustomReserved3;
```
---
Example
---
```python
import iqpilot.cereal.messaging as messaging
# in subscriber
sm = messaging.SubMaster(['sensorEvents'])
while 1:
sm.update()
print(sm['sensorEvents'])
```
```python
# in publisher
pm = messaging.PubMaster(['sensorEvents'])
dat = messaging.new_message('sensorEvents', size=1)
dat.sensorEvents[0] = {"gyro": {"v": [0.1, -0.1, 0.1]}}
pm.send('sensorEvents', dat)
```

27
iqpilot/cereal/SConscript Normal file
View File

@@ -0,0 +1,27 @@
import os
import iqdbc
Import('env', 'common', 'msgq')
cereal_dir = Dir('.')
gen_dir = Dir('gen')
# Build cereal
schema_files = ['log.capnp', 'legacy.capnp', 'custom.capnp']
car_schema_dir = os.path.join(os.path.dirname(iqdbc.__file__), 'car')
car_capnp = os.path.join(car_schema_dir, 'car.capnp')
all_output = schema_files + ['car.capnp']
env.Command([f'gen/cpp/{s}.c++' for s in all_output] + [f'gen/cpp/{s}.h' for s in all_output],
schema_files + [car_capnp],
env.PrettyAction(f"capnpc --src-prefix={cereal_dir.path} --src-prefix={car_schema_dir} --import-path={car_schema_dir} $SOURCES -o c++:{gen_dir.path}/cpp/", 'CAPNP'))
cereal = env.Library('cereal', [f'gen/cpp/{s}.c++' for s in all_output])
# Build messaging
services_h = env.Command(['services.h'], ['services.py'], env.PrettyAction('python3 ' + cereal_dir.path + '/services.py > $TARGET', 'GEN'))
env.Program('messaging/bridge', ['messaging/bridge.cc', 'messaging/msgq_to_zmq.cc'], LIBS=[msgq, common, 'pthread'])
socketmaster = env.Library('socketmaster', ['messaging/socketmaster.cc'])
Export('cereal', 'socketmaster')

12
iqpilot/cereal/__init__.py Normal file
View File

@@ -0,0 +1,12 @@
import os
import capnp
from importlib.resources import as_file, files
capnp.remove_import_hook()
with as_file(files("iqpilot.cereal")) as fspath, as_file(files("iqdbc")) as iqdbc_path:
CEREAL_PATH = fspath.as_posix()
iqdbc_import_path = os.path.join(os.path.realpath(iqdbc_path.as_posix()), "car")
car = capnp.load(os.path.join(iqdbc_import_path, "car.capnp"), imports=[iqdbc_import_path])
log = capnp.load(os.path.join(CEREAL_PATH, "log.capnp"), imports=[iqdbc_import_path])
custom = capnp.load(os.path.join(CEREAL_PATH, "custom.capnp"), imports=[iqdbc_import_path])

976
iqpilot/cereal/custom.capnp Normal file
View File

@@ -0,0 +1,976 @@
using Cxx = import "/include/c++.capnp";
$Cxx.namespace("cereal");
@0xb526ba661d550a59;
# Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
# custom.capnp: a home for reserved structs used by IQ-specific extensions.
struct AlwaysOnLateral {
state @0 :AlwaysOnLateralState;
enabled @1 :Bool;
active @2 :Bool;
available @3 :Bool;
enum AlwaysOnLateralState {
disabled @0;
paused @1;
enabled @2;
softDisabling @3;
overriding @4;
}
}
# Same struct as Log.RadarState.LeadData
struct LeadData {
dRel @0 :Float32;
yRel @1 :Float32;
vRel @2 :Float32;
aRel @3 :Float32;
vLead @4 :Float32;
dPath @6 :Float32;
vLat @7 :Float32;
vLeadK @8 :Float32;
aLeadK @9 :Float32;
fcw @10 :Bool;
status @11 :Bool;
aLeadTau @12 :Float32;
modelProb @13 :Float32;
radar @14 :Bool;
radarTrackId @15 :Int32 = -1;
aLeadDEPRECATED @5 :Float32;
}
struct IQState @0xfb0932cf1bde8c5a {
aol @0 :AlwaysOnLateral;
enum AudibleAlert {
none @0;
engage @1;
disengage @2;
refuse @3;
warningSoft @4;
warningImmediate @5;
prompt @6;
promptRepeat @7;
promptDistracted @8;
# unused, these are reserved for upstream events so we don't collide
reserved9 @9;
reserved10 @10;
reserved11 @11;
reserved12 @12;
reserved13 @13;
reserved14 @14;
reserved15 @15;
reserved16 @16;
reserved17 @17;
reserved18 @18;
reserved19 @19;
reserved20 @20;
reserved21 @21;
reserved22 @22;
reserved23 @23;
reserved24 @24;
reserved25 @25;
reserved26 @26;
reserved27 @27;
reserved28 @28;
reserved29 @29;
reserved30 @30;
promptSingleLow @31;
promptSingleHigh @32;
}
}
struct IQModelManager @0xe91d6987759290bb {
activeBundle @0 :ModelBundle;
selectedBundle @1 :ModelBundle;
availableBundles @2 :List(ModelBundle);
struct DownloadUri {
uri @0 :Text;
sha256 @1 :Text;
}
enum DownloadStatus {
notDownloading @0;
downloading @1;
downloaded @2;
cached @3;
failed @4;
}
struct DownloadProgress {
status @0 :DownloadStatus;
progress @1 :Float32;
eta @2 :UInt32;
}
struct Artifact {
fileName @0 :Text;
downloadUri @1 :DownloadUri;
downloadProgress @2 :DownloadProgress;
}
struct Model {
type @0 :Type;
artifact @1 :Artifact; # Main artifact
metadata @2 :Artifact; # Metadata artifact
enum Type {
supercombo @0;
navigation @1;
vision @2;
policy @3;
offPolicy @4;
onPolicy @5;
}
}
enum Runner {
snpe @0;
tinygrad @1;
stock @2;
}
struct Override {
key @0 :Text;
value @1 :Text;
}
struct ModelBundle {
index @0 :UInt32;
internalName @1 :Text;
displayName @2 :Text;
models @3 :List(Model);
status @4 :DownloadStatus;
generation @5 :UInt32;
environment @6 :Text;
runner @7 :Runner;
is20hz @8 :Bool;
ref @9 :Text;
minimumSelectorVersion @10 :UInt32;
overrides @11 :List(Override);
}
}
struct IQPlan @0xda401323ae805f2b {
iqDynamic @0 :IQDynamicControl;
longitudinalPlanSource @1 :LongitudinalPlanSource;
iqNavState @2 :IQNavPlanState;
speedLimit @3 :SpeedLimit;
vTarget @4 :Float32;
aTarget @5 :Float32;
events @6 :List(IQOnroadEvent.Event);
e2eAlerts @7 :E2eAlerts;
struct IQDynamicControl {
state @0 :IQDynamicControlState;
enabled @1 :Bool;
active @2 :Bool;
enum IQDynamicControlState {
acc @0;
blended @1;
}
}
struct IQNavPlanState {
nav @0 :Nav;
struct Nav {
engaged @0 :Bool;
provider @1 :IQNavState.LongitudinalProvider;
state @2 :IQNavState.LongitudinalState;
speedTarget @3 :Float32;
accelTarget @4 :Float32;
valid @5 :Bool;
}
}
struct SpeedLimit {
resolver @0 :Resolver;
assist @1 :Assist;
struct Resolver {
speedLimit @0 :Float32;
distToSpeedLimit @1 :Float32;
source @2 :Source;
speedLimitOffset @3 :Float32;
speedLimitLast @4 :Float32;
speedLimitFinal @5 :Float32;
speedLimitFinalLast @6 :Float32;
speedLimitValid @7 :Bool;
speedLimitLastValid @8 :Bool;
}
struct Assist {
state @0 :AssistState;
enabled @1 :Bool;
active @2 :Bool;
vTarget @3 :Float32;
aTarget @4 :Float32;
}
enum Source {
none @0;
car @1;
map @2;
}
enum AssistState {
disabled @0;
inactive @1; # No speed limit set or not enabled by parameter.
preActive @2;
pending @3; # Awaiting new speed limit.
adapting @4; # Reducing speed to match new speed limit.
active @5; # Cruising at speed limit.
}
}
enum LongitudinalPlanSource {
cruise @0;
nav @1;
speedLimitAssist @2;
}
struct E2eAlerts {
pathOpen @0 :Bool;
leadPullaway @1 :Bool;
}
}
struct IQOnroadEvent @0xf4621d3ee9233bc9 {
events @0 :List(Event);
struct Event {
name @0 :EventName;
# event types
enable @1 :Bool;
noEntry @2 :Bool;
warning @3 :Bool; # alerts presented only when enabled or soft disabling
userDisable @4 :Bool;
softDisable @5 :Bool;
immediateDisable @6 :Bool;
preEnable @7 :Bool;
permanent @8 :Bool; # alerts presented regardless of openpilot state
overrideLateral @10 :Bool;
overrideLongitudinal @9 :Bool;
}
# Grouped by IQ.Pilot subsystem. Ordinals are IQ-native and are not stable
# across schema revisions; all consumers reference members by name.
enum EventName {
# lateral / LKAS engagement core
alcEngaged @0;
alcDisengaged @1;
alcEngagedSilent @2;
alcDisengagedSilent @3;
steerManually @4;
speedManually @5;
latMismatch @6;
steeringOverrideReengageAlc @7;
# silent pause conditions (gear / door / belt / brake)
gearNotDriveSilent @8;
reverseSilent @9;
doorAjarSilent @10;
seatbeltUnbuckledSilent @11;
parkBrakeSilent @12;
brakeHoldSilent @13;
# alert-only notices
carModeMismatchNotice @14;
pedalHeldNotice @15;
# lane-turn desires
modelTurnLeft @16;
modelTurnRight @17;
# navigation maneuvers
navTurnLeft @18;
navTurnRight @19;
navExitLeft @20;
navExitRight @21;
# speed limit / speed camera
speedLimitPreActive @22;
speedLimitActive @23;
speedLimitChanged @24;
speedLimitPending @25;
speedCameraAhead @26;
# miscellaneous
hyundaiRadarTracksConfirmed @27;
experimentalToggled @28;
e2eChime @29;
# construction zone assist
constructionZoneDetected @30;
# model management
modelUpdating @31;
# camera hardware
wideCamFaulty @32;
# lane-change safety
lateralEdgeBlocked @33;
}
}
struct IQCarParams @0xd4189b5c8aca9f78 {
# Ordinals are IQ-native; all consumers access by name. Live copies self-heal
# via CLEAR_ON_MANAGER_START on the "IQCarParams" param; the persistent cache
# is versioned separately (see IQCarParamsPersistentV2).
iqSafetyFlags @0 :Int16; # iqpilot custom safety flags (read in C++ panda_safety)
flags @1 :UInt32; # car-specific iqpilot quirks
pcmCruiseSpeed @2 :Bool;
enableGasInterceptor @3 :Bool;
iqLateralNet @4 :LateralNet;
longitudinalStoppingSpeedOverride @5 :Float32; # m/s; zero keeps the upstream default
stoppingDecelRateOverride @6 :Float32; # m/s^3; zero keeps the upstream default
longActiveWithGasOverride @7 :Bool; # keep long control active while the driver is on the gas
struct LateralNet {
fuzzyFingerprint @0 :Bool;
model @1 :Model;
struct Model {
name @0 :Text;
path @1 :Text;
}
}
}
struct IQCarControl @0xdc6c97009c7ba28f {
aol @0 :AlwaysOnLateral;
params @1 :List(Param);
leadOne @2 :LeadData;
leadTwo @3 :LeadData;
angleOffsetDeg @4 :Float32;
radarBlendActive @5 :Bool; # feature enabled + PQ + alpha long active
radarEngageReq @6 :Bool; # want stock radar cruise engaged (RadarHandler sends SET on bus 2)
radarCancelReq @7 :Bool; # cancel stock radar cruise now (1kph stop / brake / teardown)
useRadarAccel @8 :Bool; # chill mode + radar active -> pass radar ACS_Sollbeschl as ACC_System payload
radarSetSpeedKph @9 :Float32; # OP set speed (km/h) to sync radar ACA_V_Wunsch toward via GRA_Up/Down
radarGapBars @10 :UInt8; # OP follow-distance bars to mirror to radar GRA_Zeitluecke
struct Param {
key @0 :Text;
type @2 :ParamType;
value @3 :Data;
valueDEPRECATED @1 :Text; # The data type change may cause issues with backwards compatibility.
}
enum ParamType {
string @0;
bool @1;
int @2;
float @3;
time @4;
json @5;
bytes @6;
}
}
# IQ.Pilot device backup/restore state. Ordinals are IQ-native and were
# renumbered/reordered from earlier revisions; persisted BackupInfo blobs are
# not backward compatible across this change and reset on first run. Every
# consumer accesses fields by name.
struct IQBackupManager @0x9f371a75483cf0a3 {
saveProgress @0 :Float32;
loadProgress @1 :Float32;
savePhase @2 :Phase;
loadPhase @3 :Phase;
activeSnapshot @4 :Snapshot;
snapshotLog @5 :List(Snapshot);
faultText @6 :Text;
enum Phase {
idle @0;
completed @1;
inProgress @2;
failed @3;
}
# nested struct names diverge from any upstream schema; field names below are the
# cloud backup JSON contract (to_dict keys) and MUST stay stable for restore.
struct BuildStamp {
build @0 :UInt16;
major @1 :UInt16;
minor @2 :UInt16;
patch @3 :UInt16;
branch @4 :Text;
}
struct MetaField {
value @0 :Text;
key @1 :Text;
tags @2 :List(Text);
}
struct Snapshot {
version @0 :UInt32;
isEncrypted @1 :Bool;
deviceId @2 :Text;
config @3 :Text;
createdAt @4 :Text; # ISO timestamp
updatedAt @5 :Text; # ISO timestamp
iqpilotVersion @6 :BuildStamp;
backupMetadata @7 :List(MetaField);
}
}
struct IQCarState @0xb1c39318bb6bc2b3 {
speedLimit @0 :Float32;
accelPressed @1 :Bool;
decelPressed @2 :Bool;
alcOverrideAlert @3 :Bool;
# VW PQ stock ACC radar feedback for the IQ.Dynamics radar_manager (Blend feature)
accRadarStaAdr @4 :UInt8; # ACC_System.ACS_Sta_ADR (0 not-active, 1 active, 2 passive, 3 irrev_Fehler)
accRadarFehler @5 :Bool; # ACC_System.ACS_Fehler (stored fault -> radar dead for the drive)
}
struct IQLiveData @0xf2e2b608e51f4b0e {
speedLimitValid @0 :Bool;
speedLimit @1 :Float32;
speedLimitAheadValid @2 :Bool;
speedLimitAhead @3 :Float32;
speedLimitAheadDistance @4 :Float32;
roadName @5 :Text;
}
struct IQLiveLocation @0xc04dbadb81776876 {
ecefPosition @0 :VectorSample;
geodeticPosition @1 :VectorSample;
ecefVelocity @2 :VectorSample;
nedVelocity @3 :VectorSample;
bodyVelocity @4 :VectorSample;
bodyAcceleration @5 :VectorSample;
ecefOrientation @6 :VectorSample;
alignedOrientationEcef @7 :VectorSample;
nedOrientation @8 :VectorSample;
bodyAngularRate @9 :VectorSample;
alignedOrientationNed @10 :VectorSample;
alignedVelocity @11 :VectorSample;
alignedAcceleration @12 :VectorSample;
alignedAngularRate @13 :VectorSample;
solutionState @14 :SolutionState;
unixTimestampMillis @15 :Int64;
inputsHealthy @16 :Bool = true;
visionHealthy @17 :Bool = true;
gpsHealthy @18 :Bool = true;
sensorsHealthy @19 :Bool = true;
deviceStable @20 :Bool = true;
secondsSinceReset @21 :Float64;
excessiveResets @22 :Bool;
timeToFirstFix @23 :Float32;
debugState @24 :VectorSample;
gpsWeek @25 :Int32;
gpsTimeOfWeek @26 :Float64;
enum SolutionState {
booting @0;
coarse @1;
ready @2;
}
struct VectorSample {
values @0 :List(Float64);
deviations @1 :List(Float64);
isValid @2 :Bool;
}
}
enum IQTurnSignalDirection {
none @0;
turnLeft @1;
turnRight @2;
}
enum IQLateralEdgeBlock {
none @0;
left @1;
right @2;
}
struct IQDriveModelData @0xcdf0f7f14f46cb86 {
turnSignalDirection @0 :IQTurnSignalDirection;
lateralEdgeBlock @1 :IQLateralEdgeBlock;
}
enum NavDirection {
none @0;
left @1;
right @2;
}
struct IQNavState @0xaae9afb364368cd9 {
# Navigation state and guidance information
active @0 :Bool; # Whether navigation is currently active
destinationValid @1 :Bool; # Whether we have a valid destination
# Current position and route info
distanceRemaining @2 :Float32; # Total distance remaining to destination (m)
timeRemaining @3 :Float32; # Estimated time remaining to destination (s)
currentSegmentIndex @4 :UInt32; # Index of current route segment
totalSegments @5 :UInt32; # Total number of segments in route
# Next maneuver information
nextManeuverValid @6 :Bool; # Whether next maneuver data is valid
nextManeuverDistance @7 :Float32; # Distance to next maneuver (m)
nextManeuverType @8 :ManeuverType; # Type of next maneuver
nextManeuverDirection @9 :IQTurnSignalDirection; # Direction for next maneuver
nextManeuverDescription @10 :Text; # Human-readable maneuver description
nextManeuverAngle @21 :Float32; # Turn angle in degrees (for angle-adaptive enforcement)
# Turn desire control for lateral planning
shouldSendTurnDesire @11 :Bool; # Whether to send turn desires to model
turnDesireDirection @12 :IQTurnSignalDirection; # Direction for turn desire
# Lane change desire control for highway exits/ramps (>45 mph)
shouldSendLaneChangeDesire @22 :Bool; # Whether to send lane change desires for high-speed exits
laneChangeDesireDirection @23 :IQTurnSignalDirection; # Direction for lane change desire
# Speed guidance for longitudinal planning
targetSpeed @13 :Float32; # Target speed for upcoming maneuver (m/s)
targetSpeedValid @14 :Bool; # Whether target speed is valid
# Destination info
destinationLatitude @15 :Float64;
destinationLongitude @16 :Float64;
destinationName @17 :Text;
# Lane positioning guidance for exits/turns
shouldSendLanePositioning @18 :Bool; # Whether to send lane positioning desires (keepLeft/keepRight)
lanePositioningDirection @19 :IQTurnSignalDirection; # Direction for lane positioning
# Lane tracking debug info (model vs GPS comparison for testing)
laneDebugInfo @20 :LaneDebugInfo;
# Navigation-specific UI event fields (separate from model/desire system)
navTurnDesireDirection @24 :NavDirection; # For "Navigation: Turning Left/Right" UI alerts
navLaneChangeDesireDirection @25 :NavDirection; # For "Navigation: Initiating Lane Change" UI alerts
navLanePositioningDirection @26 :NavDirection; # For future lane positioning UI alerts
navSpeedTargetActive @27 :Bool; # For "Navigation: Reducing Speed" UI alert
# Second next maneuver information (for "Then" section in navigation banner UI)
secondNextManeuverValid @28 :Bool; # Whether second next maneuver data is valid
secondNextManeuverType @29 :ManeuverType; # Type of second next maneuver
secondNextManeuverDirection @30 :NavDirection; # Direction for second next maneuver
secondNextManeuverDistance @31 :Float32; # Distance to second next maneuver (m)
nextManeuverModifier @32 :Text; # Raw Mapbox modifier for next maneuver (slight_left, sharp_right, etc.)
secondNextManeuverModifier @33 :Text; # Raw Mapbox modifier for second next maneuver
longitudinalProvider @34 :LongitudinalProvider; # Active source of nav longitudinal influence
longitudinalState @35 :LongitudinalState; # Current nav longitudinal state machine output
longitudinalEngaged @36 :Bool; # Whether nav longitudinal influence is currently active
speedTarget @37 :Float32; # Nav longitudinal speed target (m/s)
accelTarget @38 :Float32; # Nav longitudinal accel target (m/s^2)
valid @39 :Bool; # Whether nav longitudinal target is valid
maneuverPhase @40 :ManeuverPhase; # IQ nav maneuver phase for desire/FSM integration
maneuverDirection @41 :NavDirection; # Direction of active IQ nav maneuver phase
command @42 :Command; # Short-lived IQ nav command trigger for desire/FSM integration
commandDirection @43 :NavDirection; # Direction associated with current IQ nav command
commandIndex @44 :UInt32; # Monotonic counter incremented when nav emits a new IQ command
cameraValid @45 :Bool; # Whether a speed camera ahead is currently detected
cameraType @46 :CameraType; # Type of the upcoming speed camera
cameraDistance @47 :Float32; # Distance to the upcoming camera (m)
cameraSpeedLimit @48 :Float32; # Enforced speed limit at the camera (m/s)
timeRemainingTypical @49 :Float32; # Remaining time under typical traffic conditions (s)
trafficDelay @50 :Float32; # Remaining delay versus typical conditions (s)
trafficDataAge @51 :Float32; # Age of the active traffic snapshot (s)
trafficDataValid @52 :Bool; # Whether traffic metadata is fresh enough to trust
trafficClosureCount @53 :UInt16; # Closures reported along the active route
trafficRouteId @54 :Text; # Stable identity of the active traffic route
trafficSource @55 :Text; # Routing provider that supplied traffic metadata
trafficTimeRemaining @56 :Float32; # Remaining time from the latest traffic snapshot (s)
trafficDataTimestamp @57 :Float64; # UTC acquisition time of the active traffic snapshot
cameraAlertId @58 :Text;
cameraChime @59 :Bool;
cameraTrusted @60 :Bool;
cameraSourceAge @61 :Float32;
enum CameraType {
none @0;
fixedSpeed @1; # Fixed speed camera
mobileSpeed @2; # Mobile/handheld speed camera
sectionStart @3; # Average-speed (section) zone start
sectionEnd @4; # Average-speed (section) zone end
averageZone @5; # Within an average-speed zone
redLight @6; # Red-light camera
bump @7; # Speed bump
alpr @8; # ALPR / Flock surveillance camera (DeFlock/OSM surveillance:type=ALPR)
police @9;
}
enum ManeuverType {
none @0;
turn @1; # Regular turn at intersection
exit @2; # Highway exit
merge @3; # Merge onto highway
fork @4; # Road fork
continueStraight @5; # Continue straight
arrive @6; # Arrive at destination
roundabout @7; # Enter/exit roundabout
}
enum LongitudinalProvider {
none @0;
route @1;
mapbox @2;
vision @3;
offlineOsm @4;
camera @5;
}
enum LongitudinalState {
disabled @0;
enabled @1;
entering @2;
active @3;
leaving @4;
overriding @5;
}
enum ManeuverPhase {
none @0;
turnPrepare @1;
turnActive @2;
highwayPrepare @3;
highwayCommit @4;
}
enum Command {
none @0;
laneChange @1;
}
struct LaneDebugInfo {
modelLane @0 :Text; # "left", "middle", "right", "unknown"
modelConfidence @1 :Float32; # 0.0-1.0
gpsLane @2 :Text; # "left", "middle", "right", "unknown"
gpsConfidence @3 :Float32; # 0.0-1.0
lateralOffset @4 :Float32; # Meters from road centerline (negative=left, positive=right)
gpsAccuracy @5 :Float32; # GPS position accuracy (meters)
agreement @6 :Bool; # Do model and GPS agree?
}
}
struct IQRoadIncidentFeed @0xf2b94d5ed8504f81 {
valid @0 :Bool;
status @1 :Status;
fetchedMonoTime @2 :UInt64;
fetchedWallTime @3 :Float64;
reports @4 :List(Report);
enum Status {
idle @0;
ok @1;
missingKey @2;
noLocation @3;
networkError @4;
invalidResponse @5;
}
struct Report {
id @0 :Text;
latitude @1 :Float64;
longitude @2 :Float64;
publishedAt @3 :Float64;
street @4 :Text;
city @5 :Text;
headingValid @6 :Bool;
heading @7 :Float32;
}
}
struct IQNavRenderState @0xf6e4a54ca6c92276 {
active @0 :Bool;
currentLatitude @1 :Float64;
currentLongitude @2 :Float64;
bearingDeg @3 :Float32;
routePolyline @4 :List(NavPoint);
routePolylineSimplified @5 :List(NavPoint);
nextManeuverLatitude @6 :Float64;
nextManeuverLongitude @7 :Float64;
nextManeuverType @8 :IQNavState.ManeuverType;
nextManeuverDirection @9 :NavDirection;
nextManeuverDistance @10 :Float32;
destinationLatitude @11 :Float64;
destinationLongitude @12 :Float64;
zoomHint @13 :Float32;
struct NavPoint {
latitude @0 :Float64;
longitude @1 :Float64;
}
}
struct IQPerfTrace @0xa8e2e4a8c6f4d3b2 {
process @0 :Text;
eventClass @1 :Text;
severity @2 :Severity;
frameId @3 :UInt32;
totalTimeUs @4 :UInt32;
rkRemainingUs @5 :Int32;
batchSize @6 :UInt16;
droppedFrames @7 :UInt16;
backlog @8 :UInt16;
flags @9 :UInt32;
samples @10 :List(Sample);
missingServices @11 :List(Text);
topProcesses @12 :List(Text);
detail @13 :Text;
enum Severity {
info @0;
warning @1;
error @2;
critical @3;
}
struct Sample {
frameId @0 :UInt32;
loopDtUs @1 :UInt32;
updateUs @2 :UInt32;
stateControlUs @3 :UInt32;
publishUs @4 :UInt32;
tailWorkUs @5 :UInt32;
rkRemainingUs @6 :Int32;
staleCarControlUs @7 :UInt32;
staleCarControlFrames @8 :UInt16;
sendcanGapUs @9 :UInt32;
modelEvalUs @10 :UInt32;
modelDroppedFrames @11 :UInt16;
modelBacklog @12 :UInt16;
textureDecodeUs @13 :UInt32;
textureUploadUs @14 :UInt32;
textureUnloadUs @15 :UInt32;
texturePruneUs @16 :UInt32;
textureConsumeUs @17 :UInt32;
textureBatchSize @18 :UInt16;
textureBytes @19 :UInt32;
textureCacheBefore @20 :UInt16;
textureCacheAfter @21 :UInt16;
textureUnloaded @22 :UInt16;
memoryUsagePercent @23 :UInt16;
gpuUsagePercent @24 :UInt16;
cpuUsagePercent @25 :UInt16;
flags @26 :UInt32;
}
}
struct IQConstructionZone @0xb54d6e69da4ddc9f {
state @0 :State;
active @1 :Bool;
orangeFraction @2 :Float32; # hot-orange fraction of ROI chroma samples this analysis
secondsSinceHit @3 :Float32; # time since last frame that passed the hit threshold
enum State {
inactive @0;
pending @1; # hits seen, not yet enough persistence to enter
active @2;
}
}
struct IQVehicleTracks @0xb877ef4b20a4ae22 {
frameId @0 :UInt32;
frameWidth @1 :UInt16;
frameHeight @2 :UInt16;
processingMs @3 :Float32;
tracks @4 :List(Track);
wide @5 :Bool;
struct Track {
# box corners normalized [0,1] in the road-camera frame
x1 @0 :Float32;
y1 @1 :Float32;
x2 @2 :Float32;
y2 @3 :Float32;
prob @4 :Float32;
label @5 :Label;
enum Label {
car @0;
motorcycle @1;
bus @2;
truck @3;
person @4;
bicycle @5;
stopSign @6;
trafficLight @7;
}
}
}
struct IQEnvironment @0xfd960244a79e2804 {
frameId @0 :UInt32;
offloaded @1 :Bool;
modelValid @2 :Bool;
objects @3 :List(Object);
struct Object {
x @0 :Float32;
y @1 :Float32;
z @2 :Float32;
width @3 :Float32;
height @4 :Float32;
length @5 :Float32;
prob @6 :Float32;
label @7 :Label;
enum Label {
car @0;
motorcycle @1;
bus @2;
truck @3;
person @4;
bicycle @5;
stopSign @6;
trafficLight @7;
}
}
}
struct CustomReserved14 @0xa6e5a1ce8ca5258e {
}
struct CustomReserved15 @0xdb8042111e62cc87 {
}
struct CustomReserved16 @0xf59900ccf47b651a {
}
# pfeiferj/mapd v2 output schema (struct ids must match the mapd binary exactly).
struct MapdDownloadLocationDetails @0xff889853e7b0987f {
location @0 :Text;
totalFiles @1 :UInt32;
downloadedFiles @2 :UInt32;
}
struct MapdDownloadProgress @0xfaa35dcac85073a2 {
active @0 :Bool;
cancelled @1 :Bool;
totalFiles @2 :UInt32;
downloadedFiles @3 :UInt32;
locations @4 :List(Text);
locationDetails @5 :List(MapdDownloadLocationDetails);
}
struct MapdPathPoint @0xd6f78acca1bc3939 {
latitude @0 :Float64;
longitude @1 :Float64;
curvature @2 :Float32;
targetVelocity @3 :Float32;
}
enum MapdRoadContext {
freeway @0;
city @1;
unknown @2;
}
enum MapdWaySelectionType {
current @0;
predicted @1;
possible @2;
extended @3;
fail @4;
}
enum MapdInputType {
download @0;
setTargetLateralAccel @1;
setSpeedLimitOffset @2;
setSpeedLimitControl @3;
setMapCurveSpeedControl @4;
setVisionCurveSpeedControl @5;
setLogLevel @6;
setVisionCurveTargetLatA @7;
setVisionCurveMinTargetV @8;
reloadSettings @9;
saveSettings @10;
setEnableSpeed @11;
setVisionCurveUseEnableSpeed @12;
setMapCurveUseEnableSpeed @13;
setSpeedLimitUseEnableSpeed @14;
setHoldLastSeenSpeedLimit @15;
setTargetSpeedJerk @16;
setTargetSpeedAccel @17;
setTargetSpeedTimeOffset @18;
setDefaultLaneWidth @19;
setMapCurveTargetLatA @20;
loadDefaultSettings @21;
loadRecommendedSettings @22;
setSlowDownForNextSpeedLimit @23;
setSpeedUpForNextSpeedLimit @24;
setHoldSpeedLimitWhileChangingSetSpeed @25;
loadPersistentSettings @26;
cancelDownload @27;
setLogJson @28;
setLogSource @29;
setExternalSpeedLimitControl @30;
setExternalSpeedLimit @31;
setSpeedLimitPriority @32;
setSpeedLimitChangeRequiresAccept @33;
acceptSpeedLimit @34;
setPressGasToAcceptSpeedLimit @35;
setAdjustSetSpeedToAcceptSpeedLimit @36;
setAcceptSpeedLimitTimeout @37;
setPressGasToOverrideSpeedLimit @38;
}
struct MapdExtendedOut @0x8d12e6a08c60a1a1 {
downloadProgress @0 :MapdDownloadProgress;
settings @1 :Text;
path @2 :List(MapdPathPoint);
}
struct MapdIn @0xb9ceb3ea89cecc23 {
type @0 :MapdInputType;
float @1 :Float32;
str @2 :Text;
bool @3 :Bool;
}
struct MapdOut @0xd615f9fe1608a3c0 {
wayName @0 :Text;
wayRef @1 :Text;
roadName @2 :Text;
speedLimit @3 :Float32;
nextSpeedLimit @4 :Float32;
nextSpeedLimitDistance @5 :Float32;
hazard @6 :Text;
nextHazard @7 :Text;
nextHazardDistance @8 :Float32;
advisorySpeed @9 :Float32;
nextAdvisorySpeed @10 :Float32;
nextAdvisorySpeedDistance @11 :Float32;
oneWay @12 :Bool;
lanes @13 :UInt8;
tileLoaded @14 :Bool;
speedLimitSuggestedSpeed @15 :Float32;
suggestedSpeed @16 :Float32;
estimatedRoadWidth @17 :Float32;
roadContext @18 :MapdRoadContext;
distanceFromWayCenter @19 :Float32;
visionCurveSpeed @20 :Float32;
mapCurveSpeed @21 :Float32;
waySelectionType @22 :MapdWaySelectionType;
speedLimitAccepted @23 :Bool;
}

26
iqpilot/cereal/include/c++.capnp Normal file
View File

@@ -0,0 +1,26 @@
# Copyright (c) 2013-2014 Sandstorm Development Group, Inc. and contributors
# Licensed under the MIT License:
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
@0xbdf87d7bb8304e81;
$namespace("capnp::annotations");
annotation namespace(file): Text;
annotation name(field, enumerant, struct, enum, interface, method, param, group, union): Text;

573
iqpilot/cereal/legacy.capnp Normal file
View File

@@ -0,0 +1,573 @@
using Cxx = import "/include/c++.capnp";
$Cxx.namespace("cereal");
@0x80ef1ec4889c2a63;
# legacy.capnp: a home for deprecated structs
struct LogRotate @0x9811e1f38f62f2d1 {
segmentNum @0 :Int32;
path @1 :Text;
}
struct LiveUI @0xc08240f996aefced {
rearViewCam @0 :Bool;
alertText1 @1 :Text;
alertText2 @2 :Text;
awarenessStatus @3 :Float32;
}
struct UiLayoutState @0x88dcce08ad29dda0 {
activeApp @0 :App;
sidebarCollapsed @1 :Bool;
mapEnabled @2 :Bool;
mockEngaged @3 :Bool;
enum App @0x9917470acf94d285 {
home @0;
music @1;
nav @2;
settings @3;
none @4;
}
}
struct OrbslamCorrection @0x8afd33dc9b35e1aa {
correctionMonoTime @0 :UInt64;
prePositionECEF @1 :List(Float64);
postPositionECEF @2 :List(Float64);
prePoseQuatECEF @3 :List(Float32);
postPoseQuatECEF @4 :List(Float32);
numInliers @5 :UInt32;
}
struct EthernetPacket @0xa99a9d5b33cf5859 {
pkt @0 :Data;
ts @1 :Float32;
}
struct CellInfo @0xcff7566681c277ce {
timestamp @0 :UInt64;
repr @1 :Text; # android toString() for now
}
struct WifiScan @0xd4df5a192382ba0b {
bssid @0 :Text;
ssid @1 :Text;
capabilities @2 :Text;
frequency @3 :Int32;
level @4 :Int32;
timestamp @5 :Int64;
centerFreq0 @6 :Int32;
centerFreq1 @7 :Int32;
channelWidth @8 :ChannelWidth;
operatorFriendlyName @9 :Text;
venueName @10 :Text;
is80211mcResponder @11 :Bool;
passpoint @12 :Bool;
distanceCm @13 :Int32;
distanceSdCm @14 :Int32;
enum ChannelWidth @0xcb6a279f015f6b51 {
w20Mhz @0;
w40Mhz @1;
w80Mhz @2;
w160Mhz @3;
w80Plus80Mhz @4;
}
}
struct LiveEventData @0x94b7baa90c5c321e {
name @0 :Text;
value @1 :Int32;
}
struct ModelData @0xb8aad62cffef28a9 {
frameId @0 :UInt32;
frameAge @12 :UInt32;
frameDropPerc @13 :Float32;
timestampEof @9 :UInt64;
modelExecutionTime @14 :Float32;
gpuExecutionTime @16 :Float32;
rawPred @15 :Data;
path @1 :PathData;
leftLane @2 :PathData;
rightLane @3 :PathData;
lead @4 :LeadData;
freePath @6 :List(Float32);
settings @5 :ModelSettings;
leadFuture @7 :LeadData;
speed @8 :List(Float32);
meta @10 :MetaData;
longitudinal @11 :LongitudinalData;
struct PathData @0x8817eeea389e9f08 {
points @0 :List(Float32);
prob @1 :Float32;
std @2 :Float32;
stds @3 :List(Float32);
poly @4 :List(Float32);
validLen @5 :Float32;
}
struct LeadData @0xd1c9bef96d26fa91 {
dist @0 :Float32;
prob @1 :Float32;
std @2 :Float32;
relVel @3 :Float32;
relVelStd @4 :Float32;
relY @5 :Float32;
relYStd @6 :Float32;
relA @7 :Float32;
relAStd @8 :Float32;
}
struct ModelSettings @0xa26e3710efd3e914 {
bigBoxX @0 :UInt16;
bigBoxY @1 :UInt16;
bigBoxWidth @2 :UInt16;
bigBoxHeight @3 :UInt16;
boxProjection @4 :List(Float32);
yuvCorrection @5 :List(Float32);
inputTransform @6 :List(Float32);
}
struct MetaData @0x9744f25fb60f2bf8 {
engagedProb @0 :Float32;
desirePrediction @1 :List(Float32);
brakeDisengageProb @2 :Float32;
gasDisengageProb @3 :Float32;
steerOverrideProb @4 :Float32;
desireState @5 :List(Float32);
}
struct LongitudinalData @0xf98f999c6a071122 {
distances @2 :List(Float32);
speeds @0 :List(Float32);
accelerations @1 :List(Float32);
}
}
struct ECEFPoint @0xc25bbbd524983447 {
x @0 :Float64;
y @1 :Float64;
z @2 :Float64;
}
struct ECEFPointDEPRECATED @0xe10e21168db0c7f7 {
x @0 :Float32;
y @1 :Float32;
z @2 :Float32;
}
struct GPSPlannerPoints @0xab54c59699f8f9f3 {
curPosDEPRECATED @0 :ECEFPointDEPRECATED;
pointsDEPRECATED @1 :List(ECEFPointDEPRECATED);
curPos @6 :ECEFPoint;
points @7 :List(ECEFPoint);
valid @2 :Bool;
trackName @3 :Text;
speedLimit @4 :Float32;
accelTarget @5 :Float32;
}
struct GPSPlannerPlan @0xf5ad1d90cdc1dd6b {
valid @0 :Bool;
poly @1 :List(Float32);
trackName @2 :Text;
speed @3 :Float32;
acceleration @4 :Float32;
pointsDEPRECATED @5 :List(ECEFPointDEPRECATED);
points @6 :List(ECEFPoint);
xLookahead @7 :Float32;
}
struct UiNavigationEvent @0x90c8426c3eaddd3b {
type @0: Type;
status @1: Status;
distanceTo @2: Float32;
endRoadPointDEPRECATED @3: ECEFPointDEPRECATED;
endRoadPoint @4: ECEFPoint;
enum Type @0xe8db07dcf8fcea05 {
none @0;
laneChangeLeft @1;
laneChangeRight @2;
mergeLeft @3;
mergeRight @4;
turnLeft @5;
turnRight @6;
}
enum Status @0xb9aa88c75ef99a1f {
none @0;
passive @1;
approaching @2;
active @3;
}
}
struct LiveLocationData @0xb99b2bc7a57e8128 {
status @0 :UInt8;
# 3D fix
lat @1 :Float64;
lon @2 :Float64;
alt @3 :Float32; # m
# speed
speed @4 :Float32; # m/s
# NED velocity components
vNED @5 :List(Float32);
# roll, pitch, heading (x,y,z)
roll @6 :Float32; # WRT to center of earth?
pitch @7 :Float32; # WRT to center of earth?
heading @8 :Float32; # WRT to north?
# what are these?
wanderAngle @9 :Float32;
trackAngle @10 :Float32;
# car frame -- https://upload.wikimedia.org/wikipedia/commons/f/f5/RPY_angles_of_cars.png
# gyro, in car frame, deg/s
gyro @11 :List(Float32);
# accel, in car frame, m/s^2
accel @12 :List(Float32);
accuracy @13 :Accuracy;
source @14 :SensorSource;
# if we are fixing a location in the past
fixMonoTime @15 :UInt64;
gpsWeek @16 :Int32;
timeOfWeek @17 :Float64;
positionECEF @18 :List(Float64);
poseQuatECEF @19 :List(Float32);
pitchCalibration @20 :Float32;
yawCalibration @21 :Float32;
imuFrame @22 :List(Float32);
struct Accuracy @0x943dc4625473b03f {
pNEDError @0 :List(Float32);
vNEDError @1 :List(Float32);
rollError @2 :Float32;
pitchError @3 :Float32;
headingError @4 :Float32;
ellipsoidSemiMajorError @5 :Float32;
ellipsoidSemiMinorError @6 :Float32;
ellipsoidOrientationError @7 :Float32;
}
enum SensorSource @0xc871d3cc252af657 {
applanix @0;
kalman @1;
orbslam @2;
timing @3;
dummy @4;
}
}
struct OrbOdometry @0xd7700859ed1f5b76 {
# timing first
startMonoTime @0 :UInt64;
endMonoTime @1 :UInt64;
# fundamental matrix and error
f @2: List(Float64);
err @3: Float64;
# number of inlier points
inliers @4: Int32;
# for debug only
# indexed by endMonoTime features
# value is startMonoTime feature match
# -1 if no match
matches @5: List(Int16);
}
struct OrbFeatures @0xcd60164a8a0159ef {
timestampEof @0 :UInt64;
# transposed arrays of normalized image coordinates
# len(xs) == len(ys) == len(descriptors) * 32
xs @1 :List(Float32);
ys @2 :List(Float32);
descriptors @3 :Data;
octaves @4 :List(Int8);
# match index to last OrbFeatures
# -1 if no match
timestampLastEof @5 :UInt64;
matches @6: List(Int16);
}
struct OrbFeaturesSummary @0xd500d30c5803fa4f {
timestampEof @0 :UInt64;
timestampLastEof @1 :UInt64;
featureCount @2 :UInt16;
matchCount @3 :UInt16;
computeNs @4 :UInt64;
}
struct OrbKeyFrame @0xc8233c0345e27e24 {
# this is a globally unique id for the KeyFrame
id @0: UInt64;
# this is the location of the KeyFrame
pos @1: ECEFPoint;
# these are the features in the world
# len(dpos) == len(descriptors) * 32
dpos @2 :List(ECEFPoint);
descriptors @3 :Data;
}
struct KalmanOdometry @0x92e21bb7ea38793a {
trans @0 :List(Float32); # m/s in device frame
rot @1 :List(Float32); # rad/s in device frame
transStd @2 :List(Float32); # std m/s in device frame
rotStd @3 :List(Float32); # std rad/s in device frame
}
struct OrbObservation @0x9b326d4e436afec7 {
observationMonoTime @0 :UInt64;
normalizedCoordinates @1 :List(Float32);
locationECEF @2 :List(Float64);
matchDistance @3: UInt32;
}
struct CalibrationFeatures @0x8fdfadb254ea867a {
frameId @0 :UInt32;
p0 @1 :List(Float32);
p1 @2 :List(Float32);
status @3 :List(Int8);
}
struct NavStatus @0xbd8822120928120c {
isNavigating @0 :Bool;
currentAddress @1 :Address;
struct Address @0xce7cd672cacc7814 {
title @0 :Text;
lat @1 :Float64;
lng @2 :Float64;
house @3 :Text;
address @4 :Text;
street @5 :Text;
city @6 :Text;
state @7 :Text;
country @8 :Text;
}
}
struct NavUpdate @0xdb98be6565516acb {
isNavigating @0 :Bool;
curSegment @1 :Int32;
segments @2 :List(Segment);
struct LatLng @0x9eaef9187cadbb9b {
lat @0 :Float64;
lng @1 :Float64;
}
struct Segment @0xa5b39b4fc4d7da3f {
from @0 :LatLng;
to @1 :LatLng;
updateTime @2 :Int32;
distance @3 :Int32;
crossTime @4 :Int32;
exitNo @5 :Int32;
instruction @6 :Instruction;
parts @7 :List(LatLng);
enum Instruction @0xc5417a637451246f {
turnLeft @0;
turnRight @1;
keepLeft @2;
keepRight @3;
straight @4;
roundaboutExitNumber @5;
roundaboutExit @6;
roundaboutTurnLeft @7;
unkn8 @8;
roundaboutStraight @9;
unkn10 @10;
roundaboutTurnRight @11;
unkn12 @12;
roundaboutUturn @13;
unkn14 @14;
arrive @15;
exitLeft @16;
exitRight @17;
unkn18 @18;
uturn @19;
# ...
}
}
}
struct TrafficEvent @0xacfa74a094e62626 {
type @0 :Type;
distance @1 :Float32;
action @2 :Action;
resuming @3 :Bool;
enum Type @0xd85d75253435bf4b {
stopSign @0;
lightRed @1;
lightYellow @2;
lightGreen @3;
stopLight @4;
}
enum Action @0xa6f6ce72165ccb49 {
none @0;
yield @1;
stop @2;
resumeReady @3;
}
}
struct AndroidGnss @0xdfdf30d03fc485bd {
union {
measurements @0 :Measurements;
navigationMessage @1 :NavigationMessage;
}
struct Measurements @0xa20710d4f428d6cd {
clock @0 :Clock;
measurements @1 :List(Measurement);
struct Clock @0xa0e27b453a38f450 {
timeNanos @0 :Int64;
hardwareClockDiscontinuityCount @1 :Int32;
hasTimeUncertaintyNanos @2 :Bool;
timeUncertaintyNanos @3 :Float64;
hasLeapSecond @4 :Bool;
leapSecond @5 :Int32;
hasFullBiasNanos @6 :Bool;
fullBiasNanos @7 :Int64;
hasBiasNanos @8 :Bool;
biasNanos @9 :Float64;
hasBiasUncertaintyNanos @10 :Bool;
biasUncertaintyNanos @11 :Float64;
hasDriftNanosPerSecond @12 :Bool;
driftNanosPerSecond @13 :Float64;
hasDriftUncertaintyNanosPerSecond @14 :Bool;
driftUncertaintyNanosPerSecond @15 :Float64;
}
struct Measurement @0xd949bf717d77614d {
svId @0 :Int32;
constellation @1 :Constellation;
timeOffsetNanos @2 :Float64;
state @3 :Int32;
receivedSvTimeNanos @4 :Int64;
receivedSvTimeUncertaintyNanos @5 :Int64;
cn0DbHz @6 :Float64;
pseudorangeRateMetersPerSecond @7 :Float64;
pseudorangeRateUncertaintyMetersPerSecond @8 :Float64;
accumulatedDeltaRangeState @9 :Int32;
accumulatedDeltaRangeMeters @10 :Float64;
accumulatedDeltaRangeUncertaintyMeters @11 :Float64;
hasCarrierFrequencyHz @12 :Bool;
carrierFrequencyHz @13 :Float32;
hasCarrierCycles @14 :Bool;
carrierCycles @15 :Int64;
hasCarrierPhase @16 :Bool;
carrierPhase @17 :Float64;
hasCarrierPhaseUncertainty @18 :Bool;
carrierPhaseUncertainty @19 :Float64;
hasSnrInDb @20 :Bool;
snrInDb @21 :Float64;
multipathIndicator @22 :MultipathIndicator;
enum Constellation @0x9ef1f3ff0deb5ffb {
unknown @0;
gps @1;
sbas @2;
glonass @3;
qzss @4;
beidou @5;
galileo @6;
}
enum State @0xcbb9490adce12d72 {
unknown @0;
codeLock @1;
bitSync @2;
subframeSync @3;
towDecoded @4;
msecAmbiguous @5;
symbolSync @6;
gloStringSync @7;
gloTodDecoded @8;
bdsD2BitSync @9;
bdsD2SubframeSync @10;
galE1bcCodeLock @11;
galE1c2ndCodeLock @12;
galE1bPageSync @13;
sbasSync @14;
}
enum MultipathIndicator @0xc04e7b6231d4caa8 {
unknown @0;
detected @1;
notDetected @2;
}
}
}
struct NavigationMessage @0xe2517b083095fd4e {
type @0 :Int32;
svId @1 :Int32;
messageId @2 :Int32;
submessageId @3 :Int32;
data @4 :Data;
status @5 :Status;
enum Status @0xec1ff7996b35366f {
unknown @0;
parityPassed @1;
parityRebuilt @2;
}
}
}
struct LidarPts @0xe3d6685d4e9d8f7a {
r @0 :List(UInt16); # uint16 m*500.0
theta @1 :List(UInt16); # uint16 deg*100.0
reflect @2 :List(UInt8); # uint8 0-255
# For storing out of file.
idx @3 :UInt64;
# For storing in file
pkt @4 :Data;
}

2739
iqpilot/cereal/log.capnp Normal file
View File

File diff suppressed because it is too large Load Diff

269
iqpilot/cereal/messaging/__init__.py Normal file
View File

@@ -0,0 +1,269 @@
# must be built with scons
from msgq import fake_event_handle, drain_sock_raw, MultiplePublishersError, IpcError, \
Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event
import msgq
import os
import capnp
import time
from typing import Optional, List, Union, Dict
from iqpilot.cereal import log
from iqpilot.cereal.services import SERVICE_LIST
from iqpilot.common.utils import MovingAverage
NO_TRAVERSAL_LIMIT = 2**64-1
def pub_sock(endpoint: str) -> PubSocket:
service = SERVICE_LIST.get(endpoint)
segment_size = service.queue_size if service else 0
return msgq.pub_sock(endpoint, segment_size)
def sub_sock(endpoint: str, poller: Optional[Poller] = None, addr: str = "127.0.0.1",
conflate: bool = False, timeout: Optional[int] = None) -> SubSocket:
service = SERVICE_LIST.get(endpoint)
segment_size = service.queue_size if service else 0
return msgq.sub_sock(endpoint, poller=poller, addr=addr, conflate=conflate,
timeout=timeout, segment_size=segment_size)
def reset_context():
msgq.context = Context()
def log_from_bytes(dat: bytes, struct: capnp.lib.capnp._StructModule = log.Event) -> capnp.lib.capnp._DynamicStructReader:
with struct.from_bytes(dat, traversal_limit_in_words=NO_TRAVERSAL_LIMIT) as msg:
return msg
def new_message(service: Optional[str], size: Optional[int] = None, **kwargs) -> capnp.lib.capnp._DynamicStructBuilder:
args = {
'valid': False,
'logMonoTime': int(time.monotonic() * 1e9),
**kwargs
}
dat = log.Event.new_message(**args)
if service is not None:
if size is None:
dat.init(service)
else:
dat.init(service, size)
return dat
def drain_sock(sock: SubSocket, wait_for_one: bool = False) -> List[capnp.lib.capnp._DynamicStructReader]:
"""Receive all message currently available on the queue"""
msgs = drain_sock_raw(sock, wait_for_one=wait_for_one)
return [log_from_bytes(m) for m in msgs]
# TODO: print when we drop packets?
def recv_sock(sock: SubSocket, wait: bool = False) -> Optional[capnp.lib.capnp._DynamicStructReader]:
"""Same as drain sock, but only returns latest message. Consider using conflate instead."""
dat = None
while 1:
if wait and dat is None:
recv = sock.receive()
else:
recv = sock.receive(non_blocking=True)
if recv is None: # Timeout hit
break
dat = recv
if dat is not None:
dat = log_from_bytes(dat)
return dat
def recv_one(sock: SubSocket) -> Optional[capnp.lib.capnp._DynamicStructReader]:
dat = sock.receive()
if dat is not None:
dat = log_from_bytes(dat)
return dat
def recv_one_or_none(sock: SubSocket) -> Optional[capnp.lib.capnp._DynamicStructReader]:
dat = sock.receive(non_blocking=True)
if dat is not None:
dat = log_from_bytes(dat)
return dat
def recv_one_retry(sock: SubSocket) -> capnp.lib.capnp._DynamicStructReader:
"""Keep receiving until we get a message"""
while True:
dat = sock.receive()
if dat is not None:
return log_from_bytes(dat)
class FrequencyTracker:
def __init__(self, service_freq: float, update_freq: float, is_poll: bool):
freq = max(min(service_freq, update_freq), 1.)
if is_poll:
min_freq = max_freq = freq
else:
max_freq = min(freq, update_freq)
if service_freq >= 2 * update_freq:
min_freq = update_freq
elif update_freq >= 2* service_freq:
min_freq = freq
else:
min_freq = min(freq, freq / 2.)
self.min_freq = min_freq * 0.8
self.max_freq = max_freq * 1.2
self.avg_dt = MovingAverage(int(10 * freq))
self.recent_avg_dt = MovingAverage(int(freq))
self.prev_time = 0.0
def record_recv_time(self, cur_time: float) -> None:
# TODO: Handle case where cur_time is less than prev_time
if self.prev_time > 1e-5:
dt = cur_time - self.prev_time
self.avg_dt.add_value(dt)
self.recent_avg_dt.add_value(dt)
self.prev_time = cur_time
@property
def valid(self) -> bool:
if self.avg_dt.count == 0:
return False
avg_freq = 1.0 / self.avg_dt.get_average()
if self.min_freq <= avg_freq <= self.max_freq:
return True
avg_freq_recent = 1.0 / self.recent_avg_dt.get_average()
return self.min_freq <= avg_freq_recent <= self.max_freq
class SubMaster:
def __init__(self, services: List[str], poll: Optional[str] = None,
ignore_alive: Optional[List[str]] = None, ignore_avg_freq: Optional[List[str]] = None,
ignore_valid: Optional[List[str]] = None, addr: str = "127.0.0.1", frequency: Optional[float] = None):
self.frame = -1
self.services = services
self.seen = {s: False for s in services}
self.updated = {s: False for s in services}
self.recv_time = {s: 0. for s in services}
self.recv_frame = {s: 0 for s in services}
self.sock = {}
self.data = {}
self.logMonoTime = {s: 0 for s in services}
# zero-frequency / on-demand services are always alive and presumed valid; all others must pass checks
on_demand = {s: SERVICE_LIST[s].frequency <= 1e-5 for s in services}
self.static_freq_services = set(s for s in services if not on_demand[s])
self.alive = {s: on_demand[s] for s in services}
self.freq_ok = {s: on_demand[s] for s in services}
self.valid = {s: on_demand[s] for s in services}
self.freq_tracker: Dict[str, FrequencyTracker] = {}
self.poller = Poller()
polled_services = set([poll, ] if poll is not None else services)
self.non_polled_services = set(services) - polled_services
self.ignore_average_freq = [] if ignore_avg_freq is None else ignore_avg_freq
self.ignore_alive = [] if ignore_alive is None else ignore_alive
self.ignore_valid = [] if ignore_valid is None else ignore_valid
self.simulation = bool(int(os.getenv("SIMULATION", "0")))
# if freq and poll aren't specified, assume the max to be conservative
assert frequency is None or poll is None, "Do not specify 'frequency' - frequency of the polled service will be used."
self.update_freq = frequency or max([SERVICE_LIST[s].frequency for s in polled_services])
for s in services:
p = self.poller if s not in self.non_polled_services else None
self.sock[s] = sub_sock(s, poller=p, addr=addr, conflate=True)
try:
data = new_message(s)
except capnp.lib.capnp.KjException:
data = new_message(s, 0) # lists
self.data[s] = getattr(data.as_reader(), s)
self.freq_tracker[s] = FrequencyTracker(SERVICE_LIST[s].frequency, self.update_freq, s == poll)
def __getitem__(self, s: str) -> capnp.lib.capnp._DynamicStructReader:
return self.data[s]
def _check_avg_freq(self, s: str) -> bool:
return SERVICE_LIST[s].frequency > 0.99 and (s not in self.ignore_average_freq) and (s not in self.ignore_alive)
def update(self, timeout: int = 100) -> None:
msgs = []
for sock in self.poller.poll(timeout):
msgs.append(recv_one_or_none(sock))
# non-blocking receive for non-polled sockets
for s in self.non_polled_services:
msgs.append(recv_one_or_none(self.sock[s]))
self.update_msgs(time.monotonic(), msgs)
def update_msgs(self, cur_time: float, msgs: List[capnp.lib.capnp._DynamicStructReader]) -> None:
self.frame += 1
self.updated = dict.fromkeys(self.services, False)
for msg in msgs:
if msg is None:
continue
s = msg.which()
self.seen[s] = True
self.updated[s] = True
self.freq_tracker[s].record_recv_time(cur_time)
self.recv_time[s] = cur_time
self.recv_frame[s] = self.frame
self.data[s] = getattr(msg, s)
self.logMonoTime[s] = msg.logMonoTime
self.valid[s] = msg.valid
for s in self.static_freq_services:
# alive if delay is within 10x the expected frequency; checks relaxed in simulator
self.alive[s] = (cur_time - self.recv_time[s]) < (10. / SERVICE_LIST[s].frequency) or (self.seen[s] and self.simulation)
self.freq_ok[s] = self.freq_tracker[s].valid or self.simulation
def all_alive(self, service_list: Optional[List[str]] = None) -> bool:
return all(self.alive[s] for s in (service_list or self.services) if s not in self.ignore_alive)
def all_freq_ok(self, service_list: Optional[List[str]] = None) -> bool:
return all(self.freq_ok[s] for s in (service_list or self.services) if self._check_avg_freq(s))
def all_valid(self, service_list: Optional[List[str]] = None) -> bool:
return all(self.valid[s] for s in (service_list or self.services) if s not in self.ignore_valid)
def all_checks(self, service_list: Optional[List[str]] = None) -> bool:
return self.all_alive(service_list) and self.all_freq_ok(service_list) and self.all_valid(service_list)
class PubMaster:
def __init__(self, services: List[str]):
self.sock = {}
for s in services:
self.sock[s] = pub_sock(s)
def send(self, s: str, dat: Union[bytes, capnp.lib.capnp._DynamicStructBuilder]) -> None:
if not isinstance(dat, bytes):
dat = dat.to_bytes()
self.sock[s].send(dat)
def wait_for_readers_to_update(self, s: str, timeout: int, dt: float = 0.05) -> bool:
for _ in range(int(timeout*(1./dt))):
if self.sock[s].all_readers_updated():
return True
time.sleep(dt)
return False
def all_readers_updated(self, s: str) -> bool:
return self.sock[s].all_readers_updated() # type: ignore

72
iqpilot/cereal/messaging/bridge.cc Normal file
View File

@@ -0,0 +1,72 @@
#include <cassert>
#include "cereal/messaging/msgq_to_zmq.h"
#include "cereal/services.h"
#include "common/util.h"
ExitHandler do_exit;
static std::vector<std::string> get_services(const std::string &whitelist_str, bool zmq_to_msgq) {
std::vector<std::string> service_list;
for (const auto& it : services) {
std::string name = it.second.name;
bool in_whitelist = whitelist_str.find(name) != std::string::npos;
if (zmq_to_msgq && !in_whitelist) {
continue;
}
service_list.push_back(name);
}
return service_list;
}
void msgq_to_zmq(const std::vector<std::string> &endpoints, const std::string &ip) {
MsgqToZmq bridge;
bridge.run(endpoints, ip);
}
void zmq_to_msgq(const std::vector<std::string> &endpoints, const std::string &ip) {
auto poller = std::make_unique<ZMQPoller>();
auto pub_context = std::make_unique<MSGQContext>();
auto sub_context = std::make_unique<ZMQContext>();
std::map<SubSocket *, PubSocket *> sub2pub;
for (auto endpoint : endpoints) {
auto pub_sock = new MSGQPubSocket();
auto sub_sock = new ZMQSubSocket();
size_t queue_size = services.at(endpoint).queue_size;
pub_sock->connect(pub_context.get(), endpoint, true, queue_size);
sub_sock->connect(sub_context.get(), endpoint, ip, false);
poller->registerSocket(sub_sock);
sub2pub[sub_sock] = pub_sock;
}
while (!do_exit) {
for (auto sub_sock : poller->poll(100)) {
std::unique_ptr<Message> msg(sub_sock->receive(true));
if (msg) {
sub2pub[sub_sock]->sendMessage(msg.get());
}
}
}
// Clean up allocated sockets
for (auto &[sub_sock, pub_sock] : sub2pub) {
delete sub_sock;
delete pub_sock;
}
}
int main(int argc, char **argv) {
bool is_zmq_to_msgq = argc > 2;
std::string ip = is_zmq_to_msgq ? argv[1] : "127.0.0.1";
std::string whitelist_str = is_zmq_to_msgq ? std::string(argv[2]) : "";
std::vector<std::string> endpoints = get_services(whitelist_str, is_zmq_to_msgq);
if (is_zmq_to_msgq) {
zmq_to_msgq(endpoints, ip);
} else {
msgq_to_zmq(endpoints, ip);
}
return 0;
}

102
iqpilot/cereal/messaging/messaging.h Normal file
View File

@@ -0,0 +1,102 @@
#pragma once
#include <cstddef>
#include <map>
#include <string>
#include <vector>
#include <utility>
#include <capnp/serialize.h>
#include "cereal/gen/cpp/log.capnp.h"
#include "common/timing.h"
#include "msgq/ipc.h"
class SubMaster {
public:
SubMaster(const std::vector<const char *> &service_list, const std::vector<const char *> &poll = {},
const char *address = nullptr, const std::vector<const char *> &ignore_alive = {});
void update(int timeout = 1000);
void update_msgs(uint64_t current_time, const std::vector<std::pair<std::string, cereal::Event::Reader>> &messages);
inline bool allAlive(const std::vector<const char *> &service_list = {}) { return all_(service_list, false, true); }
inline bool allValid(const std::vector<const char *> &service_list = {}) { return all_(service_list, true, false); }
inline bool allAliveAndValid(const std::vector<const char *> &service_list = {}) { return all_(service_list, true, true); }
void drain();
~SubMaster();
uint64_t frame = 0;
bool updated(const char *name) const;
bool alive(const char *name) const;
bool valid(const char *name) const;
uint64_t rcv_frame(const char *name) const;
uint64_t rcv_time(const char *name) const;
cereal::Event::Reader &operator[](const char *name) const;
private:
bool all_(const std::vector<const char *> &service_list, bool valid, bool alive);
Poller *poller_ = nullptr;
struct SubMessage;
std::map<SubSocket *, SubMessage *> messages_;
std::map<std::string, SubMessage *> services_;
};
class MessageBuilder : public capnp::MallocMessageBuilder {
public:
MessageBuilder() = default;
cereal::Event::Builder initEvent(bool valid = true) {
cereal::Event::Builder event = initRoot<cereal::Event>();
event.setLogMonoTime(nanos_since_boot());
event.setValid(valid);
return event;
}
kj::ArrayPtr<capnp::byte> toBytes() {
heapArray_ = capnp::messageToFlatArray(*this);
return heapArray_.asBytes();
}
size_t getSerializedSize() {
return capnp::computeSerializedSizeInWords(*this) * sizeof(capnp::word);
}
int serializeToBuffer(unsigned char *buffer, size_t buffer_size) {
size_t serialized_size = getSerializedSize();
if (serialized_size > buffer_size) { return -1; }
kj::ArrayOutputStream out(kj::ArrayPtr<capnp::byte>(buffer, buffer_size));
capnp::writeMessage(out, *this);
return serialized_size;
}
private:
kj::Array<capnp::word> heapArray_;
};
class PubMaster {
public:
PubMaster(const std::vector<const char *> &service_list);
inline int send(const char *name, capnp::byte *data, size_t size) { return sockets_.at(name)->send((char *)data, size); }
int send(const char *name, MessageBuilder &msg);
~PubMaster();
private:
std::map<std::string, PubSocket *> sockets_;
};
class AlignedBuffer {
public:
kj::ArrayPtr<const capnp::word> align(const char *data, const size_t size) {
words_size = size / sizeof(capnp::word) + 1;
if (aligned_buf.size() < words_size) {
aligned_buf = kj::heapArray<capnp::word>(words_size < 512 ? 512 : words_size);
}
memcpy(aligned_buf.begin(), data, size);
return aligned_buf.slice(0, words_size);
}
inline kj::ArrayPtr<const capnp::word> align(Message *m) {
return align(m->getData(), m->getSize());
}
private:
kj::Array<capnp::word> aligned_buf;
size_t words_size;
};

146
iqpilot/cereal/messaging/msgq_to_zmq.cc Normal file
View File

@@ -0,0 +1,146 @@
#include "cereal/messaging/msgq_to_zmq.h"
#include <cassert>
#include "cereal/services.h"
#include "common/util.h"
extern ExitHandler do_exit;
// Max messages to process per socket per poll
constexpr int MAX_MESSAGES_PER_SOCKET = 50;
static std::string recv_zmq_msg(void *sock) {
zmq_msg_t msg;
zmq_msg_init(&msg);
std::string ret;
if (zmq_msg_recv(&msg, sock, 0) > 0) {
ret.assign((char *)zmq_msg_data(&msg), zmq_msg_size(&msg));
}
zmq_msg_close(&msg);
return ret;
}
void MsgqToZmq::run(const std::vector<std::string> &endpoints, const std::string &ip) {
zmq_context = std::make_unique<ZMQContext>();
msgq_context = std::make_unique<MSGQContext>();
// Create ZMQPubSockets for each endpoint
for (const auto &endpoint : endpoints) {
auto &socket_pair = socket_pairs.emplace_back();
socket_pair.endpoint = endpoint;
socket_pair.pub_sock = std::make_unique<ZMQPubSocket>();
int ret = socket_pair.pub_sock->connect(zmq_context.get(), endpoint);
if (ret != 0) {
printf("Failed to create ZMQ publisher for [%s]: %s\n", endpoint.c_str(), zmq_strerror(zmq_errno()));
return;
}
}
// Start ZMQ monitoring thread to monitor socket events
std::thread thread(&MsgqToZmq::zmqMonitorThread, this);
// Main loop for processing messages
while (!do_exit) {
{
std::unique_lock lk(mutex);
cv.wait(lk, [this]() { return do_exit || !sub2pub.empty(); });
if (do_exit) break;
for (auto sub_sock : msgq_poller->poll(100)) {
// Process messages for each socket
ZMQPubSocket *pub_sock = sub2pub.at(sub_sock);
for (int i = 0; i < MAX_MESSAGES_PER_SOCKET; ++i) {
auto msg = std::unique_ptr<Message>(sub_sock->receive(true));
if (!msg) break;
while (pub_sock->sendMessage(msg.get()) == -1) {
if (errno != EINTR) break;
}
}
}
}
util::sleep_for(1); // Give zmqMonitorThread a chance to acquire the mutex
}
thread.join();
}
void MsgqToZmq::zmqMonitorThread() {
std::vector<zmq_pollitem_t> pollitems;
// Set up ZMQ monitor for each pub socket
for (int i = 0; i < socket_pairs.size(); ++i) {
std::string addr = "inproc://op-bridge-monitor-" + std::to_string(i);
zmq_socket_monitor(socket_pairs[i].pub_sock->sock, addr.c_str(), ZMQ_EVENT_ACCEPTED | ZMQ_EVENT_DISCONNECTED);
void *monitor_socket = zmq_socket(zmq_context->getRawContext(), ZMQ_PAIR);
zmq_connect(monitor_socket, addr.c_str());
pollitems.emplace_back(zmq_pollitem_t{.socket = monitor_socket, .events = ZMQ_POLLIN});
}
while (!do_exit) {
int ret = zmq_poll(pollitems.data(), pollitems.size(), 1000);
if (ret < 0) {
if (errno == EINTR) {
// Due to frequent EINTR signals from msgq, introduce a brief delay (200 ms)
// to reduce CPU usage during retry attempts.
util::sleep_for(200);
}
continue;
}
for (int i = 0; i < pollitems.size(); ++i) {
if (pollitems[i].revents & ZMQ_POLLIN) {
// First frame in message contains event number and value
std::string frame = recv_zmq_msg(pollitems[i].socket);
if (frame.empty()) continue;
uint16_t event_type = *(uint16_t *)(frame.data());
// Second frame in message contains event address
frame = recv_zmq_msg(pollitems[i].socket);
if (frame.empty()) continue;
std::unique_lock lk(mutex);
auto &pair = socket_pairs[i];
if (event_type & ZMQ_EVENT_ACCEPTED) {
printf("socket [%s] connected\n", pair.endpoint.c_str());
if (++pair.connected_clients == 1) {
// Create new MSGQ subscriber socket and map to ZMQ publisher
pair.sub_sock = std::make_unique<MSGQSubSocket>();
size_t queue_size = services.at(pair.endpoint).queue_size;
pair.sub_sock->connect(msgq_context.get(), pair.endpoint, "127.0.0.1", false, true, queue_size);
sub2pub[pair.sub_sock.get()] = pair.pub_sock.get();
registerSockets();
}
} else if (event_type & ZMQ_EVENT_DISCONNECTED) {
printf("socket [%s] disconnected\n", pair.endpoint.c_str());
if (pair.connected_clients == 0 || --pair.connected_clients == 0) {
// Remove MSGQ subscriber socket from mapping and reset it
sub2pub.erase(pair.sub_sock.get());
pair.sub_sock.reset(nullptr);
registerSockets();
}
}
cv.notify_one();
}
}
}
// Clean up monitor sockets
for (int i = 0; i < pollitems.size(); ++i) {
zmq_socket_monitor(socket_pairs[i].pub_sock->sock, nullptr, 0);
zmq_close(pollitems[i].socket);
}
cv.notify_one();
}
void MsgqToZmq::registerSockets() {
msgq_poller = std::make_unique<MSGQPoller>();
for (const auto &socket_pair : socket_pairs) {
if (socket_pair.sub_sock) {
msgq_poller->registerSocket(socket_pair.sub_sock.get());
}
}
}

37
iqpilot/cereal/messaging/msgq_to_zmq.h Normal file
View File

@@ -0,0 +1,37 @@
#pragma once
#include <condition_variable>
#include <map>
#include <memory>
#include <mutex>
#include <string>
#include <vector>
#define private public
#include "msgq/impl_msgq.h"
#include "msgq/impl_zmq.h"
class MsgqToZmq {
public:
MsgqToZmq() {}
void run(const std::vector<std::string> &endpoints, const std::string &ip);
protected:
void registerSockets();
void zmqMonitorThread();
struct SocketPair {
std::string endpoint;
std::unique_ptr<ZMQPubSocket> pub_sock;
std::unique_ptr<MSGQSubSocket> sub_sock;
int connected_clients = 0;
};
std::unique_ptr<MSGQContext> msgq_context;
std::unique_ptr<ZMQContext> zmq_context;
std::mutex mutex;
std::condition_variable cv;
std::unique_ptr<MSGQPoller> msgq_poller;
std::map<SubSocket *, ZMQPubSocket *> sub2pub;
std::vector<SocketPair> socket_pairs;
};

211
iqpilot/cereal/messaging/socketmaster.cc Normal file
View File

@@ -0,0 +1,211 @@
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string>
#include <mutex>
#include "cereal/services.h"
#include "cereal/messaging/messaging.h"
const bool SIMULATION = (getenv("SIMULATION") != nullptr) && (std::string(getenv("SIMULATION")) == "1");
static inline bool inList(const std::vector<const char *> &list, const char *value) {
for (auto &v : list) {
if (strcmp(value, v) == 0) return true;
}
return false;
}
class MessageContext {
public:
MessageContext() : ctx_(nullptr) {}
~MessageContext() { delete ctx_; }
inline Context *context() {
std::call_once(init_flag, [=]() { ctx_ = Context::create(); });
return ctx_;
}
private:
Context *ctx_;
std::once_flag init_flag;
};
MessageContext message_context;
struct SubMaster::SubMessage {
std::string name;
SubSocket *socket = nullptr;
float freq = 0.0f;
bool updated = false, alive = false, valid = false, ignore_alive;
uint64_t rcv_time = 0, rcv_frame = 0;
void *allocated_msg_reader = nullptr;
bool is_polled = false;
capnp::FlatArrayMessageReader *msg_reader = nullptr;
AlignedBuffer aligned_buf;
cereal::Event::Reader event;
};
SubMaster::SubMaster(const std::vector<const char *> &service_list, const std::vector<const char *> &poll,
const char *address, const std::vector<const char *> &ignore_alive) {
poller_ = Poller::create();
for (auto name : service_list) {
if (services.count(std::string(name)) == 0) {
fprintf(stderr, "SubMaster: unknown service '%s', skipping subscription\n", name);
continue;
}
service serv = services.at(std::string(name));
SubSocket *socket = SubSocket::create(message_context.context(), name, address ? address : "127.0.0.1", true, true, serv.queue_size);
assert(socket != 0);
bool is_polled = inList(poll, name) || poll.empty();
if (is_polled) poller_->registerSocket(socket);
SubMessage *m = new SubMessage{
.name = name,
.socket = socket,
.freq = serv.frequency,
.ignore_alive = inList(ignore_alive, name),
.allocated_msg_reader = malloc(sizeof(capnp::FlatArrayMessageReader)),
.is_polled = is_polled};
m->msg_reader = new (m->allocated_msg_reader) capnp::FlatArrayMessageReader({});
messages_[socket] = m;
services_[name] = m;
}
}
void SubMaster::update(int timeout) {
for (auto &kv : messages_) kv.second->updated = false;
auto sockets = poller_->poll(timeout);
// add non-polled sockets for non-blocking receive
for (auto &kv : messages_) {
SubMessage *m = kv.second;
SubSocket *s = kv.first;
if (!m->is_polled) sockets.push_back(s);
}
uint64_t current_time = nanos_since_boot();
std::vector<std::pair<std::string, cereal::Event::Reader>> messages;
for (auto s : sockets) {
Message *msg = s->receive(true);
if (msg == nullptr) continue;
SubMessage *m = messages_.at(s);
m->msg_reader->~FlatArrayMessageReader();
capnp::ReaderOptions options;
options.traversalLimitInWords = kj::maxValue; // Don't limit
m->msg_reader = new (m->allocated_msg_reader) capnp::FlatArrayMessageReader(m->aligned_buf.align(msg), options);
delete msg;
messages.push_back({m->name, m->msg_reader->getRoot<cereal::Event>()});
}
update_msgs(current_time, messages);
}
void SubMaster::update_msgs(uint64_t current_time, const std::vector<std::pair<std::string, cereal::Event::Reader>> &messages){
if (++frame == UINT64_MAX) frame = 1;
for (auto &kv : messages) {
auto m_find = services_.find(kv.first);
if (m_find == services_.end()){
continue;
}
SubMessage *m = m_find->second;
m->event = kv.second;
m->updated = true;
m->rcv_time = current_time;
m->rcv_frame = frame;
m->valid = m->event.getValid();
if (SIMULATION) m->alive = true;
}
if (!SIMULATION) {
for (auto &kv : messages_) {
SubMessage *m = kv.second;
m->alive = (m->freq <= (1e-5) || ((current_time - m->rcv_time) * (1e-9)) < (10.0 / m->freq));
}
}
}
bool SubMaster::all_(const std::vector<const char *> &service_list, bool valid, bool alive) {
int found = 0;
for (auto &kv : messages_) {
SubMessage *m = kv.second;
if (service_list.size() == 0 || inList(service_list, m->name.c_str())) {
found += (!valid || m->valid) && (!alive || (m->alive || m->ignore_alive));
}
}
return service_list.size() == 0 ? found == messages_.size() : found == service_list.size();
}
void SubMaster::drain() {
while (true) {
auto polls = poller_->poll(0);
if (polls.size() == 0)
break;
for (auto sock : polls) {
Message *msg = sock->receive(true);
delete msg;
}
}
}
bool SubMaster::updated(const char *name) const {
return services_.at(name)->updated;
}
bool SubMaster::alive(const char *name) const {
return services_.at(name)->alive;
}
bool SubMaster::valid(const char *name) const {
return services_.at(name)->valid;
}
uint64_t SubMaster::rcv_frame(const char *name) const {
return services_.at(name)->rcv_frame;
}
uint64_t SubMaster::rcv_time(const char *name) const {
return services_.at(name)->rcv_time;
}
cereal::Event::Reader &SubMaster::operator[](const char *name) const {
return services_.at(name)->event;
}
SubMaster::~SubMaster() {
delete poller_;
for (auto &kv : messages_) {
SubMessage *m = kv.second;
m->msg_reader->~FlatArrayMessageReader();
free(m->allocated_msg_reader);
delete m->socket;
delete m;
}
}
PubMaster::PubMaster(const std::vector<const char *> &service_list) {
for (auto name : service_list) {
if (services.count(name) == 0) {
fprintf(stderr, "PubMaster: unknown service '%s', skipping publisher\n", name);
continue;
}
service serv = services.at(std::string(name));
PubSocket *socket = PubSocket::create(message_context.context(), name, true, serv.queue_size);
assert(socket);
sockets_[name] = socket;
}
}
int PubMaster::send(const char *name, MessageBuilder &msg) {
auto bytes = msg.toBytes();
return send(name, bytes.begin(), bytes.size());
}
PubMaster::~PubMaster() {
for (auto s : sockets_) delete s.second;
}

0
iqpilot/cereal/messaging/tests/__init__.py Normal file
View File

186
iqpilot/cereal/messaging/tests/test_messaging.py Normal file
View File

@@ -0,0 +1,186 @@
import os
import capnp
import numbers
import random
import threading
import time
from parameterized import parameterized
from iqpilot.cereal import log, car
import iqpilot.cereal.messaging as messaging
from iqpilot.cereal.services import SERVICE_LIST
events = [evt for evt in log.Event.schema.union_fields if evt in SERVICE_LIST.keys()]
def random_sock():
return random.choice(events)
def random_socks(num_socks=10):
return list({random_sock() for _ in range(num_socks)})
def random_bytes(length=1000):
return bytes([random.randrange(0xFF) for _ in range(length)])
def zmq_sleep(t=1):
if "ZMQ" in os.environ:
time.sleep(t)
# TODO: this should take any capnp struct and returrn a msg with random populated data
def random_carstate():
fields = ["vEgo", "aEgo", "brake", "steeringAngleDeg"]
msg = messaging.new_message("carState")
cs = msg.carState
for f in fields:
setattr(cs, f, random.random() * 10)
return msg
# TODO: this should compare any capnp structs
def assert_carstate(cs1, cs2):
for f in car.CarState.schema.non_union_fields:
# TODO: check all types
val1, val2 = getattr(cs1, f), getattr(cs2, f)
if isinstance(val1, numbers.Number):
assert val1 == val2, f"{f}: sent '{val1}' vs recvd '{val2}'"
def delayed_send(delay, sock, dat):
def send_func():
sock.send(dat)
threading.Timer(delay, send_func).start()
class TestMessaging:
def setUp(self):
# TODO: ZMQ tests are too slow; all sleeps will need to be
# replaced with logic to block on the necessary condition
assert "ZMQ" not in os.environ
# ZMQ pub socket takes too long to die
# sleep to prevent multiple publishers error between tests
zmq_sleep()
@parameterized.expand(events)
def test_new_message(self, evt):
try:
msg = messaging.new_message(evt)
except capnp.lib.capnp.KjException:
msg = messaging.new_message(evt, random.randrange(200))
assert (time.monotonic() - msg.logMonoTime) < 0.1
assert not msg.valid
assert evt == msg.which()
@parameterized.expand(events)
def test_pub_sock(self, evt):
messaging.pub_sock(evt)
@parameterized.expand(events)
def test_sub_sock(self, evt):
messaging.sub_sock(evt)
@parameterized.expand([
(messaging.drain_sock, capnp._DynamicStructReader),
(messaging.drain_sock_raw, bytes),
])
def test_drain_sock(self, func, expected_type):
sock = "carState"
pub_sock = messaging.pub_sock(sock)
sub_sock = messaging.sub_sock(sock, timeout=1000)
zmq_sleep()
# no wait and no msgs in queue
msgs = func(sub_sock)
assert isinstance(msgs, list)
assert len(msgs) == 0
# no wait but msgs are queued up
num_msgs = random.randrange(3, 10)
for _ in range(num_msgs):
pub_sock.send(messaging.new_message(sock).to_bytes())
time.sleep(0.1)
msgs = func(sub_sock)
assert isinstance(msgs, list)
assert all(isinstance(msg, expected_type) for msg in msgs)
assert len(msgs) == num_msgs
def test_recv_sock(self):
sock = "carState"
pub_sock = messaging.pub_sock(sock)
sub_sock = messaging.sub_sock(sock, timeout=100)
zmq_sleep()
# no wait and no msg in queue, socket should timeout
recvd = messaging.recv_sock(sub_sock)
assert recvd is None
# no wait and one msg in queue
msg = random_carstate()
pub_sock.send(msg.to_bytes())
time.sleep(0.01)
recvd = messaging.recv_sock(sub_sock)
assert isinstance(recvd, capnp._DynamicStructReader)
# https://github.com/python/mypy/issues/13038
assert_carstate(msg.carState, recvd.carState)
def test_recv_one(self):
sock = "carState"
pub_sock = messaging.pub_sock(sock)
sub_sock = messaging.sub_sock(sock, timeout=1000)
zmq_sleep()
# no msg in queue, socket should timeout
recvd = messaging.recv_one(sub_sock)
assert recvd is None
# one msg in queue
msg = random_carstate()
pub_sock.send(msg.to_bytes())
recvd = messaging.recv_one(sub_sock)
assert isinstance(recvd, capnp._DynamicStructReader)
assert_carstate(msg.carState, recvd.carState)
def test_recv_one_or_none(self):
sock = "carState"
pub_sock = messaging.pub_sock(sock)
sub_sock = messaging.sub_sock(sock)
zmq_sleep()
# no msg in queue, socket shouldn't block
recvd = messaging.recv_one_or_none(sub_sock)
assert recvd is None
# one msg in queue
msg = random_carstate()
pub_sock.send(msg.to_bytes())
recvd = messaging.recv_one_or_none(sub_sock)
assert isinstance(recvd, capnp._DynamicStructReader)
assert_carstate(msg.carState, recvd.carState)
def test_recv_one_retry(self):
sock = "carState"
sock_timeout = 0.1
pub_sock = messaging.pub_sock(sock)
sub_sock = messaging.sub_sock(sock, timeout=round(sock_timeout*1000))
zmq_sleep()
# this test doesn't work with ZMQ since multiprocessing interrupts it
if "ZMQ" not in os.environ:
# wait 5 socket timeouts and make sure it's still retrying
result = []
thread = threading.Thread(target=lambda: result.append(messaging.recv_one_retry(sub_sock)))
thread.start()
time.sleep(sock_timeout*5)
assert thread.is_alive()
msg = random_carstate()
pub_sock.send(msg.to_bytes())
thread.join(timeout=1)
assert not thread.is_alive()
assert_carstate(msg.carState, result[0].carState)
# wait 5 socket timeouts before sending
msg = random_carstate()
start_time = time.monotonic()
delayed_send(sock_timeout*5, pub_sock, msg.to_bytes())
recvd = messaging.recv_one_retry(sub_sock)
assert (time.monotonic() - start_time) >= sock_timeout*5
assert isinstance(recvd, capnp._DynamicStructReader)
assert_carstate(msg.carState, recvd.carState)

161
iqpilot/cereal/messaging/tests/test_pub_sub_master.py Normal file
View File

@@ -0,0 +1,161 @@
import random
import time
from typing import Sized, cast
import iqpilot.cereal.messaging as messaging
from iqpilot.cereal.messaging.tests.test_messaging import events, random_sock, random_socks, \
random_bytes, random_carstate, assert_carstate, \
zmq_sleep
from iqpilot.cereal.services import SERVICE_LIST
from iqpilot.common.timeout import Timeout
class TestSubMaster:
def setup_method(self):
# ZMQ pub socket takes too long to die
# sleep to prevent multiple publishers error between tests
zmq_sleep(3)
def test_init(self):
sm = messaging.SubMaster(events)
for p in [sm.updated, sm.recv_time, sm.recv_frame, sm.alive,
sm.sock, sm.data, sm.logMonoTime, sm.valid]:
assert len(cast(Sized, p)) == len(events)
def test_init_state(self):
socks = random_socks()
sm = messaging.SubMaster(socks)
assert sm.frame == -1
assert not any(sm.updated.values())
assert not any(sm.seen.values())
on_demand = {s: SERVICE_LIST[s].frequency <= 1e-5 for s in sm.services}
assert all(sm.alive[s] == sm.valid[s] == sm.freq_ok[s] == on_demand[s] for s in sm.services)
assert all(t == 0. for t in sm.recv_time.values())
assert all(f == 0 for f in sm.recv_frame.values())
assert all(t == 0 for t in sm.logMonoTime.values())
for p in [sm.updated, sm.recv_time, sm.recv_frame, sm.alive,
sm.sock, sm.data, sm.logMonoTime, sm.valid]:
assert len(cast(Sized, p)) == len(socks)
def test_getitem(self):
sock = "carState"
pub_sock = messaging.pub_sock(sock)
sm = messaging.SubMaster([sock,])
zmq_sleep()
msg = random_carstate()
pub_sock.send(msg.to_bytes())
sm.update(1000)
assert_carstate(msg.carState, sm[sock])
# TODO: break this test up to individually test SubMaster.update and SubMaster.update_msgs
def test_update(self):
sock = "carState"
pub_sock = messaging.pub_sock(sock)
sm = messaging.SubMaster([sock,])
zmq_sleep()
for i in range(10):
msg = messaging.new_message(sock)
pub_sock.send(msg.to_bytes())
sm.update(1000)
assert sm.frame == i
assert all(sm.updated.values())
def test_update_timeout(self):
sock = random_sock()
sm = messaging.SubMaster([sock,])
timeout = 100
start_time = time.monotonic()
with Timeout(2):
sm.update(timeout)
t = time.monotonic() - start_time
assert t >= timeout/1000.
assert not any(sm.updated.values())
def test_avg_frequency_checks(self):
for poll in (True, False):
sm = messaging.SubMaster(["modelV2", "carParams", "carState", "cameraOdometry", "extrinsicsCalibration"],
poll=("modelV2" if poll else None),
frequency=(20. if not poll else None))
checks = {
"carState": (20, 20),
"modelV2": (20, 20 if poll else 10),
"cameraOdometry": (20, 10),
"extrinsicsCalibration": (4, 4),
"carParams": (None, None),
"userBookmark": (None, None),
}
for service, (max_freq, min_freq) in checks.items():
if max_freq is not None:
assert sm._check_avg_freq(service)
assert sm.freq_tracker[service].max_freq == max_freq*1.2
assert sm.freq_tracker[service].min_freq == min_freq*0.8
else:
assert not sm._check_avg_freq(service)
def test_alive(self):
pass
def test_ignore_alive(self):
pass
def test_valid(self):
pass
# SubMaster should always conflate
def test_conflate(self):
sock = "carState"
pub_sock = messaging.pub_sock(sock)
sm = messaging.SubMaster([sock,])
n = 10
for i in range(n+1):
msg = messaging.new_message(sock)
msg.carState.vEgo = i
pub_sock.send(msg.to_bytes())
time.sleep(0.01)
sm.update(1000)
assert sm[sock].vEgo == n
class TestPubMaster:
def setup_method(self):
# ZMQ pub socket takes too long to die
# sleep to prevent multiple publishers error between tests
zmq_sleep(3)
def test_init(self):
messaging.PubMaster(events)
def test_send(self):
socks = random_socks()
pm = messaging.PubMaster(socks)
sub_socks = {s: messaging.sub_sock(s, conflate=True, timeout=1000) for s in socks}
zmq_sleep()
# PubMaster accepts either a capnp msg builder or bytes
for capnp in [True, False]:
for i in range(100):
sock = socks[i % len(socks)]
if capnp:
try:
msg = messaging.new_message(sock)
except Exception:
msg = messaging.new_message(sock, random.randrange(50))
else:
msg = random_bytes()
pm.send(sock, msg)
recvd = sub_socks[sock].receive()
if capnp:
msg.clear_write_flag()
msg = msg.to_bytes()
assert msg == recvd, i

27
iqpilot/cereal/messaging/tests/test_services.py Normal file
View File

@@ -0,0 +1,27 @@
import os
import tempfile
from typing import Dict
from parameterized import parameterized
from iqpilot.cereal import log
import iqpilot.cereal.services as services
from iqpilot.cereal.services import SERVICE_LIST
class TestServices:
@parameterized.expand(SERVICE_LIST.keys())
def test_services(self, s):
service = SERVICE_LIST[s]
assert service.frequency <= 104
assert service.decimation != 0
def test_generated_header(self):
with tempfile.NamedTemporaryFile(suffix=".h") as f:
ret = os.system(f"python3 {services.__file__} > {f.name} && clang++ {f.name} -std=c++11")
assert ret == 0, "generated services header is not valid C"
def test_all_services_exist_in_log_union(self):
event_fields = set(log.Event.schema.union_fields)
missing = sorted(s for s in SERVICE_LIST if s not in event_fields)
assert not missing, f"services missing from log.capnp Event union: {missing}"

222
iqpilot/cereal/messaging/tests/validate_sp_cereal_upstream.py Executable file
View File

@@ -0,0 +1,222 @@
#!/usr/bin/env python3
import argparse
import sys
from typing import Any, List, Tuple
DEBUG = False
def print_debug(string: str) -> None:
if DEBUG:
print(string)
def create_schema_instance(struct: Any, prop: Tuple[str, Any]) -> Any:
"""
Create a new instance of a schema type, handling different field types.
Args:
struct: The Cap'n Proto schema structure
prop: A tuple containing the field name and field metadata
Returns:
A new initialized schema instance
"""
struct_instance = struct.new_message()
field_name, field_metadata = prop
try:
field_type = field_metadata.proto.slot.type.which()
# Initialize different types of fields
if field_type in ('list', 'text', 'data'):
struct_instance.init(field_name, 1)
print_debug(f"Initialized list/text/data field: {field_name}")
elif field_type in ('struct', 'object'):
struct_instance.init(field_name)
print_debug(f"Initialized struct/object field: {field_name}")
return struct_instance
except Exception as e:
print(f"Error creating instance for {field_name}: {e}")
return None
def get_schema_fields(schema_struct: Any) -> List[Tuple[str, Any]]:
"""
Retrieve all fields from a given schema structure.
Args:
schema_struct: The Cap'n Proto schema structure
Returns:
A list of field names and their metadata
"""
try:
# Get all fields from the schema
schema_fields = list(schema_struct.schema.fields.items())
print_debug("Discovered schema fields:")
for field_name, field_metadata in schema_fields:
print_debug(f"- {field_name}")
return schema_fields
except Exception as e:
print(f"Error retrieving schema fields: {e}")
return []
def generate_schema_instances(schema_struct: Any) -> List[Any]:
"""
Generate instances for all fields in a given schema.
Args:
schema_struct: The Cap'n Proto schema structure
Returns:
A list of schema instances
"""
schema_fields = get_schema_fields(schema_struct)
instances = []
for field_prop in schema_fields:
try:
instance = create_schema_instance(schema_struct, field_prop)
if instance is not None:
instances.append(instance)
except Exception as e:
print(f"Skipping field due to error: {e}")
print(f"Generated {len(instances)} schema instances")
return instances
def persist_instances(instances: List[Any], filename: str) -> None:
"""
Write schema instances to a binary file.
Args:
instances: List of schema instances
filename: Output file path
"""
try:
with open(filename, 'wb') as f:
for instance in instances:
f.write(instance.to_bytes())
print(f"Successfully wrote {len(instances)} instances to {filename}")
except Exception as e:
print(f"Error persisting instances: {e}")
sys.exit(1)
def read_instances(filename: str, schema_type: Any) -> List[Any]:
"""
Read schema instances from a binary file.
Args:
filename: Input file path
schema_type: The schema type to use for reading
Returns:
A list of read schema instances
"""
try:
with open(filename, 'rb') as f:
data = f.read()
instances = list(schema_type.read_multiple_bytes(data))
print(f"Read {len(instances)} instances from {filename}")
return instances
except Exception as e:
print(f"Error reading instances: {e}")
sys.exit(1)
def compare_schemas(original_instances: List[Any], read_instances: List[Any]) -> bool:
"""
Compare original and read-back instances to detect potential breaking changes.
Args:
original_instances: List of originally generated instances
read_instances: List of instances read back from file
Returns:
Boolean indicating whether schemas appear compatible
"""
if len(original_instances) != len(read_instances):
print("❌ Schema Compatibility Warning: Instance count mismatch")
return False
compatible = True
for struct in read_instances:
try:
getattr(struct, struct.which()) # Attempting to access the field to validate readability
except Exception as e:
print(f"❌ Structural change detected: {struct.which()} is not readable.\nFull error: {e}")
compatible = False
return compatible
def main():
"""
CLI entry point for schema compatibility testing.
"""
# Setup argument parser
parser = argparse.ArgumentParser(
description='Cap\'n Proto Schema Compatibility Testing Tool',
epilog='Test schema compatibility by generating and reading back instances.'
)
# Add mutually exclusive group for generation or reading mode
mode_group = parser.add_mutually_exclusive_group(required=True)
mode_group.add_argument('-g', '--generate', action='store_true',
help='Generate schema instances')
mode_group.add_argument('-r', '--read', action='store_true',
help='Read and validate schema instances')
# Common arguments
parser.add_argument('-f', '--file',
default='schema_instances.bin',
help='Output/input binary file (default: schema_instances.bin)')
# Parse arguments
args = parser.parse_args()
# Import the schema dynamically
try:
from iqpilot.cereal import log
schema_type = log.Event
except ImportError:
print("Error: Unable to import schema. Ensure 'cereal' is installed.")
sys.exit(1)
# Execute based on mode
if args.generate:
print("🔧 Generating Schema Instances")
instances = generate_schema_instances(schema_type)
persist_instances(instances, args.file)
print("✅ Instance generation complete")
elif args.read:
print("🔍 Reading and Validating Schema Instances")
generated_instances = generate_schema_instances(schema_type)
read_back_instances = read_instances(args.file, schema_type)
# Compare schemas
if compare_schemas(generated_instances, read_back_instances):
print("✅ Schema Compatibility: No breaking changes detected")
sys.exit(0)
else:
print("❌ Potential Schema Breaking Changes Detected")
sys.exit(1)
if __name__ == "__main__":
main()

159
iqpilot/cereal/services.py Executable file
View File

@@ -0,0 +1,159 @@
#!/usr/bin/env python3
from enum import IntEnum
from typing import Optional
# TODO: this should be automatically determined using the capnp schema
class QueueSize(IntEnum):
BIG = 10 * 1024 * 1024 # 10MB - video frames, large AI outputs
MEDIUM = 2 * 1024 * 1024 # 2MB - high freq (CAN), livestream
SMALL = 250 * 1024 # 250KB - most services
class Service:
def __init__(self, should_log: bool, frequency: float, decimation: Optional[int] = None,
queue_size: QueueSize = QueueSize.SMALL):
self.should_log = should_log
self.frequency = frequency
self.decimation = decimation
self.queue_size = queue_size
_services: dict[str, tuple] = {
# service: (should_log, frequency, qlog decimation (optional))
# note: the "EncodeIdx" packets will still be in the log
"gyroscope": (True, 104., 104),
"accelerometer": (True, 104., 104),
"magnetometer": (True, 25.),
"lightSensor": (True, 100., 100),
"temperatureSensor": (True, 2., 200),
"gpsNMEA": (True, 9.),
"deviceState": (True, 2., 1),
"touch": (True, 20., 1),
"can": (True, 100., 2053, QueueSize.BIG), # decimation gives ~3 msgs in a full segment
"controlsState": (True, 100., 10, QueueSize.MEDIUM),
"selfdriveState": (True, 100., 10),
"pandaStates": (True, 10., 1),
"peripheralState": (True, 2., 1),
"radarState": (True, 20., 5),
"roadEncodeIdx": (False, 20., 1),
"radarTracks": (True, 20.),
"sendcan": (True, 100., 139, QueueSize.MEDIUM),
"logMessage": (True, 0., None, QueueSize.MEDIUM),
"errorLogMessage": (True, 0., 1, QueueSize.MEDIUM),
"extrinsicsCalibration": (True, 4., 4),
"lateralTorqueParameters": (True, 4., 1),
"lateralDelay": (True, 4., 1),
"androidLog": (True, 0.),
"carState": (True, 100., 10),
"carControl": (True, 100., 10),
"carOutput": (True, 100., 10),
"longitudinalPlan": (True, 20., 10),
"lateralManeuverPlan": (True, 20.),
"driverAssistance": (True, 20., 20),
"procLog": (True, 0.5, 15, QueueSize.MEDIUM),
"gpsLocationExternal": (True, 10., 10),
"gpsLocation": (True, 1., 1),
"ubloxGnss": (True, 10.),
"qcomGnss": (True, 2.),
"gnssMeasurements": (True, 10., 10),
"clocks": (True, 0.1, 1),
"ubloxRaw": (True, 20.),
"deviceMotion": (True, 20., 4),
"vehicleParameters": (True, 20., 5),
"cameraOdometry": (True, 20., 10),
"thumbnail": (True, 1 / 60., 1),
"onroadEvents": (True, 1., 1),
"carParams": (True, 0.02, 1),
"roadCameraState": (True, 20., 20),
"driverCameraState": (True, 20., 20),
"driverEncodeIdx": (False, 20., 1),
"driverStateV2": (True, 20., 10),
"driverMonitoringState": (True, 20., 10),
"wideRoadEncodeIdx": (False, 20., 1),
"wideRoadCameraState": (True, 20., 20),
"drivingModelData": (True, 20., 10),
"modelV2": (True, 20., None, QueueSize.MEDIUM),
"managerState": (True, 2., 1),
"uploaderState": (True, 0., 1),
"navInstruction": (True, 1., 10),
"navRoute": (True, 0.),
"navThumbnail": (True, 0.),
"qRoadEncodeIdx": (False, 20.),
"userBookmark": (True, 0., 1),
"soundPressure": (True, 10., 10),
"rawAudioData": (False, 20.),
"webrtcAudioData": (False, 50.),
"bookmarkButton": (True, 0., 1),
"audioFeedback": (True, 0., 1),
"roadEncodeData": (False, 20., None, QueueSize.BIG),
"driverEncodeData": (False, 20., None, QueueSize.BIG),
"wideRoadEncodeData": (False, 20., None, QueueSize.BIG),
"qRoadEncodeData": (False, 20., None, QueueSize.BIG),
# iqpilot
"iqModelManager": (False, 1., 1, QueueSize.MEDIUM),
"backupManagerK3": (False, 1., 1),
"iqCarParams": (True, 0.02, 1),
"iqCarControl": (True, 100., 10),
"iqCarState": (True, 100., 10),
"iqLiveData": (True, 1., 1),
"iqConstructionZone": (True, 2., 2),
"iqVehicleTracks": (True, 4., 4),
"iqEnvironment": (True, 4., 4),
"mapdOut": (True, 20., 20, QueueSize.MEDIUM),
"mapdExtendedOut": (False, 1., -1, QueueSize.MEDIUM),
"mapdIn": (False, 1., -1, QueueSize.MEDIUM),
"iqNavState": (True, 5., 10),
"iqRoadIncidentFeed": (True, 0.2, 1),
"iqNavRenderState": (True, 5., 10, QueueSize.MEDIUM),
"iqState": (True, 100., 10),
"iqPlan": (True, 20., 10),
"iqOnroadEvents": (True, 1., 1),
"iqDriveModelData": (True, 20., None, QueueSize.MEDIUM),
"iqLiveLocation": (True, 20.),
"liveLocationKalman": (True, 20.),
"iqPerfTrace": (True, 0., 1, QueueSize.SMALL),
# debug
"uiDebug": (True, 0., 1),
"testJoystick": (True, 0.),
"alertDebug": (True, 20., 5),
"livestreamWideRoadEncodeIdx": (False, 20.),
"livestreamRoadEncodeIdx": (False, 20.),
"livestreamDriverEncodeIdx": (False, 20.),
"livestreamWideRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
"livestreamRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
"livestreamDriverEncodeData": (False, 20., None, QueueSize.MEDIUM),
"customReservedRawData0": (True, 0.),
"customReservedRawData1": (True, 0.),
"customReservedRawData2": (True, 0.),
}
SERVICE_LIST = {name: Service(*vals) for
idx, (name, vals) in enumerate(_services.items())}
def build_header():
h = ""
h += "/* THIS IS AN AUTOGENERATED FILE, PLEASE EDIT services.py */\n"
h += "#ifndef __SERVICES_H\n"
h += "#define __SERVICES_H\n"
h += "#include <map>\n"
h += "#include <string>\n"
h += "struct service { std::string name; bool should_log; float frequency; int decimation; size_t queue_size; };\n"
h += "static std::map<std::string, service> services = {\n"
for k, v in SERVICE_LIST.items():
should_log = "true" if v.should_log else "false"
decimation = -1 if v.decimation is None else v.decimation
h += ' { "%s", {"%s", %s, %f, %d, %d}},\n' % \
(k, k, should_log, v.frequency, decimation, v.queue_size)
h += "};\n"
h += "#endif\n"
return h
if __name__ == "__main__":
print(build_header())

8
iqpilot/cereal/visionipc.py Normal file
View File

@@ -0,0 +1,8 @@
from enum import IntEnum
class VisionStreamType(IntEnum):
VISION_STREAM_ROAD = 0
VISION_STREAM_DRIVER = 1
VISION_STREAM_WIDE_ROAD = 2
VISION_STREAM_MAP = 3

10
iqpilot/cereal/visionstream.h Normal file
View File

@@ -0,0 +1,10 @@
#pragma once
#include "msgq/visionipc/visionbuf.h"
enum VisionStreamValues : VisionStreamType {
VISION_STREAM_ROAD = 0,
VISION_STREAM_DRIVER = 1,
VISION_STREAM_WIDE_ROAD = 2,
VISION_STREAM_MAP = 3,
};