import sys import capnp from pathlib import Path NO_DISCRIMINANT = 65535 SCALAR_KINDS = { "bool": "Bool", "int8": "Int", "int16": "Int", "int32": "Int", "int64": "Int", "uint8": "UInt", "uint16": "UInt", "uint32": "UInt", "uint64": "UInt", "float32": "Float", "float64": "Float", "enum": "Enum", } NESTED_TYPE_KINDS = {"struct", "list"} IGNORED_TYPE_KINDS = {"void", "text", "data", "interface", "anyPointer"} def cxx_string(value): return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"' def accessor(prefix, name): return prefix + name[:1].upper() + name[1:] def field_type(field): if field.proto.which() == "group": return "struct" return field.proto.slot.type.which() def field_type_proto(field): return field.proto.slot.type if field.proto.which() == "slot" else None def scalar_kind(type_proto): if type_proto is None: return None return SCALAR_KINDS.get(type_proto.which()) def enum_names(schema): if schema is None: return [] names_by_ordinal = schema.enumerants if not names_by_ordinal: return [] max_ordinal = max(names_by_ordinal.values()) out = [""] * (max_ordinal + 1) for name, ordinal in names_by_ordinal.items(): out[ordinal] = name return out class Generator: def __init__(self, event_schema): self.event_schema = event_schema self.fixed_paths = [] self.event_base_slots = {} self.static_enums = [] self.tmp_index = 0 self.lines = [] self.emits_memo = {} def tmp(self, prefix): self.tmp_index += 1 return f"{prefix}_{self.tmp_index}" def add_fixed_path(self, path): slot = len(self.fixed_paths) self.fixed_paths.append(path) return slot def emit(self, indent, text=""): self.lines.append(" " * indent + text) def scalar_double_expr(self, value_expr, kind): if kind == "Bool": return f"({value_expr} ? 1.0 : 0.0)" if kind == "Enum": return f"static_cast(static_cast({value_expr}))" return f"static_cast({value_expr})" def emit_enum_capture(self, indent, path_expr, names): if not names: return names_expr = "{" + ", ".join(cxx_string(name) for name in names) + "}" self.emit(indent, f"capture_static_enum_info({path_expr}, {names_expr}, series);") def emit_node(self, indent, type_kind, type_proto, schema, expr, path, path_expr, dynamic_path): if not self.node_emits(type_kind, type_proto, schema): return kind = scalar_kind(type_proto) if kind is not None: double_expr = self.scalar_double_expr(expr, kind) if dynamic_path: if kind == "Enum": self.emit_enum_capture(indent, path_expr, enum_names(schema)) self.emit(indent, f"append_dynamic_scalar_point({path_expr}, tm, {double_expr}, series);") else: slot = self.add_fixed_path(path) names = enum_names(schema) if kind == "Enum" else [] if names: enum_index = len(self.static_enums) self.static_enums.append(names) self.emit(indent, f"append_fixed_enum_point({slot}, {enum_index}, tm, {double_expr}, series);") else: self.emit(indent, f"append_fixed_scalar_point(&series->fixed_series[{slot}], tm, {double_expr});") return if type_kind == "struct": self.emit_struct(indent, schema, expr, path, path_expr, dynamic_path) return if type_kind == "list": self.emit_list(indent, type_proto, schema, expr, path, path_expr, dynamic_path) def emit_field(self, indent, struct_schema, reader_expr, field_name, base_path, base_path_expr, dynamic_path): field = struct_schema.fields[field_name] proto = field.proto type_kind = field_type(field) type_proto = field_type_proto(field) kind = scalar_kind(type_proto) value_schema = field.schema if kind == "Enum" or type_kind in NESTED_TYPE_KINDS else None if not self.node_emits(type_kind, type_proto, value_schema): return field_path = f"{base_path}/{field_name}" field_path_expr = None if dynamic_path: field_path_var = self.tmp("path") self.emit(indent, f"const std::string {field_path_var} = {base_path_expr} + {cxx_string('/' + field_name)};") field_path_expr = field_path_var get_call = f"{reader_expr}.{accessor('get', field_name)}()" has_call = f"{reader_expr}.{accessor('has', field_name)}()" conditions = [] if proto.discriminantValue != NO_DISCRIMINANT: conditions.append(f"{reader_expr}.which() == static_cast({proto.discriminantValue})") if proto.which() == "slot" and type_kind in NESTED_TYPE_KINDS: conditions.append(has_call) if conditions: self.emit(indent, f"if ({' && '.join(conditions)}) {{") indent += 2 # Scalar getters are only consumed once. Emitting them directly avoids # thousands of single-use locals in the generated extractor. value_expr = get_call if kind is None: value_expr = self.tmp("value") self.emit(indent, f"const auto {value_expr} = {get_call};") self.emit_node(indent, type_kind, type_proto, value_schema, value_expr, field_path, field_path_expr, dynamic_path) if conditions: indent -= 2 self.emit(indent, "}") def emit_struct(self, indent, schema, reader_expr, path, path_expr, dynamic_path): if schema is None: return for field_name in schema.fieldnames: self.emit_field(indent, schema, reader_expr, field_name, path, path_expr, dynamic_path) def emit_list(self, indent, type_proto, schema, list_expr, path, path_expr, dynamic_path): elem_type = type_proto.list.elementType elem_kind = elem_type.which() if elem_kind in IGNORED_TYPE_KINDS: return base_path_var = path_expr if base_path_var is None: base_path_var = self.tmp("base_path") self.emit(indent, f"const std::string {base_path_var} = {cxx_string(path)};") elem_scalar = scalar_kind(elem_type) if elem_scalar is not None: self.emit(indent, f"if ({list_expr}.size() <= 16) {{") index_var = self.tmp("i") self.emit(indent + 2, f"for (uint {index_var} = 0; {index_var} < {list_expr}.size(); ++{index_var}) {{") item_series = self.tmp("item_series") self.emit(indent + 4, f"RouteSeries *{item_series} = ensure_list_scalar_series({base_path_var}, {index_var}, series);") if elem_scalar == "Enum": self.emit_enum_capture(indent + 4, f"{item_series}->path", enum_names(schema.elementType)) self.emit(indent + 4, f"append_fixed_scalar_point({item_series}, tm, {self.scalar_double_expr(f'{list_expr}[{index_var}]', elem_scalar)});") self.emit(indent + 2, "}") self.emit(indent, "}") return if elem_kind in {"struct", "list"}: index_var = self.tmp("i") self.emit(indent, f"for (uint {index_var} = 0; {index_var} < {list_expr}.size(); ++{index_var}) {{") item_path = self.tmp("item_path") self.emit(indent + 2, f"const std::string {item_path} = {base_path_var} + \"/\" + std::to_string({index_var});") item = self.tmp("item") self.emit(indent + 2, f"const auto {item} = {list_expr}[{index_var}];") if elem_kind == "struct": self.emit_struct(indent + 2, schema.elementType, item, path, item_path, True) else: self.emit_list(indent + 2, elem_type, schema.elementType, item, path, item_path, True) self.emit(indent, "}") def node_emits(self, type_kind, type_proto, schema, seen=frozenset()): if scalar_kind(type_proto) is not None: return True if type_kind == "struct": if schema is None: return False schema_id = int(schema.node.id) if schema_id in seen: return False if schema_id in self.emits_memo: return self.emits_memo[schema_id] next_seen = seen | {schema_id} for field_name in schema.fieldnames: field = schema.fields[field_name] ft = field_type(field) ftp = field_type_proto(field) fkind = scalar_kind(ftp) if ft in IGNORED_TYPE_KINDS: continue fschema = field.schema if fkind == "Enum" or ft in NESTED_TYPE_KINDS else None if self.node_emits(ft, ftp, fschema, next_seen): self.emits_memo[schema_id] = True return True self.emits_memo[schema_id] = False return False if type_kind == "list": if type_proto is None or schema is None: return False elem_type = type_proto.list.elementType elem_kind = elem_type.which() if elem_kind in IGNORED_TYPE_KINDS: return False if scalar_kind(elem_type) is not None: return True if elem_kind == "struct": return self.node_emits("struct", None, schema.elementType, seen) if elem_kind == "list": return self.node_emits("list", elem_type, schema.elementType, seen) return False def emit_can_special(self, indent, service_name): service_kind = "CanServiceKind::Can" if service_name == "can" else "CanServiceKind::Sendcan" self.emit(indent, f"const CanServiceKind can_service = {service_kind};") self.emit(indent, f"for (const auto &msg : event.{accessor('get', service_name)}()) {{") # IQ.Pilot patch: iqpilot's cereal CanData keeps the deprecated busTime as a flat # `busTimeDEPRECATED` field rather than upstream's `deprecated :group { busTime }`. self.emit(indent + 2, "append_can_frame(can_service, static_cast(msg.getSrc()), msg.getAddress(), msg.getBusTimeDEPRECATED(), msg.getDat(), tm, series);") # noqa: E501 self.emit(indent + 2, "if (skip_raw_can) {") self.emit(indent + 4, "const auto dat = msg.getDat();") self.emit(indent + 4, f"decode_can_frame(can_dbc, {cxx_string(service_name)}, static_cast(msg.getSrc()), msg.getAddress(), dat.begin(), dat.size(), tm, series);") # noqa: E501 self.emit(indent + 2, "}") self.emit(indent, "}") self.emit(indent, "if (skip_raw_can) {") self.emit(indent + 2, "return;") self.emit(indent, "}") def emit_event_reader(self, field_name): field = self.event_schema.fields[field_name] proto = field.proto type_kind = field_type(field) type_proto = field_type_proto(field) kind = scalar_kind(type_proto) schema = field.schema if kind == "Enum" or type_kind in NESTED_TYPE_KINDS else None valid_slot = self.add_fixed_path(f"/{field_name}/valid") self.add_fixed_path(f"/{field_name}/logMonoTime") self.add_fixed_path(f"/{field_name}/t") self.event_base_slots[proto.discriminantValue] = valid_slot emits_payload = self.node_emits(type_kind, type_proto, schema) if field_name not in {"can", "sendcan"} and not emits_payload: return None reader_name = f"append_event_{proto.discriminantValue}" needs_can = field_name in {"can", "sendcan"} header_index = len(self.lines) self.emit(0, "") if needs_can: self.emit_can_special(2, field_name) if emits_payload: payload = self.tmp("payload") self.emit(2, f"const auto {payload} = event.{accessor('get', field_name)}();") self.emit_node(2, type_kind, type_proto, schema, payload, f"/{field_name}", None, False) self.emit(0, "}") self.emit(0, "") if needs_can: signature = "const cereal::Event::Reader &event, const dbc::Database *can_dbc, bool skip_raw_can, double tm, SeriesAccumulator *series" else: signature = "const cereal::Event::Reader &event, double tm, SeriesAccumulator *series" self.lines[header_index] = f"__attribute__((noinline)) void {reader_name}({signature}) {{" return reader_name, needs_can def generate(self): self.lines = [] self.emit(0, "// Generated by openpilot/tools/jotpluggler/generate_event_extractors.py; do not edit.") self.emit(0, "") self.emit(0, "const std::vector &static_event_fixed_paths() {") self.emit(2, "static const std::vector paths = {") path_insert_at = len(self.lines) self.emit(2, "};") self.emit(2, "return paths;") self.emit(0, "}") self.emit(0, "") self.emit(0, "void capture_static_enum_info(const std::string &path, std::initializer_list names, SeriesAccumulator *series) {") self.emit(2, "if (series->enum_info.find(path) != series->enum_info.end()) {") self.emit(4, "return;") self.emit(2, "}") self.emit(2, "EnumInfo info;") self.emit(2, "info.names.reserve(names.size());") self.emit(2, "for (std::string_view name : names) {") self.emit(4, "info.names.emplace_back(name);") self.emit(2, "}") self.emit(2, "if (!info.names.empty()) {") self.emit(4, "series->enum_info.emplace(path, std::move(info));") self.emit(2, "}") self.emit(0, "}") self.emit(0, "") self.emit(0, "__attribute__((noinline)) void append_fixed_enum_point(size_t series_slot, size_t enum_index, double tm, double value, SeriesAccumulator *series);") # noqa: E501 self.emit(0, "") self.emit(0, "// Keep each event payload behind its own optimizer boundary. Combining the") self.emit(0, "// whole schema into one function creates much more code and runs slower.") event_readers = {} for field_name in self.event_schema.union_fields: event_readers[field_name] = self.emit_event_reader(field_name) self.emit(0, "static const std::initializer_list static_event_enum_names[] = {") for names in self.static_enums: names_expr = "{" + ", ".join(cxx_string(name) for name in names) + "}" self.emit(2, f"{names_expr},") self.emit(0, "};") self.emit(0, "") self.emit(0, "__attribute__((noinline)) void append_fixed_enum_point(size_t series_slot, size_t enum_index, double tm, double value, SeriesAccumulator *series) {") # noqa: E501 self.emit(2, "RouteSeries *fixed_series = &series->fixed_series[series_slot];") self.emit(2, "capture_static_enum_info(fixed_series->path, static_event_enum_names[enum_index], series);") self.emit(2, "fixed_series->times.push_back(tm);") self.emit(2, "fixed_series->values.push_back(value);") self.emit(0, "}") self.emit(0, "") self.emit(0, "bool append_event_static_reader(cereal::Event::Which which, const cereal::Event::Reader &event, const dbc::Database *can_dbc, bool skip_raw_can, double time_offset, SeriesAccumulator *series) {") # noqa: E501 self.emit(2, "const auto log_mono_time = event.getLogMonoTime();") self.emit(2, "const double tm = static_cast(log_mono_time) / 1.0e9 - time_offset;") invalid_slot = "static_cast(-1)" max_discriminant = max(self.event_base_slots) base_slots = [self.event_base_slots.get(i, invalid_slot) for i in range(max_discriminant + 1)] self.emit(2, "static constexpr size_t event_base_slots[] = {") for slot in base_slots: self.emit(4, f"{slot},") self.emit(2, "};") self.emit(2, "const size_t event_index = static_cast(which);") self.emit(2, "if (event_index >= sizeof(event_base_slots) / sizeof(event_base_slots[0])) {") self.emit(4, "return false;") self.emit(2, "}") self.emit(2, "const size_t base_slot = event_base_slots[event_index];") self.emit(2, f"if (base_slot == {invalid_slot}) {{") self.emit(4, "return false;") self.emit(2, "}") self.emit(2, "RouteSeries *base_series = &series->fixed_series[base_slot];") self.emit(2, "base_series[0].times.push_back(tm);") self.emit(2, "base_series[0].values.push_back(event.getValid() ? 1.0 : 0.0);") self.emit(2, "base_series[1].times.push_back(tm);") self.emit(2, "base_series[1].values.push_back(static_cast(log_mono_time));") self.emit(2, "base_series[2].times.push_back(tm);") self.emit(2, "base_series[2].values.push_back(tm);") self.emit(2, "switch (which) {") for field_name in self.event_schema.union_fields: field = self.event_schema.fields[field_name] self.emit(4, f"case static_cast({field.proto.discriminantValue}):") reader = event_readers[field_name] if reader is not None: if reader[1]: self.emit(6, f"{reader[0]}(event, can_dbc, skip_raw_can, tm, series);") else: self.emit(6, f"{reader[0]}(event, tm, series);") self.emit(6, "return true;") self.emit(4, "default:") self.emit(6, "return false;") self.emit(2, "}") self.emit(0, "}") path_lines = [" " + cxx_string(path) + "," for path in self.fixed_paths] self.lines[path_insert_at:path_insert_at] = path_lines return "\n".join(self.lines) + "\n" if __name__ == "__main__": if len(sys.argv) != 4: print(f"usage: {sys.argv[0]}

", file=sys.stderr) sys.exit(2) repo_root = Path(sys.argv[1]).resolve() iqdbc_root = Path(sys.argv[2]).resolve() output = Path(sys.argv[3]) capnp.remove_import_hook() log = capnp.load(str(repo_root / "iqpilot" / "cereal" / "log.capnp"), imports=[str(iqdbc_root / "car")]) generated = Generator(log.Event.schema).generate() output.parent.mkdir(parents=True, exist_ok=True) output.write_text(generated)