IQ.Pilot Release Commit @ b6534c0

iqpilot/selfdrive/iqmodeld/runtime/ort.py

@@ -0,0 +1,62 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+"""
+import itertools
+
+import numpy as np
+import onnx
+import onnxruntime as ort
+
+
+ORT_TYPES_TO_NP_TYPES = {
+  "tensor(float16)": np.float16,
+  "tensor(float)": np.float32,
+  "tensor(uint8)": np.uint8,
+}
+
+
+def _promote_raw_half_blob(attribute):
+  float32_values = np.frombuffer(attribute.raw_data, dtype=np.float16)
+  attribute.data_type = 1
+  attribute.raw_data = float32_values.astype(np.float32).tobytes()
+
+
+def _rewrite_tensor_io_types(model):
+  for value_info in itertools.chain(model.graph.input, model.graph.output):
+    if value_info.type.tensor_type.elem_type == 10:
+      value_info.type.tensor_type.elem_type = 1
+
+
+def _rewrite_cast_nodes(model):
+  for node in model.graph.node:
+    if node.op_type == "Cast" and node.attribute[0].i == 10:
+      node.attribute[0].i = 1
+    for attribute in node.attribute:
+      if hasattr(attribute, "t") and attribute.t.data_type == 10:
+        _promote_raw_half_blob(attribute.t)
+
+
+def attributeproto_fp16_to_fp32(attr):
+  _promote_raw_half_blob(attr)
+
+
+def convert_fp16_to_fp32(model):
+  for initializer in model.graph.initializer:
+    if initializer.data_type == 10:
+      _promote_raw_half_blob(initializer)
+  _rewrite_tensor_io_types(model)
+  _rewrite_cast_nodes(model)
+  return model.SerializeToString()
+
+
+def _cpu_session_options():
+  options = ort.SessionOptions()
+  options.intra_op_num_threads = 4
+  options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
+  options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
+  return options
+
+
+def make_onnx_cpu_runner(model_path):
+  model_blob = convert_fp16_to_fp32(onnx.load(model_path))
+  return ort.InferenceSession(model_blob, _cpu_session_options(), providers=["CPUExecutionProvider"])