IQ.Pilot Release Commit @ 0798119

cereal/messaging/tests/test_messaging.py

@@ -0,0 +1,185 @@
+import os
+import capnp
+import multiprocessing
+import numbers
+import random
+import threading
+import time
+from parameterized import parameterized
+import pytest
+
+from cereal import log, car
+import cereal.messaging as messaging
+from cereal.services import SERVICE_LIST
+
+events = [evt for evt in log.Event.schema.union_fields if evt in SERVICE_LIST.keys()]
+
+def random_sock():
+  return random.choice(events)
+
+def random_socks(num_socks=10):
+  return list({random_sock() for _ in range(num_socks)})
+
+def random_bytes(length=1000):
+  return bytes([random.randrange(0xFF) for _ in range(length)])
+
+def zmq_sleep(t=1):
+  if "ZMQ" in os.environ:
+    time.sleep(t)
+
+
+# TODO: this should take any capnp struct and returrn a msg with random populated data
+def random_carstate():
+  fields = ["vEgo", "aEgo", "brake", "steeringAngleDeg"]
+  msg = messaging.new_message("carState")
+  cs = msg.carState
+  for f in fields:
+    setattr(cs, f, random.random() * 10)
+  return msg
+
+# TODO: this should compare any capnp structs
+def assert_carstate(cs1, cs2):
+  for f in car.CarState.schema.non_union_fields:
+    # TODO: check all types
+    val1, val2 = getattr(cs1, f), getattr(cs2, f)
+    if isinstance(val1, numbers.Number):
+      assert val1 == val2, f"{f}: sent '{val1}' vs recvd '{val2}'"
+
+def delayed_send(delay, sock, dat):
+  def send_func():
+    sock.send(dat)
+  threading.Timer(delay, send_func).start()
+
+
+class TestMessaging:
+  def setUp(self):
+    # TODO: ZMQ tests are too slow; all sleeps will need to be
+    # replaced with logic to block on the necessary condition
+    if "ZMQ" in os.environ:
+      pytest.skip()
+
+    # ZMQ pub socket takes too long to die
+    # sleep to prevent multiple publishers error between tests
+    zmq_sleep()
+
+  @parameterized.expand(events)
+  def test_new_message(self, evt):
+    try:
+      msg = messaging.new_message(evt)
+    except capnp.lib.capnp.KjException:
+      msg = messaging.new_message(evt, random.randrange(200))
+    assert (time.monotonic() - msg.logMonoTime) < 0.1
+    assert not msg.valid
+    assert evt == msg.which()
+
+  @parameterized.expand(events)
+  def test_pub_sock(self, evt):
+    messaging.pub_sock(evt)
+
+  @parameterized.expand(events)
+  def test_sub_sock(self, evt):
+    messaging.sub_sock(evt)
+
+  @parameterized.expand([
+    (messaging.drain_sock, capnp._DynamicStructReader),
+    (messaging.drain_sock_raw, bytes),
+  ])
+  def test_drain_sock(self, func, expected_type):
+    sock = "carState"
+    pub_sock = messaging.pub_sock(sock)
+    sub_sock = messaging.sub_sock(sock, timeout=1000)
+    zmq_sleep()
+
+    # no wait and no msgs in queue
+    msgs = func(sub_sock)
+    assert isinstance(msgs, list)
+    assert len(msgs) == 0
+
+    # no wait but msgs are queued up
+    num_msgs = random.randrange(3, 10)
+    for _ in range(num_msgs):
+      pub_sock.send(messaging.new_message(sock).to_bytes())
+    time.sleep(0.1)
+    msgs = func(sub_sock)
+    assert isinstance(msgs, list)
+    assert all(isinstance(msg, expected_type) for msg in msgs)
+    assert len(msgs) == num_msgs
+
+  def test_recv_sock(self):
+    sock = "carState"
+    pub_sock = messaging.pub_sock(sock)
+    sub_sock = messaging.sub_sock(sock, timeout=100)
+    zmq_sleep()
+
+    # no wait and no msg in queue, socket should timeout
+    recvd = messaging.recv_sock(sub_sock)
+    assert recvd is None
+
+    # no wait and one msg in queue
+    msg = random_carstate()
+    pub_sock.send(msg.to_bytes())
+    time.sleep(0.01)
+    recvd = messaging.recv_sock(sub_sock)
+    assert isinstance(recvd, capnp._DynamicStructReader)
+    # https://github.com/python/mypy/issues/13038
+    assert_carstate(msg.carState, recvd.carState)
+
+  def test_recv_one(self):
+    sock = "carState"
+    pub_sock = messaging.pub_sock(sock)
+    sub_sock = messaging.sub_sock(sock, timeout=1000)
+    zmq_sleep()
+
+    # no msg in queue, socket should timeout
+    recvd = messaging.recv_one(sub_sock)
+    assert recvd is None
+
+    # one msg in queue
+    msg = random_carstate()
+    pub_sock.send(msg.to_bytes())
+    recvd = messaging.recv_one(sub_sock)
+    assert isinstance(recvd, capnp._DynamicStructReader)
+    assert_carstate(msg.carState, recvd.carState)
+
+  @pytest.mark.xfail(condition="ZMQ" in os.environ, reason='ZMQ detected')
+  def test_recv_one_or_none(self):
+    sock = "carState"
+    pub_sock = messaging.pub_sock(sock)
+    sub_sock = messaging.sub_sock(sock)
+    zmq_sleep()
+
+    # no msg in queue, socket shouldn't block
+    recvd = messaging.recv_one_or_none(sub_sock)
+    assert recvd is None
+
+    # one msg in queue
+    msg = random_carstate()
+    pub_sock.send(msg.to_bytes())
+    recvd = messaging.recv_one_or_none(sub_sock)
+    assert isinstance(recvd, capnp._DynamicStructReader)
+    assert_carstate(msg.carState, recvd.carState)
+
+  def test_recv_one_retry(self):
+    sock = "carState"
+    sock_timeout = 0.1
+    pub_sock = messaging.pub_sock(sock)
+    sub_sock = messaging.sub_sock(sock, timeout=round(sock_timeout*1000))
+    zmq_sleep()
+
+    # this test doesn't work with ZMQ since multiprocessing interrupts it
+    if "ZMQ" not in os.environ:
+      # wait 5 socket timeouts and make sure it's still retrying
+      p = multiprocessing.Process(target=messaging.recv_one_retry, args=(sub_sock,))
+      p.start()
+      time.sleep(sock_timeout*5)
+      assert p.is_alive()
+      p.terminate()
+
+    # wait 5 socket timeouts before sending
+    msg = random_carstate()
+    start_time = time.monotonic()
+    delayed_send(sock_timeout*5, pub_sock, msg.to_bytes())
+    recvd = messaging.recv_one_retry(sub_sock)
+    assert (time.monotonic() - start_time) >= sock_timeout*5
+    assert isinstance(recvd, capnp._DynamicStructReader)
+    assert_carstate(msg.carState, recvd.carState)

cereal/messaging/tests/test_pub_sub_master.py

@@ -0,0 +1,160 @@
+import random
+import time
+from typing import Sized, cast
+
+import cereal.messaging as messaging
+from cereal.messaging.tests.test_messaging import events, random_sock, random_socks, \
+                                                  random_bytes, random_carstate, assert_carstate, \
+                                                  zmq_sleep
+from cereal.services import SERVICE_LIST
+
+
+class TestSubMaster:
+
+  def setup_method(self):
+    # ZMQ pub socket takes too long to die
+    # sleep to prevent multiple publishers error between tests
+    zmq_sleep(3)
+
+  def test_init(self):
+    sm = messaging.SubMaster(events)
+    for p in [sm.updated, sm.recv_time, sm.recv_frame, sm.alive,
+              sm.sock, sm.data, sm.logMonoTime, sm.valid]:
+      assert len(cast(Sized, p)) == len(events)
+
+  def test_init_state(self):
+    socks = random_socks()
+    sm = messaging.SubMaster(socks)
+    assert sm.frame == -1
+    assert not any(sm.updated.values())
+    assert not any(sm.seen.values())
+    on_demand = {s: SERVICE_LIST[s].frequency <= 1e-5 for s in sm.services}
+    assert all(sm.alive[s] == sm.valid[s] == sm.freq_ok[s] == on_demand[s] for s in sm.services)
+    assert all(t == 0. for t in sm.recv_time.values())
+    assert all(f == 0 for f in sm.recv_frame.values())
+    assert all(t == 0 for t in sm.logMonoTime.values())
+
+    for p in [sm.updated, sm.recv_time, sm.recv_frame, sm.alive,
+              sm.sock, sm.data, sm.logMonoTime, sm.valid]:
+      assert len(cast(Sized, p)) == len(socks)
+
+  def test_getitem(self):
+    sock = "carState"
+    pub_sock = messaging.pub_sock(sock)
+    sm = messaging.SubMaster([sock,])
+    zmq_sleep()
+
+    msg = random_carstate()
+    pub_sock.send(msg.to_bytes())
+    sm.update(1000)
+    assert_carstate(msg.carState, sm[sock])
+
+  # TODO: break this test up to individually test SubMaster.update and SubMaster.update_msgs
+  def test_update(self):
+    sock = "carState"
+    pub_sock = messaging.pub_sock(sock)
+    sm = messaging.SubMaster([sock,])
+    zmq_sleep()
+
+    for i in range(10):
+      msg = messaging.new_message(sock)
+      pub_sock.send(msg.to_bytes())
+      sm.update(1000)
+      assert sm.frame == i
+      assert all(sm.updated.values())
+
+  def test_update_timeout(self):
+    sock = random_sock()
+    sm = messaging.SubMaster([sock,])
+    timeout = random.randrange(1000, 3000)
+    start_time = time.monotonic()
+    sm.update(timeout)
+    t = time.monotonic() - start_time
+    assert t >= timeout/1000.
+    assert t < 3
+    assert not any(sm.updated.values())
+
+  def test_avg_frequency_checks(self):
+    for poll in (True, False):
+      sm = messaging.SubMaster(["modelV2", "carParams", "carState", "cameraOdometry", "liveCalibration"],
+                               poll=("modelV2" if poll else None),
+                               frequency=(20. if not poll else None))
+
+      checks = {
+        "carState": (20, 20),
+        "modelV2": (20, 20 if poll else 10),
+        "cameraOdometry": (20, 10),
+        "liveCalibration": (4, 4),
+        "carParams": (None, None),
+        "userBookmark": (None, None),
+      }
+
+      for service, (max_freq, min_freq) in checks.items():
+        if max_freq is not None:
+          assert sm._check_avg_freq(service)
+          assert sm.freq_tracker[service].max_freq == max_freq*1.2
+          assert sm.freq_tracker[service].min_freq == min_freq*0.8
+        else:
+          assert not sm._check_avg_freq(service)
+
+  def test_alive(self):
+    pass
+
+  def test_ignore_alive(self):
+    pass
+
+  def test_valid(self):
+    pass
+
+  # SubMaster should always conflate
+  def test_conflate(self):
+    sock = "carState"
+    pub_sock = messaging.pub_sock(sock)
+    sm = messaging.SubMaster([sock,])
+
+    n = 10
+    for i in range(n+1):
+      msg = messaging.new_message(sock)
+      msg.carState.vEgo = i
+      pub_sock.send(msg.to_bytes())
+      time.sleep(0.01)
+    sm.update(1000)
+    assert sm[sock].vEgo == n
+
+
+class TestPubMaster:
+
+  def setup_method(self):
+    # ZMQ pub socket takes too long to die
+    # sleep to prevent multiple publishers error between tests
+    zmq_sleep(3)
+
+  def test_init(self):
+    messaging.PubMaster(events)
+
+  def test_send(self):
+    socks = random_socks()
+    pm = messaging.PubMaster(socks)
+    sub_socks = {s: messaging.sub_sock(s, conflate=True, timeout=1000) for s in socks}
+    zmq_sleep()
+
+    # PubMaster accepts either a capnp msg builder or bytes
+    for capnp in [True, False]:
+      for i in range(100):
+        sock = socks[i % len(socks)]
+
+        if capnp:
+          try:
+            msg = messaging.new_message(sock)
+          except Exception:
+            msg = messaging.new_message(sock, random.randrange(50))
+        else:
+          msg = random_bytes()
+
+        pm.send(sock, msg)
+        recvd = sub_socks[sock].receive()
+
+        if capnp:
+          msg.clear_write_flag()
+          msg = msg.to_bytes()
+        assert msg == recvd, i

cereal/messaging/tests/test_services.py

@@ -0,0 +1,27 @@
+import os
+import tempfile
+from typing import Dict
+from parameterized import parameterized
+
+from cereal import log
+import cereal.services as services
+from cereal.services import SERVICE_LIST
+
+
+class TestServices:
+
+  @parameterized.expand(SERVICE_LIST.keys())
+  def test_services(self, s):
+    service = SERVICE_LIST[s]
+    assert service.frequency <= 104
+    assert service.decimation != 0
+
+  def test_generated_header(self):
+    with tempfile.NamedTemporaryFile(suffix=".h") as f:
+      ret = os.system(f"python3 {services.__file__} > {f.name} && clang++ {f.name} -std=c++11")
+      assert ret == 0, "generated services header is not valid C"
+
+  def test_all_services_exist_in_log_union(self):
+    event_fields = set(log.Event.schema.union_fields)
+    missing = sorted(s for s in SERVICE_LIST if s not in event_fields)
+    assert not missing, f"services missing from log.capnp Event union: {missing}"

cereal/messaging/tests/validate_sp_cereal_upstream.py

@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+import argparse
+import sys
+from typing import Any, List, Tuple
+
+DEBUG = False
+
+
+def print_debug(string: str) -> None:
+  if DEBUG:
+    print(string)
+
+
+def create_schema_instance(struct: Any, prop: Tuple[str, Any]) -> Any:
+  """
+  Create a new instance of a schema type, handling different field types.
+
+  Args:
+      struct: The Cap'n Proto schema structure
+      prop: A tuple containing the field name and field metadata
+
+  Returns:
+      A new initialized schema instance
+  """
+  struct_instance = struct.new_message()
+  field_name, field_metadata = prop
+
+  try:
+    field_type = field_metadata.proto.slot.type.which()
+
+    # Initialize different types of fields
+    if field_type in ('list', 'text', 'data'):
+      struct_instance.init(field_name, 1)
+      print_debug(f"Initialized list/text/data field: {field_name}")
+    elif field_type in ('struct', 'object'):
+      struct_instance.init(field_name)
+      print_debug(f"Initialized struct/object field: {field_name}")
+
+    return struct_instance
+
+  except Exception as e:
+    print(f"Error creating instance for {field_name}: {e}")
+    return None
+
+
+def get_schema_fields(schema_struct: Any) -> List[Tuple[str, Any]]:
+  """
+  Retrieve all fields from a given schema structure.
+
+  Args:
+      schema_struct: The Cap'n Proto schema structure
+
+  Returns:
+      A list of field names and their metadata
+  """
+  try:
+    # Get all fields from the schema
+    schema_fields = list(schema_struct.schema.fields.items())
+
+    print_debug("Discovered schema fields:")
+    for field_name, field_metadata in schema_fields:
+      print_debug(f"- {field_name}")
+
+    return schema_fields
+
+  except Exception as e:
+    print(f"Error retrieving schema fields: {e}")
+    return []
+
+
+def generate_schema_instances(schema_struct: Any) -> List[Any]:
+  """
+  Generate instances for all fields in a given schema.
+
+  Args:
+      schema_struct: The Cap'n Proto schema structure
+
+  Returns:
+      A list of schema instances
+  """
+  schema_fields = get_schema_fields(schema_struct)
+  instances = []
+
+  for field_prop in schema_fields:
+    try:
+      instance = create_schema_instance(schema_struct, field_prop)
+      if instance is not None:
+        instances.append(instance)
+    except Exception as e:
+      print(f"Skipping field due to error: {e}")
+
+  print(f"Generated {len(instances)} schema instances")
+  return instances
+
+
+def persist_instances(instances: List[Any], filename: str) -> None:
+  """
+  Write schema instances to a binary file.
+
+  Args:
+      instances: List of schema instances
+      filename: Output file path
+  """
+  try:
+    with open(filename, 'wb') as f:
+      for instance in instances:
+        f.write(instance.to_bytes())
+
+    print(f"Successfully wrote {len(instances)} instances to {filename}")
+
+  except Exception as e:
+    print(f"Error persisting instances: {e}")
+    sys.exit(1)
+
+
+def read_instances(filename: str, schema_type: Any) -> List[Any]:
+  """
+  Read schema instances from a binary file.
+
+  Args:
+      filename: Input file path
+      schema_type: The schema type to use for reading
+
+  Returns:
+      A list of read schema instances
+  """
+  try:
+    with open(filename, 'rb') as f:
+      data = f.read()
+
+    instances = list(schema_type.read_multiple_bytes(data))
+
+    print(f"Read {len(instances)} instances from {filename}")
+    return instances
+
+  except Exception as e:
+    print(f"Error reading instances: {e}")
+    sys.exit(1)
+
+
+def compare_schemas(original_instances: List[Any], read_instances: List[Any]) -> bool:
+  """
+  Compare original and read-back instances to detect potential breaking changes.
+
+  Args:
+      original_instances: List of originally generated instances
+      read_instances: List of instances read back from file
+
+  Returns:
+      Boolean indicating whether schemas appear compatible
+  """
+  if len(original_instances) != len(read_instances):
+    print("❌ Schema Compatibility Warning: Instance count mismatch")
+    return False
+
+  compatible = True
+  for struct in read_instances:
+    try:
+      getattr(struct, struct.which())  # Attempting to access the field to validate readability
+    except Exception as e:
+      print(f"❌ Structural change detected: {struct.which()} is not readable.\nFull error: {e}")
+      compatible = False
+
+  return compatible
+
+
+def main():
+  """
+  CLI entry point for schema compatibility testing.
+  """
+  # Setup argument parser
+  parser = argparse.ArgumentParser(
+    description='Cap\'n Proto Schema Compatibility Testing Tool',
+    epilog='Test schema compatibility by generating and reading back instances.'
+  )
+
+  # Add mutually exclusive group for generation or reading mode
+  mode_group = parser.add_mutually_exclusive_group(required=True)
+  mode_group.add_argument('-g', '--generate', action='store_true',
+                          help='Generate schema instances')
+  mode_group.add_argument('-r', '--read', action='store_true',
+                          help='Read and validate schema instances')
+
+  # Common arguments
+  parser.add_argument('-f', '--file',
+                      default='schema_instances.bin',
+                      help='Output/input binary file (default: schema_instances.bin)')
+
+  # Parse arguments
+  args = parser.parse_args()
+
+  # Import the schema dynamically 
+  try:
+    from cereal import log
+    schema_type = log.Event
+  except ImportError:
+    print("Error: Unable to import schema. Ensure 'cereal' is installed.")
+    sys.exit(1)
+
+  # Execute based on mode
+  if args.generate:
+    print("🔧 Generating Schema Instances")
+    instances = generate_schema_instances(schema_type)
+    persist_instances(instances, args.file)
+    print("✅ Instance generation complete")
+
+  elif args.read:
+    print("🔍 Reading and Validating Schema Instances")
+    generated_instances = generate_schema_instances(schema_type)
+    read_back_instances = read_instances(args.file, schema_type)
+
+    # Compare schemas
+    if compare_schemas(generated_instances, read_back_instances):
+      print("✅ Schema Compatibility: No breaking changes detected")
+      sys.exit(0)
+    else:
+      print("❌ Potential Schema Breaking Changes Detected")
+      sys.exit(1)
+
+
+if __name__ == "__main__":
+  main()