IQ.Pilot Release Commit @ f2a861c

artifacts/package_sources/tinygrad/extra/nv_pma/.gitignore

@@ -0,0 +1 @@
+examples/

artifacts/package_sources/tinygrad/extra/nv_pma/collect.py

@@ -0,0 +1,135 @@
+import pickle, os, sys, functools, numpy as np
+from pathlib import Path
+
+os.environ["DEV"] = "CUDA"
+os.environ["PROFILE"] = os.environ.get("PROFILE", "2")
+from extra.nv_pma.cupti import cu_prof_ext
+cu_prof_ext.enable_auto()
+
+from tinygrad import Tensor, Device
+
+if not os.environ.get("IOCTL") or not os.environ.get("GRAB_PMA"):
+  print("Usage: GRAB_PMA=1 IOCTL=1 IOCTL_PRINT=0 python3 extra/nv_pma/collect.py")
+  sys.exit(1)
+
+assert Device.DEFAULT == "CUDA", "only works with CUDA"
+
+EXAMPLES_DIR = Path(__file__).parent / "examples"
+_collectors: list[tuple[str, callable]] = []
+
+def pcsampling_test(name: str):
+  def decorator(fn):
+    @functools.wraps(fn)
+    def wrapper():
+      cu_prof_ext.clear_pma_raw_dumps()
+      cu_prof_ext.clear_cupti_pc_samples()
+
+      fn()
+      Device["CUDA"].synchronize()
+
+      dumps = cu_prof_ext.get_pma_raw_dumps()
+      # from hexdump import hexdump
+      # hexdump(dumps[0][:0x40])
+
+      return {"test_name": name, "pma_raw_dumps": list(cu_prof_ext.get_pma_raw_dumps()), "cupti_pc_samples": list(cu_prof_ext.get_cupti_pc_samples())}
+    _collectors.append((name, wrapper))
+    return wrapper
+  return decorator
+
+# Refs
+
+@pcsampling_test("test_plus")
+def test_plus():
+  a = Tensor([1, 2, 3, 4])
+  b = Tensor([5, 6, 7, 8])
+  (a + b).realize()
+
+@pcsampling_test("test_matmul")
+def test_matmul():
+  a = Tensor(np.random.rand(12, 12).astype(np.float32))
+  b = Tensor(np.random.rand(12, 12).astype(np.float32))
+  (a @ b).realize()
+
+@pcsampling_test("test_reduce_sum")
+def test_reduce_sum():
+  a = Tensor(np.random.rand(1024).astype(np.float32))
+  a.sum().realize()
+
+@pcsampling_test("test_reduce_max")
+def test_reduce_max():
+  a = Tensor(np.random.rand(1024).astype(np.float32))
+  a.max().realize()
+
+@pcsampling_test("test_exp")
+def test_exp():
+  a = Tensor(np.random.rand(256).astype(np.float32))
+  a.exp().realize()
+
+@pcsampling_test("test_softmax")
+def test_softmax():
+  a = Tensor(np.random.rand(64, 64).astype(np.float32))
+  a.softmax().realize()
+
+@pcsampling_test("test_conv2d")
+def test_conv2d():
+  x = Tensor(np.random.rand(1, 3, 32, 32).astype(np.float32))
+  w = Tensor(np.random.rand(8, 3, 3, 3).astype(np.float32))
+  x.conv2d(w).realize()
+
+@pcsampling_test("test_large_matmul")
+def test_large_matmul():
+  a = Tensor(np.random.rand(128, 128).astype(np.float32))
+  b = Tensor(np.random.rand(128, 128).astype(np.float32))
+  (a @ b).realize()
+
+@pcsampling_test("test_elementwise_chain")
+def test_elementwise_chain():
+  a = Tensor(np.random.rand(512).astype(np.float32))
+  ((a + 1) * 2 - 0.5).relu().realize()
+
+@pcsampling_test("test_broadcast")
+def test_broadcast():
+  a = Tensor(np.random.rand(64, 1).astype(np.float32))
+  b = Tensor(np.random.rand(1, 64).astype(np.float32))
+  (a + b).realize()
+
+@pcsampling_test("test_plus_big")
+def test_plus_big():
+  a = Tensor(np.random.rand(64, 32).astype(np.float32))
+  b = Tensor(np.random.rand(64, 32).astype(np.float32))
+  (a + b).realize()
+
+def save_example(name: str, data: dict):
+  pma_bytes = sum(len(d) for d in data['pma_raw_dumps'])
+  cupti_samples = sum(r['samples'] for r in data['cupti_pc_samples'])
+  print(f"  PMA: {len(data['pma_raw_dumps'])} buffers, {pma_bytes} bytes")
+  print(f"  CUPTI: {len(data['cupti_pc_samples'])} records, {cupti_samples} samples")
+
+  outfile = EXAMPLES_DIR / f"{name}.pkl"
+  with open(outfile, "wb") as f:
+    pickle.dump(data, f)
+  print(f"  Saved to {outfile}")
+
+if __name__ == "__main__":
+  EXAMPLES_DIR.mkdir(exist_ok=True)
+
+  # Run specific tests if provided as arguments, otherwise run all
+  if len(sys.argv) > 1:
+    test_names = sys.argv[1:]
+    collectors = [(name, fn) for name, fn in _collectors if name in test_names]
+    if not collectors:
+      print(f"Unknown tests: {test_names}")
+      print(f"Available: {[name for name, _ in _collectors]}")
+      sys.exit(1)
+  else:
+    collectors = _collectors
+
+  for name, collect_fn in collectors:
+    print(f"\nCollecting {name}...")
+    try:
+      data = collect_fn()
+      save_example(name, data)
+    except Exception as e:
+      print(f"  ERROR: {e}")
+      import traceback
+      traceback.print_exc()

artifacts/package_sources/tinygrad/extra/nv_pma/cupti/__init__.py

@@ -0,0 +1,26 @@
+# CUPTI autogen loader for nv_pma
+# To regenerate: REGEN=1 python -c "import extra.nv_pma.cupti"
+import importlib, pathlib
+from tinygrad.helpers import getenv
+
+root = pathlib.Path(__file__).parents[3]
+here = pathlib.Path(__file__).parent
+
+def load(name, dll, files, **kwargs):
+  if not (f:=here/f"{name}.py").exists() or getenv('REGEN'):
+    kwargs['args'] = kwargs.get('args', [])
+    f.write_text(importlib.import_module("tinygrad.runtime.support.autogen").gen(name, dll, files, **kwargs))
+  return importlib.import_module(f"extra.nv_pma.cupti.{name}")
+
+def __getattr__(nm):
+  match nm:
+    case "cupti":
+      return load("cupti", "'/usr/local/cuda/targets/x86_64-linux/lib/libcupti.so'", [
+        "/usr/local/cuda/include/cupti_result.h", "/usr/local/cuda/include/cupti_activity.h",
+        "/usr/local/cuda/include/cupti_callbacks.h", "/usr/local/cuda/include/cupti_events.h",
+        "/usr/local/cuda/include/cupti_metrics.h", "/usr/local/cuda/include/cupti_driver_cbid.h",
+        "/usr/local/cuda/include/cupti_runtime_cbid.h", "/usr/local/cuda/include/cupti_profiler_target.h",
+        "/usr/local/cuda/include/cupti_profiler_host.h", "/usr/local/cuda/include/cupti_pmsampling.h",
+        "/usr/local/cuda/include/generated_cuda_meta.h", "/usr/local/cuda/include/generated_cuda_runtime_api_meta.h"
+      ], args=["-D__CUDA_API_VERSION_INTERNAL", "-I/usr/local/cuda/include"], parse_macros=False)
+    case _: raise AttributeError(f"no such autogen: {nm}")

artifacts/package_sources/tinygrad/extra/nv_pma/cupti/cu_prof_ext.py

@@ -0,0 +1,164 @@
+from __future__ import annotations
+import ctypes
+from tinygrad.helpers import DEBUG, getenv
+from extra.nv_pma.cupti import cupti
+
+def stall_reason_name(reason: int) -> str:
+  name = cupti.CUpti_ActivityPCSamplingStallReason.get(reason)
+  return name.replace("CUPTI_ACTIVITY_PC_SAMPLING_STALL_", "").lower() if name else str(reason)
+
+class CUPTIProfiler:
+  def __init__(self):
+    self.initialized = False
+    self.pc_sampling_enabled = False
+    self.buffers: list[ctypes.Array] = []
+    self.kernel_stalls: dict[int, dict[int, int]] = {}
+    self.raw_buffers: list[bytes] = []
+    self.pc_samples: list[dict] = []
+
+  def _check_cupti(self, status, soft=False):
+    if status != cupti.CUPTI_SUCCESS:
+      if soft: return False
+      raise RuntimeError(f"CUPTI Error {status}")
+    return True
+
+  def init(self, ctx, device_id: int = 0, profile_level: int = 2):
+    if self.initialized: return
+
+    # Initialize profiler API
+    init_params = cupti.CUpti_Profiler_Initialize_Params()
+    init_params.structSize = 16
+    cupti.cuptiProfilerInitialize(ctypes.byref(init_params))
+
+    # Register buffer callbacks for Activity API
+    self._buf_req_cb = cupti.CUpti_BuffersCallbackRequestFunc(self._buffer_requested)
+    self._buf_comp_cb = cupti.CUpti_BuffersCallbackCompleteFunc(self._buffer_completed)
+    self._check_cupti(cupti.cuptiActivityRegisterCallbacks(self._buf_req_cb, self._buf_comp_cb))
+
+    # PROFILE=1: kernel timing, PROFILE=2: PC sampling with stall reasons
+    if profile_level >= 2:
+      # PC sampling for stall analysis (requires elevated privileges)
+      if DEBUG >= 1: print("  CUPTI: PC sampling mode (before)")
+      pc_status = cupti.cuptiActivityEnable(cupti.CUPTI_ACTIVITY_KIND_PC_SAMPLING)
+      if pc_status == cupti.CUPTI_SUCCESS:
+        config = cupti.CUpti_ActivityPCSamplingConfig()
+        config.size, config.samplingPeriod = 16, cupti.CUPTI_ACTIVITY_PC_SAMPLING_PERIOD_MIN
+        cfg_status = cupti.dll.cuptiActivityConfigurePCSampling(ctx, ctypes.byref(config))
+        if cfg_status == cupti.CUPTI_SUCCESS:
+          if DEBUG >= 1: print("  CUPTI: PC sampling mode (before stall analysis)")
+          cupti.cuptiActivityEnable(cupti.CUPTI_ACTIVITY_KIND_PC_SAMPLING_RECORD_INFO)
+          self.pc_sampling_enabled = True
+          if DEBUG >= 1: print("  CUPTI: PC sampling mode (stall analysis)")
+        elif cfg_status == 35:
+          if DEBUG >= 1: print("  CUPTI: PC sampling needs: echo 'options nvidia NVreg_RestrictProfilingToAdminUsers=0'|sudo tee /etc/modprobe.d/nvidia.conf && sudo reboot")
+      # Fall back to kernel timing if PC sampling setup failed
+      if not self.pc_sampling_enabled:
+        self._check_cupti(cupti.cuptiActivityEnable(cupti.CUPTI_ACTIVITY_KIND_KERNEL))
+    else:
+      # Kernel activity tracing for timing
+      self._check_cupti(cupti.cuptiActivityEnable(cupti.CUPTI_ACTIVITY_KIND_KERNEL))
+
+    self.initialized = True
+
+  def _buffer_requested(self, buffer, size, max_num_records):
+    buf = (ctypes.c_uint8 * 1024 * 1024)()  # 1MB buffer
+    self.buffers.append(buf)
+    buffer[0] = ctypes.cast(buf, ctypes.POINTER(ctypes.c_uint8))
+    size[0] = ctypes.sizeof(buf)
+    max_num_records[0] = 0
+
+  def _buffer_completed(self, ctx, stream_id, buffer, size, valid_size):
+    if valid_size > 0:
+      record = ctypes.POINTER(cupti.CUpti_Activity)()
+      while cupti.cuptiActivityGetNextRecord(buffer, valid_size, ctypes.byref(record)) == cupti.CUPTI_SUCCESS:
+        kind = record.contents.kind
+        if kind == cupti.CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL:
+          kernel = ctypes.cast(record, ctypes.POINTER(cupti.CUpti_ActivityKernel9)).contents
+          name = ctypes.string_at(kernel.name).decode() if kernel.name else "unknown"
+          duration_us = (kernel.end - kernel.start) / 1000.0
+          grid, block = (kernel.gridX, kernel.gridY, kernel.gridZ), (kernel.blockX, kernel.blockY, kernel.blockZ)
+          print(f"  CUPTI: {name[:40]:40s} | {duration_us:10.2f} us | grid={grid} block={block} | regs={kernel.registersPerThread:3d} smem={kernel.staticSharedMemory + kernel.dynamicSharedMemory:6d}B")
+        elif kind == cupti.CUPTI_ACTIVITY_KIND_PC_SAMPLING:
+          pc = ctypes.cast(record, ctypes.POINTER(cupti.CUpti_ActivityPCSampling3)).contents
+          cid = pc.correlationId
+          if cid not in self.kernel_stalls: self.kernel_stalls[cid] = {}
+          self.kernel_stalls[cid][pc.stallReason] = self.kernel_stalls[cid].get(pc.stallReason, 0) + pc.samples
+          self.pc_samples.append({
+            'correlationId': pc.correlationId, 'pcOffset': pc.pcOffset, 'stallReason': pc.stallReason,
+            'samples': pc.samples, 'latencySamples': pc.latencySamples, 'functionId': pc.functionId, 'sourceLocatorId': pc.sourceLocatorId
+          })
+          if DEBUG >= 3:
+            print(f"    PC {pc.pcOffset:#x} stall={stall_reason_name(pc.stallReason)} samples={pc.samples} latency={pc.latencySamples} func={pc.functionId} src={pc.sourceLocatorId}")
+        elif kind == cupti.CUPTI_ACTIVITY_KIND_PC_SAMPLING_RECORD_INFO:
+          info = ctypes.cast(record, ctypes.POINTER(cupti.CUpti_ActivityPCSamplingRecordInfo)).contents
+          cid = info.correlationId
+          if cid in self.kernel_stalls:
+            stalls = self.kernel_stalls[cid]
+            total = sum(stalls.values())
+            if total > 0:
+              top = sorted(stalls.items(), key=lambda x: -x[1])[:5]
+              stall_str = " ".join(f"{stall_reason_name(r)}:{100*c//total}%" for r,c in top if c > 0)
+              print(f"  CUPTI stalls (corr={cid}): {total} samples | {stall_str}")
+            del self.kernel_stalls[cid]
+        else: print(f"  CUPTI: Unhandled activity kind {kind}")
+
+  def flush(self):
+    if not self.initialized: return
+    self._check_cupti(cupti.cuptiActivityFlushAll(0))
+
+# Module-level profiler instance
+_profiler: CUPTIProfiler | None = None
+
+def get_profiler() -> CUPTIProfiler | None:
+  return _profiler
+
+def get_cupti_raw_buffers() -> list[bytes]:
+  return _profiler.raw_buffers if _profiler else []
+
+def clear_cupti_raw_buffers():
+  if _profiler: _profiler.raw_buffers.clear()
+
+def get_cupti_pc_samples() -> list[dict]:
+  return _profiler.pc_samples if _profiler else []
+
+def clear_cupti_pc_samples():
+  if _profiler: _profiler.pc_samples.clear()
+
+# Raw PMA buffer access (from ioctl interception)
+def get_pma_raw_dumps() -> list[bytes]:
+  try:
+    from extra.nv_gpu_driver.nv_ioctl import get_pma_raw_dumps as _get
+    return _get()
+  except ImportError: return []
+
+def clear_pma_raw_dumps():
+  try:
+    from extra.nv_gpu_driver.nv_ioctl import clear_pma_raw_dumps as _clear
+    _clear()
+  except ImportError: pass
+
+def enable(profile_level:int=2):
+  global _profiler
+  if _profiler is not None: return
+
+  _profiler = CUPTIProfiler()
+
+  # Patch CUDADevice to initialize CUPTI profiler
+  from tinygrad.runtime.ops_cuda import CUDADevice
+  _orig_init = CUDADevice.__init__
+  _orig_sync = CUDADevice.synchronize
+
+  def _patched_init(self, device: str):
+    _orig_init(self, device)
+    device_id = int(device.split(":")[1]) if ":" in device else 0
+    _profiler.init(self.context, device_id, profile_level)
+
+  def _patched_sync(self):
+    _orig_sync(self)
+    if _profiler: _profiler.flush()
+
+  CUDADevice.__init__ = _patched_init
+  CUDADevice.synchronize = _patched_sync
+
+def enable_auto():
+  if (profile_level:=getenv("PROFILE", 0)) > 0: enable(profile_level)

artifacts/package_sources/tinygrad/extra/nv_pma/decode.py

@@ -0,0 +1,189 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+import enum, collections
+from typing import Iterator
+from tinygrad.helpers import colored
+from tinygrad.renderer.amd.sqtt import PacketType, bits
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# STALL REASONS
+# ═══════════════════════════════════════════════════════════════════════════════
+
+class StallReason(enum.IntEnum):
+  # Based on CUpti_ActivityPCSamplingStallReason
+  INVALID = 0
+  NONE = 1              # selected, selected_not_issued
+  INST_FETCH = 2        # branch_resolving, no_instructions
+  EXEC_DEPENDENCY = 3   # short_scoreboard, wait
+  MEMORY_DEPENDENCY = 4 # long_scoreboard
+  TEXTURE = 5           # tex_throttle
+  SYNC = 6              # barrier, membar
+  CONSTANT_MEMORY = 7   # imc_miss
+  PIPE_BUSY = 8         # mio_throttle, math_pipe_throttle
+  MEMORY_THROTTLE = 9   # drain, lg_throttle
+  NOT_SELECTED = 10     # not_selected
+  OTHER = 11            # misc, dispatch_stall
+  SLEEPING = 12         # sleeping
+
+STALL_KEY_MAP_AMPERE: dict[int, StallReason] = {
+  1: StallReason.MEMORY_THROTTLE, 15: StallReason.MEMORY_THROTTLE,
+  2: StallReason.CONSTANT_MEMORY,
+  3: StallReason.SYNC,
+  6: StallReason.INST_FETCH, 11: StallReason.INST_FETCH,
+  7: StallReason.EXEC_DEPENDENCY, 10: StallReason.EXEC_DEPENDENCY,
+  9: StallReason.MEMORY_DEPENDENCY,
+  12: StallReason.PIPE_BUSY,
+  17: StallReason.OTHER, 20: StallReason.OTHER,
+  18: StallReason.NONE,
+}
+
+STALL_KEY_MAP_BLACKWELL: dict[int, StallReason] = {
+  0x01: StallReason.MEMORY_THROTTLE, 0x0e: StallReason.MEMORY_THROTTLE,
+  0x02: StallReason.SYNC,
+  0x05: StallReason.INST_FETCH, 0x0a: StallReason.INST_FETCH,
+  0x06: StallReason.EXEC_DEPENDENCY, 0x09: StallReason.EXEC_DEPENDENCY,
+  0x08: StallReason.MEMORY_DEPENDENCY,
+  0x0b: StallReason.PIPE_BUSY, 0x0f: StallReason.PIPE_BUSY,
+  0x10: StallReason.OTHER, 0x13: StallReason.OTHER,
+  0x11: StallReason.NONE,
+}
+
+# Lookup table for extracting sample bytes from 32-byte packet (bytes 0-3, 8-31, skipping header at 4-7)
+LOOKUP_28B = [0, 1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# PACKET HEADER
+# ═══════════════════════════════════════════════════════════════════════════════
+
+class PMAHeader(PacketType):
+  num_bytes  = bits[4:0]    # number of sample bytes in this packet
+  tpc_id_lo  = bits[15:8]   # TPC identifier low 8 bits
+  tpc_id_hi  = bits[27:25]  # TPC identifier high 3 bits
+  dropped    = bits[28:28]  # dropped flag (resets byte accumulator)
+  @property
+  def tpc_id(self) -> int: return self.tpc_id_lo | (self.tpc_id_hi << 8)
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# 8-BYTE SAMPLE FORMAT (Ampere/Ada/Hopper)
+# ═══════════════════════════════════════════════════════════════════════════════
+
+class PMASampleAmpere8B(PacketType):
+  pc_raw     = bits[44:0]   # raw PC value (pc_offset = pc_raw << 4)
+  stall_key  = bits[49:45]  # stall reason key
+  wave_id    = bits[55:50]  # warp/wave identifier
+  active     = bits[62:62]  # 1 if warp was executing, 0 if scheduled but not issued
+  @property
+  def pc_offset(self) -> int: return self.pc_raw << 4
+  @property
+  def stall_reason(self) -> StallReason: return STALL_KEY_MAP_AMPERE.get(self.stall_key, StallReason.OTHER)
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# 9-BYTE SAMPLE FORMAT (Blackwell+)
+# ═══════════════════════════════════════════════════════════════════════════════
+
+class PMASampleBlackwell9B(PacketType):
+  stall_key  = bits[5:0]    # stall reason key
+  pc_raw     = bits[60:8]   # raw PC value (pc_offset = pc_raw << 4)
+  wave_hi    = bits[7:6]    # wave_id high 2 bits
+  wave_lo    = bits[71:68]  # wave_id low 4 bits
+  active     = bits[67:67]  # 1 if warp was executing, 0 if scheduled but not issued
+  @property
+  def pc_offset(self) -> int: return self.pc_raw << 4
+  @property
+  def stall_reason(self) -> StallReason: return STALL_KEY_MAP_BLACKWELL.get(self.stall_key, StallReason.OTHER)
+  @property
+  def wave_id(self) -> int: return (self.wave_hi << 4) | self.wave_lo
+
+PMASample = PMASampleAmpere8B|PMASampleBlackwell9B
+
+def decode(data: bytes, sm_version: int = 0x800) -> Iterator[tuple[PMASample, int]]:
+  use_9byte = sm_version >= 0xa04
+  record_size = 9 if use_9byte else 8
+  sample_cls = PMASampleBlackwell9B if use_9byte else PMASampleAmpere8B
+
+  tpc_state: dict[int, list[int]] = collections.defaultdict(list)
+  for pkt_idx in range(len(data) // 32):
+    pkt = data[pkt_idx * 32:(pkt_idx + 1) * 32]
+    hdr = PMAHeader.from_raw(int.from_bytes(pkt[4:8], 'little'))
+
+    if hdr.dropped: tpc_state[hdr.tpc_id].clear()
+
+    for i in range(hdr.num_bytes):
+      tpc_state[hdr.tpc_id].append(pkt[LOOKUP_28B[i]])
+
+    while len(tpc_state[hdr.tpc_id]) >= record_size:
+      yield sample_cls.from_raw(int.from_bytes(bytes(tpc_state[hdr.tpc_id][:record_size]), 'little')), hdr.tpc_id
+      del tpc_state[hdr.tpc_id][:record_size]
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# CLI
+# ═══════════════════════════════════════════════════════════════════════════════
+
+STALL_COLORS = {
+  StallReason.NONE: "green", StallReason.INST_FETCH: "yellow", StallReason.EXEC_DEPENDENCY: "cyan",
+  StallReason.MEMORY_DEPENDENCY: "red", StallReason.SYNC: "magenta", StallReason.CONSTANT_MEMORY: "blue",
+  StallReason.PIPE_BUSY: "yellow", StallReason.MEMORY_THROTTLE: "RED", StallReason.OTHER: "white",
+}
+
+def decode_tpc_id(tpc_id:int) -> tuple[int, int, int]:
+  # NOTE: valid only for ops_nv, cuda encoding is different
+  return (tpc_id >> 5, (tpc_id >> 1) & 0xf, tpc_id & 1)
+
+def print_packets(data:bytes, sm_version:int=0x800) -> None:
+  record_size = 9 if sm_version >= 0x890 else 8
+  tpc_state: dict[int, list[int]] = collections.defaultdict(list)
+  for i in range(len(data) // 32):
+    pkt = data[i * 32:(i + 1) * 32]
+    hdr = PMAHeader.from_raw(int.from_bytes(pkt[4:8], 'little'))
+    if hdr.dropped: tpc_state[hdr.tpc_id].clear()
+    for j in range(hdr.num_bytes): tpc_state[hdr.tpc_id].append(pkt[LOOKUP_28B[j]])
+    # Show complete records extracted from this packet
+    records = []
+    while len(tpc_state[hdr.tpc_id]) >= record_size:
+      records.append(bytes(tpc_state[hdr.tpc_id][:record_size]).hex())
+      del tpc_state[hdr.tpc_id][:record_size]
+    leftover = len(tpc_state[hdr.tpc_id])
+    print(f"Pkt {i:3d}: tpc={hdr.tpc_id:4d} n={hdr.num_bytes:2d} drop={hdr.dropped} left={leftover} | {' '.join(records)}")
+
+def print_aggregated(samples:list[tuple[PMASample, int]]) -> None:
+  if not samples: return
+  base_pc = min(s.pc_offset for s, _ in samples)
+  counter: collections.Counter[tuple[int, StallReason]] = collections.Counter((s.pc_offset, s.stall_reason) for s, _ in samples)
+  print(f"\nAggregated samples (base_pc=0x{base_pc:x}):")
+  for (pc, reason), cnt in sorted(counter.items()):
+    stall_str = colored(f"{reason.name:17}", STALL_COLORS.get(reason, "white"))
+    print(f"  pc=0x{pc - base_pc:06x} {stall_str} samples={cnt:4d}")
+
+if __name__ == "__main__":
+  import sys, pickle
+
+  if len(sys.argv) < 2:
+    print("Usage: python decode.py <pkl_file> [--raw] [--sm=0xNNN]")
+    sys.exit(1)
+
+  with open(sys.argv[1], "rb") as f:
+    data = pickle.load(f)
+
+  if isinstance(data, dict):
+    sm_version = 0x800  # default to Ampere
+    for arg in sys.argv:
+      if arg.startswith("--sm="): sm_version = int(arg[5:], 0)
+    dumps = [(i, x, sm_version) for i, x in enumerate(data["pma_raw_dumps"])]
+  else:
+    devs = {e.device: e for e in data if type(e).__name__ == "ProfileDeviceEvent"}
+    dumps = []
+    for i, e in enumerate(e for e in data if type(e).__name__ == "ProfilePMAEvent"):
+      dumps.append((i, e.blob, devs[e.device].props.get('sm_version', 0x800)))
+
+  for dump_idx, raw, sm_ver in dumps:
+    print(f"\n{'='*60}\nDump {dump_idx} ({len(raw)} bytes, {len(raw)//32} packets)\n{'='*60}")
+    if "--raw" in sys.argv: print_packets(raw, sm_ver)
+    else:
+      samples = []
+      for s, tpc_id in decode(raw, sm_ver):
+        gpc, tpc, sm = decode_tpc_id(tpc_id)
+        stall_str = colored(f"{s.stall_reason.name:17}", STALL_COLORS.get(s.stall_reason, "white"))
+        print(f"pc=0x{s.pc_offset:06x} {stall_str} ev={s.stall_key:2d} active={s.active} wave={s.wave_id:2d} gpc={gpc} tpc={tpc} sm={sm}")
+        samples.append((s, tpc_id))
+      print(f"\nDecoded {len(samples)} samples:")
+      print_aggregated(samples)

artifacts/package_sources/tinygrad/extra/nv_pma/test/test_nvprof.py

@@ -0,0 +1,76 @@
+import pickle, unittest
+from collections import Counter
+from pathlib import Path
+
+from extra.nv_pma.decode import decode
+from tinygrad.helpers import DEBUG
+
+EXAMPLES_DIR = Path(__file__).parent.parent / "examples"
+EXAMPLES_5090_DIR = Path(__file__).parent.parent / "examples_5090"
+
+def decode_and_aggregate(raw_dumps: list[bytes], sm_version: int = 0x800) -> Counter[tuple[int, int]]:
+  """Decode all PMA buffers and aggregate by (relative_pc, stall_reason). Each dump is normalized separately."""
+  result: Counter[tuple[int, int]] = Counter()
+  for raw in raw_dumps:
+    samples = [s for s, _ in decode(raw, sm_version)]
+    if not samples: continue
+    base_pc = min(s.pc_offset for s in samples)
+    result += Counter((s.pc_offset - base_pc, int(s.stall_reason)) for s in samples)
+  return result
+
+def cupti_to_counter(cupti_records: list[dict]) -> Counter[tuple[int, int]]:
+  """Convert CUPTI records to Counter[(pcOffset, stallReason)]."""
+  counter: Counter[tuple[int, int]] = Counter()
+  for r in cupti_records:
+    counter[(r['pcOffset'], r['stallReason'])] += r['samples']
+  return counter
+
+class TestNVProf(unittest.TestCase):
+  def _test_example(self, name: str, sm_version: int = 0x800, examples_dir: Path = EXAMPLES_DIR):
+    pkl_file = examples_dir / f"{name}.pkl"
+    if not pkl_file.exists():
+      self.skipTest(f"Example data not found: {pkl_file}. Run collect.py first.")
+
+    with open(pkl_file, "rb") as f:
+      data = pickle.load(f)
+
+    self.assertEqual(data["test_name"], name)
+    pma_agg = decode_and_aggregate(data["pma_raw_dumps"], sm_version)
+    cupti_agg = cupti_to_counter(data["cupti_pc_samples"])
+
+    if DEBUG >= 2:
+      total = sum(cupti_agg.values())
+      mismatched = sum(abs(pma_agg.get(k, 0) - v) for k, v in cupti_agg.items())
+      mismatched += sum(v for k, v in pma_agg.items() if k not in cupti_agg)
+      mismatched //= 2
+
+      print(f"\n=== Test: {name} ===")
+      print(f"Total samples: {total}, Mismatched: {mismatched} ({mismatched/total*100 if total else 0:.1f}%)")
+
+    self.assertEqual(pma_agg, cupti_agg, f"PMA: {dict(pma_agg)}\nCUPTI: {dict(cupti_agg)}")
+
+  # Ampere tests (8-byte format)
+  def test_decode_test_plus(self): self._test_example("test_plus")
+  def test_decode_test_reduce_sum(self): self._test_example("test_reduce_sum")
+  def test_decode_test_broadcast(self): self._test_example("test_broadcast")
+  def test_decode_test_matmul(self): self._test_example("test_matmul")
+  def test_decode_test_plus_big(self): self._test_example("test_plus_big")
+  def test_decode_test_elementwise_chain(self): self._test_example("test_elementwise_chain")
+  def test_decode_test_conv2d(self): self._test_example("test_conv2d")
+  def test_decode_test_large_matmul(self): self._test_example("test_large_matmul")
+
+  # Blackwell/5090 tests (9-byte format)
+  def test_5090_test_plus(self): self._test_example("test_plus", 0xa04, EXAMPLES_5090_DIR)
+  def test_5090_test_plus_big(self): self._test_example("test_plus_big", 0xa04, EXAMPLES_5090_DIR)
+  def test_5090_test_broadcast(self): self._test_example("test_broadcast", 0xa04, EXAMPLES_5090_DIR)
+  def test_5090_test_matmul(self): self._test_example("test_matmul", 0xa04, EXAMPLES_5090_DIR)
+  def test_5090_test_large_matmul(self): self._test_example("test_large_matmul", 0xa04, EXAMPLES_5090_DIR)
+  def test_5090_test_reduce_sum(self): self._test_example("test_reduce_sum", 0xa04, EXAMPLES_5090_DIR)
+  def test_5090_test_reduce_max(self): self._test_example("test_reduce_max", 0xa04, EXAMPLES_5090_DIR)
+  def test_5090_test_elementwise_chain(self): self._test_example("test_elementwise_chain", 0xa04, EXAMPLES_5090_DIR)
+  def test_5090_test_conv2d(self): self._test_example("test_conv2d", 0xa04, EXAMPLES_5090_DIR)
+  def test_5090_test_exp(self): self._test_example("test_exp", 0xa04, EXAMPLES_5090_DIR)
+  def test_5090_test_softmax(self): self._test_example("test_softmax", 0xa04, EXAMPLES_5090_DIR)
+
+if __name__ == "__main__":
+  unittest.main()