forked from IQ.Lvbs/IQ.Pilot
IQ.Pilot Release Commit @ 0798119
This commit is contained in:
IQ.Lvbs history cleanup
0
tinygrad_repo/test/amd/__init__.py
Normal file
0
tinygrad_repo/test/amd/__init__.py
Normal file
945
tinygrad_repo/test/amd/disasm.py
Normal file
945
tinygrad_repo/test/amd/disasm.py
Normal file
@@ -0,0 +1,945 @@
|
||||
# RDNA3/RDNA4/CDNA disassembler
|
||||
from __future__ import annotations
|
||||
import re
|
||||
from typing import Callable
|
||||
from test.amd.helpers import decode_dpp16
|
||||
from tinygrad.renderer.amd.dsl import Inst, Reg
|
||||
|
||||
# Special register mappings for disassembly
|
||||
SPECIAL_GPRS = {106: 'vcc_lo', 107: 'vcc_hi', 124: 'null', 125: 'm0', 126: 'exec_lo', 127: 'exec_hi',
|
||||
128: '0', 240: '0.5', 241: '-0.5', 242: '1.0', 243: '-1.0', 244: '2.0', 245: '-2.0',
|
||||
246: '4.0', 247: '-4.0', 248: '0x3e22f983', 253: 'scc'}
|
||||
SPECIAL_GPRS_CDNA = {106: 'vcc_lo', 107: 'vcc_hi', 124: 'm0', 126: 'exec_lo', 127: 'exec_hi',
|
||||
128: '0', 240: '0.5', 241: '-0.5', 242: '1.0', 243: '-1.0', 244: '2.0', 245: '-2.0',
|
||||
246: '4.0', 247: '-4.0', 248: '0x3e22f983', 253: 'scc',
|
||||
102: 'flat_scratch_lo', 103: 'flat_scratch_hi', 104: 'xnack_mask_lo', 105: 'xnack_mask_hi',
|
||||
251: 'src_vccz', 252: 'src_execz'}
|
||||
SPECIAL_PAIRS = {106: 'vcc', 126: 'exec'}
|
||||
SPECIAL_PAIRS_CDNA = {106: 'vcc', 126: 'exec', 102: 'flat_scratch', 104: 'xnack_mask'}
|
||||
|
||||
def decode_src(v, cdna: bool = False) -> str:
|
||||
"""Decode a source operand encoding to its string representation."""
|
||||
v = _unwrap(v)
|
||||
gprs = SPECIAL_GPRS_CDNA if cdna else SPECIAL_GPRS
|
||||
if v in gprs: return gprs[v]
|
||||
if v < 106: return f's{v}'
|
||||
if 108 <= v < 124: return f'ttmp{v - 108}'
|
||||
if 129 <= v <= 192: return str(v - 128) # positive integers 1-64
|
||||
if 193 <= v <= 208: return str(-(v - 192)) # negative integers -1 to -16
|
||||
if v >= 256: return f'v{v - 256}'
|
||||
return f's{v}'
|
||||
|
||||
def _unwrap(v) -> int:
|
||||
"""Unwrap Reg to int offset, or return int as-is."""
|
||||
return v.offset if isinstance(v, Reg) else v
|
||||
|
||||
def _vi(v) -> int:
|
||||
"""Get VGPR index from Reg or int (for v[N] fields that encode as 256+N)."""
|
||||
off = _unwrap(v)
|
||||
return off - 256 if off >= 256 else off
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# LITERAL FORMATTING
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
_FLOAT_DEC = {240: 0.5, 241: -0.5, 242: 1.0, 243: -1.0, 244: 2.0, 245: -2.0, 246: 4.0, 247: -4.0}
|
||||
|
||||
def _lit(inst, v, neg=0, cdna=None) -> str:
|
||||
"""Format literal/inline constant value."""
|
||||
if cdna is None: cdna = _is_cdna(inst)
|
||||
v = _unwrap(v)
|
||||
if v == 255:
|
||||
lit = inst._literal
|
||||
if lit is None: return "0"
|
||||
s = f"0x{lit:x}"
|
||||
elif v in _FLOAT_DEC: s = str(_FLOAT_DEC[v])
|
||||
elif 128 <= v <= 192: s = str(v - 128)
|
||||
elif 193 <= v <= 208: s = str(-(v - 192))
|
||||
elif v < 128: s = decode_src(v, cdna)
|
||||
elif v >= 256: s = f"v{v - 256}"
|
||||
else: s = decode_src(v, cdna)
|
||||
return f"-{s}" if neg else s
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# INSTRUCTION METADATA - fallback functions when inst.num_srcs()/inst.operands unavailable
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
def _num_srcs(inst) -> int:
|
||||
"""Fallback: get number of source operands from instruction name."""
|
||||
name = getattr(inst, 'op_name', '') or ''
|
||||
n = name.upper()
|
||||
# FMAC/MAC ops are 2-source (dst is implicit accumulator), but FMA/MAD ops are 3-source
|
||||
if 'FMAC' in n or 'V_MAC_' in n: return 2
|
||||
if any(x in n for x in ('FMA', 'MAD', 'CNDMASK', 'BFE', 'BFI', 'LERP', 'MED3', 'SAD', 'DIV_FMAS', 'DIV_FIXUP', 'DIV_SCALE', 'CUBE')): return 3
|
||||
# PERMLANE_VAR ops are 2-source, but PERMLANE (non-VAR) are 3-source
|
||||
if 'PERMLANE' in n and '_VAR' not in n: return 3
|
||||
if any(x in n for x in ('_ADD3', '_LSHL_ADD', '_ADD_LSHL', '_LSHL_OR', '_AND_OR', 'OR3_B32', 'AND_OR_B32', 'ALIGNBIT',
|
||||
'ALIGNBYTE', 'V_PERM_', 'XOR3', 'XAD', 'MULLIT', 'MINMAX', 'MAXMIN', 'MINIMUMMAXIMUM', 'MAXIMUMMINIMUM',
|
||||
'MINIMUM3', 'MAXIMUM3', 'MIN3', 'MAX3', 'DOT2', 'CVT_PK_U8_F32', 'DOT4', 'DOT8', 'WMMA', 'SWMMAC')): return 3
|
||||
return 2
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# IMPORTS
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
from tinygrad.runtime.autogen.amd.rdna3.ins import (VOP1, VOP1_SDST, VOP1_SDST_LIT, VOP1_LIT, VOP2, VOP2_LIT, VOP3, VOP3_SDST, VOP3_SDST_LIT,
|
||||
VOP3_LIT, VOP3SD, VOP3SD_LIT, VOP3P, VOP3P_LIT, VOPC, VOPC_LIT, VOPD, VOPD_LIT, VINTERP, SOP1, SOP1_LIT, SOP2, SOP2_LIT, SOPC, SOPC_LIT,
|
||||
SOPK, SOPK_LIT, SOPP, SMEM, DS, FLAT, GLOBAL, SCRATCH, VOP2Op, VOPDOp, SOPPOp, HWREG, MSG)
|
||||
from tinygrad.runtime.autogen.amd.rdna4.ins import (VOP1 as R4_VOP1, VOP1_SDST as R4_VOP1_SDST,
|
||||
VOP1_SDST_LIT as R4_VOP1_SDST_LIT, VOP1_LIT as R4_VOP1_LIT,
|
||||
VOP2 as R4_VOP2, VOP2_LIT as R4_VOP2_LIT, VOP3 as R4_VOP3, VOP3_SDST as R4_VOP3_SDST, VOP3_SDST_LIT as R4_VOP3_SDST_LIT, VOP3_LIT as R4_VOP3_LIT,
|
||||
VOP3SD as R4_VOP3SD, VOP3SD_LIT as R4_VOP3SD_LIT, VOP3P as R4_VOP3P, VOP3P_LIT as R4_VOP3P_LIT, VOPC as R4_VOPC, VOPC_LIT as R4_VOPC_LIT,
|
||||
VOPD as R4_VOPD, VOPD_LIT as R4_VOPD_LIT, VINTERP as R4_VINTERP, SOP1 as R4_SOP1, SOP1_LIT as R4_SOP1_LIT, SOP2 as R4_SOP2, SOP2_LIT as R4_SOP2_LIT,
|
||||
SOPC as R4_SOPC, SOPC_LIT as R4_SOPC_LIT, SOPK as R4_SOPK, SOPK_LIT as R4_SOPK_LIT, SOPP as R4_SOPP, SMEM as R4_SMEM, DS as R4_DS,
|
||||
VOPDOp as R4_VOPDOp, HWREG as HWREG_RDNA4, VFLAT as R4_FLAT, VGLOBAL as R4_GLOBAL, VSCRATCH as R4_SCRATCH)
|
||||
from tinygrad.runtime.autogen.amd.cdna.ins import HWREG as HWREG_CDNA
|
||||
|
||||
def _is_cdna(inst: Inst) -> bool: return 'cdna' in inst.__class__.__module__
|
||||
def _is_r4(inst: Inst) -> bool: return 'rdna4' in inst.__class__.__module__
|
||||
|
||||
# CDNA opcode name aliases for disasm (new name -> old name expected by tests)
|
||||
_CDNA_DISASM_ALIASES = {'v_fmac_f64': 'v_mul_legacy_f32', 'v_dot2c_f32_bf16': 'v_mac_f32', 'v_fmamk_f32': 'v_madmk_f32', 'v_fmaak_f32': 'v_madak_f32'}
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# HELPERS
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
def _reg(p: str, b: int, n: int = 1) -> str: return f"{p}{_unwrap(b)}" if n == 1 else f"{p}[{_unwrap(b)}:{_unwrap(b)+n-1}]"
|
||||
def _sreg(b: int, n: int = 1) -> str: return _reg("s", _unwrap(b), n)
|
||||
def _vreg(b: int, n: int = 1) -> str:
|
||||
b = _unwrap(b)
|
||||
return _reg("v", b - 256 if b >= 256 else b, n)
|
||||
def _areg(b: int, n: int = 1) -> str:
|
||||
b = _unwrap(b)
|
||||
return _reg("a", b - 256 if b >= 256 else b, n) # accumulator registers for GFX90a
|
||||
def _ttmp(b, n: int = 1) -> str | None:
|
||||
b = _unwrap(b)
|
||||
return _reg("ttmp", b - 108, n) if 108 <= b <= 123 else None
|
||||
|
||||
def _fmt_sdst(v, n: int = 1, cdna: bool = False) -> str:
|
||||
v = _unwrap(v)
|
||||
if t := _ttmp(v, n): return t
|
||||
pairs = SPECIAL_PAIRS_CDNA if cdna else SPECIAL_PAIRS
|
||||
gprs = SPECIAL_GPRS_CDNA if cdna else SPECIAL_GPRS
|
||||
if n > 1: return pairs.get(v) or gprs.get(v) or _sreg(v, n) # also check gprs for null/m0
|
||||
return gprs.get(v, f"s{v}")
|
||||
|
||||
def _fmt_src(v, n: int = 1, cdna: bool = False) -> str:
|
||||
v = _unwrap(v)
|
||||
if v == 253: return "src_scc" # SCC as source operand
|
||||
if n == 1: return decode_src(v, cdna)
|
||||
if v >= 256: return _vreg(v, n)
|
||||
if v <= 101: return _sreg(v, n) # s0-s101 can be pairs, but 102+ are special on CDNA
|
||||
pairs = SPECIAL_PAIRS_CDNA if cdna else SPECIAL_PAIRS
|
||||
if n == 2 and v in pairs: return pairs[v]
|
||||
if v <= 105: return _sreg(v, n) # s102-s105 regular pairs for RDNA
|
||||
if t := _ttmp(v, n): return t
|
||||
return decode_src(v, cdna)
|
||||
|
||||
def _fmt_v16(v, base: int = 256, hi_thresh: int = 384) -> str:
|
||||
v = _unwrap(v)
|
||||
return f"v{(v - base) & 0x7f}.{'h' if v >= hi_thresh else 'l'}"
|
||||
|
||||
def _has(op: str, *subs) -> bool: return any(s in op for s in subs)
|
||||
def _omod(v: int) -> str: return {1: " mul:2", 2: " mul:4", 3: " div:2"}.get(v, "")
|
||||
def _src16(inst, v: int) -> str:
|
||||
v = _unwrap(v)
|
||||
return _fmt_v16(v) if v >= 256 else _lit(inst, v) # format 16-bit src: vgpr.h/l or literal
|
||||
def _mods(*pairs) -> str: return " ".join(m for c, m in pairs if c)
|
||||
def _fmt_bits(label: str, val: int, count: int) -> str: return f"{label}:[{','.join(str((val >> i) & 1) for i in range(count))}]"
|
||||
|
||||
def _vop3_src(inst, v: int, neg: int, abs_: int, hi: int, n: int, f16: bool) -> str:
|
||||
"""Format VOP3 source operand with modifiers."""
|
||||
v = _unwrap(v)
|
||||
if v == 255: s = _lit(inst, v) # literal constant takes priority
|
||||
elif n > 1: s = _fmt_src(v, n)
|
||||
elif f16 and v >= 256: s = f"v{v - 256}.h" if hi else f"v{v - 256}.l"
|
||||
elif v == 253: s = "src_scc" # VOP3 sources use src_scc not scc
|
||||
else: s = _lit(inst, v)
|
||||
if abs_: s = f"|{s}|"
|
||||
return f"-{s}" if neg else s
|
||||
|
||||
def _opsel_str(opsel: int, n: int, need: bool, is16_d: bool) -> str:
|
||||
"""Format op_sel modifier string."""
|
||||
if not need: return ""
|
||||
dst_hi = (opsel >> 3) & 1
|
||||
if n == 1: return f" op_sel:[{opsel & 1},{dst_hi}]"
|
||||
# Use 4-element format if bit 2 is set (src2 selection used) or if 3+ sources
|
||||
if n == 2 and not ((opsel >> 2) & 1): return f" op_sel:[{opsel & 1},{(opsel >> 1) & 1},{dst_hi}]"
|
||||
return f" op_sel:[{opsel & 1},{(opsel >> 1) & 1},{(opsel >> 2) & 1},{dst_hi}]"
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# DISASSEMBLER
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
def _disasm_vop1(inst: VOP1) -> str:
|
||||
name, cdna = inst.op_name.lower() or f'vop1_op_{inst.op}', _is_cdna(inst)
|
||||
name = name.replace('_e32', '') # Strip _e32 suffix
|
||||
if any(x in name for x in ('v_nop', 'v_pipeflush', 'v_clrexcp')): return name # no operands
|
||||
if 'readfirstlane' in name:
|
||||
src = inst.src0.fmt() if inst.src0.offset >= 256 else decode_src(inst.src0.offset, cdna)
|
||||
vdst_off = inst.vdst.offset - 256 if inst.vdst.offset >= 256 else inst.vdst.offset
|
||||
return f"{name} {_fmt_sdst(vdst_off, 1, cdna)}, {src}"
|
||||
bits = inst.canonical_op_bits
|
||||
is16_dst, is16_src = not cdna and bits['d'] == 16, not cdna and bits['s0'] == 16
|
||||
# Format dst
|
||||
if is16_dst: dst = _fmt_v16(inst.vdst)
|
||||
else: dst = inst.vdst.fmt()
|
||||
# Format src
|
||||
if inst.src0.offset == 255: src = _lit(inst, inst.src0)
|
||||
elif is16_src and inst.src0.offset >= 256: src = _fmt_v16(inst.src0)
|
||||
elif inst.src0.sz > 1: src = _fmt_src(inst.src0, inst.src0.sz, cdna)
|
||||
else: src = _lit(inst, inst.src0)
|
||||
return f"{name} {dst}, {src}"
|
||||
|
||||
_VOP2_CARRY_OUT = {'v_add_co_u32', 'v_sub_co_u32', 'v_subrev_co_u32'} # carry out only
|
||||
_VOP2_CARRY_INOUT = {'v_addc_co_u32', 'v_subb_co_u32', 'v_subbrev_co_u32'} # carry in and out (CDNA)
|
||||
_VOP2_CARRY_INOUT_RDNA = {'v_add_co_ci_u32', 'v_sub_co_ci_u32', 'v_subrev_co_ci_u32'} # carry in and out (RDNA)
|
||||
def _disasm_vop2(inst: VOP2) -> str:
|
||||
name, cdna = inst.op_name.lower(), _is_cdna(inst)
|
||||
if cdna: name = _CDNA_DISASM_ALIASES.get(name, name) # apply CDNA aliases
|
||||
suf = "" if cdna or name.endswith('_e32') or (not cdna and inst.op == VOP2Op.V_DOT2ACC_F32_F16_E32) else "_e32"
|
||||
lit = inst._literal
|
||||
is16 = not cdna and inst.canonical_op_bits['d'] == 16
|
||||
# fmaak/madak: dst = src0 * vsrc1 + K, fmamk/madmk: dst = src0 * K + vsrc1
|
||||
if 'fmaak' in name or 'madak' in name or (not cdna and inst.op in (VOP2Op.V_FMAAK_F32_E32, VOP2Op.V_FMAAK_F16_E32)):
|
||||
if lit is None: return f"op_{inst.op.value if hasattr(inst.op, 'value') else inst.op}"
|
||||
if is16: return f"{name}{suf} {_fmt_v16(inst.vdst)}, {_src16(inst, inst.src0)}, {_fmt_v16(inst.vsrc1)}, 0x{lit:x}"
|
||||
return f"{name}{suf} {inst.vdst.fmt()}, {_lit(inst, inst.src0)}, {inst.vsrc1.fmt()}, 0x{lit:x}"
|
||||
if 'fmamk' in name or 'madmk' in name or (not cdna and inst.op in (VOP2Op.V_FMAMK_F32_E32, VOP2Op.V_FMAMK_F16_E32)):
|
||||
if lit is None: return f"op_{inst.op.value if hasattr(inst.op, 'value') else inst.op}"
|
||||
if is16: return f"{name}{suf} {_fmt_v16(inst.vdst)}, {_src16(inst, inst.src0)}, 0x{lit:x}, {_fmt_v16(inst.vsrc1)}"
|
||||
return f"{name}{suf} {inst.vdst.fmt()}, {_lit(inst, inst.src0)}, 0x{lit:x}, {inst.vsrc1.fmt()}"
|
||||
if is16: return f"{name}{suf} {_fmt_v16(inst.vdst)}, {_src16(inst, inst.src0)}, {_fmt_v16(inst.vsrc1)}"
|
||||
vcc = "vcc" if cdna else "vcc_lo"
|
||||
basename = name.replace('_e32', '')
|
||||
if cdna and basename in _VOP2_CARRY_OUT: return f"{name}{suf} {inst.vdst.fmt()}, {vcc}, {_lit(inst, inst.src0)}, {inst.vsrc1.fmt()}"
|
||||
if cdna and basename in _VOP2_CARRY_INOUT: return f"{name}{suf} {inst.vdst.fmt()}, {vcc}, {_lit(inst, inst.src0)}, {inst.vsrc1.fmt()}, {vcc}"
|
||||
if not cdna and basename in _VOP2_CARRY_INOUT_RDNA:
|
||||
return f"{name}{suf} {inst.vdst.fmt()}, {vcc}, {_lit(inst, inst.src0)}, {inst.vsrc1.fmt()}, {vcc}"
|
||||
sn0 = inst.canonical_op_regs.get('s0', 1)
|
||||
if inst.vdst.sz > 1 or sn0 > 1 or inst.vsrc1.sz > 1:
|
||||
src0 = _lit(inst, inst.src0) if inst.src0.offset == 255 else _fmt_src(inst.src0, sn0, cdna)
|
||||
return f"{name.replace('_e32', '')} {inst.vdst.fmt()}, {src0}, {inst.vsrc1.fmt()}"
|
||||
return f"{name}{suf} {inst.vdst.fmt()}, {_lit(inst, inst.src0)}, {inst.vsrc1.fmt()}" + (f", {vcc}" if name == 'v_cndmask_b32' else "")
|
||||
|
||||
def _disasm_vopc(inst: VOPC) -> str:
|
||||
name, cdna = inst.op_name.lower(), _is_cdna(inst)
|
||||
bits = inst.canonical_op_bits
|
||||
is16 = bits['s0'] == 16
|
||||
if cdna:
|
||||
s0 = _lit(inst, inst.src0) if inst.src0.offset == 255 else _fmt_src(inst.src0, inst.src0.sz, cdna)
|
||||
return f"{name} vcc, {s0}, {inst.vsrc1.fmt()}" # CDNA VOPC always outputs vcc
|
||||
# RDNA: v_cmpx_* writes to exec (no vcc), v_cmp_* writes to vcc_lo
|
||||
has_vcc = 'cmpx' not in name
|
||||
if inst.src0.offset == 255: s0 = _lit(inst, inst.src0)
|
||||
elif inst.src0.sz > 1: s0 = inst.src0.fmt()
|
||||
elif is16: s0 = _src16(inst, inst.src0.offset)
|
||||
else: s0 = _lit(inst, inst.src0)
|
||||
s1 = inst.vsrc1.fmt() if inst.vsrc1.sz > 1 else _fmt_v16(inst.vsrc1) if is16 else inst.vsrc1.fmt()
|
||||
suf = "" if name.endswith('_e32') else "_e32"
|
||||
return f"{name}{suf} vcc_lo, {s0}, {s1}" if has_vcc else f"{name}{suf} {s0}, {s1}"
|
||||
|
||||
NO_ARG_SOPP = {SOPPOp.S_BARRIER, SOPPOp.S_WAKEUP, SOPPOp.S_ICACHE_INV,
|
||||
SOPPOp.S_WAIT_IDLE, SOPPOp.S_ENDPGM_SAVED, SOPPOp.S_CODE_END, SOPPOp.S_ENDPGM_ORDERED_PS_DONE, SOPPOp.S_TTRACEDATA}
|
||||
|
||||
def _disasm_sopp(inst: SOPP) -> str:
|
||||
name, cdna = inst.op_name.lower(), _is_cdna(inst)
|
||||
is_rdna4 = _is_r4(inst)
|
||||
# Ops that have no argument when simm16 == 0
|
||||
no_arg_zero = {'s_barrier', 's_wakeup', 's_icache_inv', 's_ttracedata', 's_wait_idle', 's_endpgm_saved',
|
||||
's_endpgm_ordered_ps_done', 's_code_end'}
|
||||
if name in no_arg_zero: return name if inst.simm16 == 0 else f"{name} {inst.simm16}"
|
||||
if name == 's_endpgm': return name if inst.simm16 == 0 else f"{name} {inst.simm16}"
|
||||
if cdna:
|
||||
if name == 's_waitcnt':
|
||||
# GFX9 format: vmcnt[3:0]=bits[3:0], vmcnt[5:4]=bits[15:14], expcnt=bits[6:4], lgkmcnt=bits[11:8] (4 bits, max 15)
|
||||
vm_lo, exp, lgkm, vm_hi = inst.simm16 & 0xf, (inst.simm16 >> 4) & 0x7, (inst.simm16 >> 8) & 0xf, (inst.simm16 >> 14) & 0x3
|
||||
vm = vm_lo | (vm_hi << 4)
|
||||
p = [f"vmcnt({vm})" if vm != 0x3f else "", f"expcnt({exp})" if exp != 7 else "", f"lgkmcnt({lgkm})" if lgkm != 0xf else ""]
|
||||
return f"s_waitcnt {' '.join(x for x in p if x) or '0'}"
|
||||
if name.startswith(('s_cbranch', 's_branch')): return f"{name} {inst.simm16}"
|
||||
if name == 's_set_gpr_idx_mode':
|
||||
flags = [n for i, n in enumerate(['SRC0', 'SRC1', 'SRC2', 'DST']) if inst.simm16 & (1 << i)]
|
||||
return f"{name} gpr_idx({','.join(flags)})"
|
||||
return f"{name} 0x{inst.simm16:x}" if inst.simm16 else name
|
||||
# RDNA (use name-based checks instead of enum-based for cross-arch compatibility)
|
||||
if name == 's_waitcnt':
|
||||
if is_rdna4:
|
||||
return f"{name} {inst.simm16}" if inst.simm16 else f"{name} 0"
|
||||
vm, exp, lgkm = (inst.simm16 >> 10) & 0x3f, inst.simm16 & 0xf, (inst.simm16 >> 4) & 0x3f
|
||||
p = [f"vmcnt({vm})" if vm != 0x3f else "", f"expcnt({exp})" if exp != 7 else "", f"lgkmcnt({lgkm})" if lgkm != 0x3f else ""]
|
||||
return f"s_waitcnt {' '.join(x for x in p if x) or '0'}"
|
||||
if name == 's_delay_alu':
|
||||
deps = ['VALU_DEP_1','VALU_DEP_2','VALU_DEP_3','VALU_DEP_4','TRANS32_DEP_1','TRANS32_DEP_2',
|
||||
'TRANS32_DEP_3','FMA_ACCUM_CYCLE_1','SALU_CYCLE_1','SALU_CYCLE_2','SALU_CYCLE_3']
|
||||
skips = ['SAME','NEXT','SKIP_1','SKIP_2','SKIP_3','SKIP_4']
|
||||
id0, skip, id1 = inst.simm16 & 0xf, (inst.simm16 >> 4) & 0x7, (inst.simm16 >> 7) & 0xf
|
||||
def dep(v): return deps[v-1] if 0 < v <= len(deps) else str(v)
|
||||
p = [f"instid0({dep(id0)})" if id0 else "", f"instskip({skips[skip]})" if skip else "", f"instid1({dep(id1)})" if id1 else ""]
|
||||
return f"s_delay_alu {' | '.join(x for x in p if x) or '0'}"
|
||||
if name.startswith(('s_cbranch', 's_branch')): return f"{name} {inst.simm16}"
|
||||
return f"{name} 0x{inst.simm16:x}"
|
||||
|
||||
def _disasm_smem(inst: SMEM) -> str:
|
||||
name, cdna = inst.op_name.lower(), _is_cdna(inst)
|
||||
if name in ('s_gl1_inv', 's_dcache_inv', 's_dcache_inv_vol', 's_dcache_wb', 's_dcache_wb_vol', 's_icache_inv'): return name
|
||||
soe, imm = getattr(inst, 'soe', 0) or getattr(inst, 'soffset_en', 0), getattr(inst, 'imm', 1)
|
||||
is_rdna4 = _is_r4(inst)
|
||||
offset = inst.ioffset if is_rdna4 else getattr(inst, 'offset', 0) # type: ignore[attr-defined]
|
||||
if cdna:
|
||||
if soe and imm: off_s = f"{decode_src(inst.soffset, cdna)} offset:0x{offset:x}"
|
||||
elif imm: off_s = f"0x{offset:x}"
|
||||
elif offset < 256: off_s = decode_src(offset, cdna)
|
||||
else: off_s = decode_src(inst.soffset, cdna)
|
||||
elif offset and inst.soffset != 124: off_s = f"{decode_src(inst.soffset, cdna)} offset:0x{offset:x}"
|
||||
elif offset: off_s = f"0x{offset:x}"
|
||||
else: off_s = decode_src(inst.soffset, cdna)
|
||||
is_buffer = 'buffer' in name or 's_atc_probe_buffer' == name
|
||||
sbase_idx, sbase_count = _unwrap(inst.sbase), 4 if is_buffer else 2
|
||||
if sbase_count == 2: sbase_str = _fmt_src(sbase_idx, sbase_count, cdna)
|
||||
elif sbase_idx <= 105: sbase_str = _sreg(sbase_idx, sbase_count)
|
||||
else: sbase_str = _reg("ttmp", sbase_idx - 108, sbase_count)
|
||||
if name in ('s_atc_probe', 's_atc_probe_buffer'): return f"{name} {_unwrap(inst.sdata)}, {sbase_str}, {off_s}"
|
||||
if 'prefetch' in name:
|
||||
off = getattr(inst, 'ioffset', getattr(inst, 'offset', 0))
|
||||
if off >= 0x800000: off = off - 0x1000000
|
||||
off_s = f"0x{off:x}" if off > 255 else str(off)
|
||||
soff_s = decode_src(inst.soffset, cdna) if inst.soffset != 124 else ("m0" if cdna else "null")
|
||||
if 'pc_rel' in name: return f"{name} {off_s}, {soff_s}, {_unwrap(inst.sdata)}"
|
||||
return f"{name} {sbase_str}, {off_s}, {soff_s}, {_unwrap(inst.sdata)}"
|
||||
# Use get_field_bits for register count
|
||||
dst_n = inst.canonical_op_regs.get('d', 1)
|
||||
th, scope = getattr(inst, 'th', 0), getattr(inst, 'scope', 0)
|
||||
if is_rdna4: # RDNA4 uses th/scope instead of glc/dlc
|
||||
th_names = ['TH_LOAD_RT', 'TH_LOAD_NT', 'TH_LOAD_HT', 'TH_LOAD_LU']
|
||||
scope_names = ['SCOPE_CU', 'SCOPE_SE', 'SCOPE_DEV', 'SCOPE_SYS']
|
||||
mods = (f" th:{th_names[th]}" if th else "") + (f" scope:{scope_names[scope]}" if scope else "")
|
||||
return f"{name} {_fmt_sdst(inst.sdata, dst_n, cdna)}, {sbase_str}, {off_s}{mods}"
|
||||
if th or scope:
|
||||
th_names = ['TH_LOAD_RT', 'TH_LOAD_NT', 'TH_LOAD_HT', 'TH_LOAD_LU']
|
||||
scope_names = ['SCOPE_CU', 'SCOPE_SE', 'SCOPE_DEV', 'SCOPE_SYS']
|
||||
mods = (f" th:{th_names[th]}" if th else "") + (f" scope:{scope_names[scope]}" if scope else "")
|
||||
return f"{name} {_fmt_sdst(inst.sdata, dst_n, cdna)}, {sbase_str}, {off_s}{mods}"
|
||||
if 'discard' in name: return f"{name} {sbase_str}, {off_s}" + _mods((inst.glc, " glc"), (getattr(inst, 'dlc', 0), " dlc"))
|
||||
if name in ('s_memrealtime', 's_memtime'): return f"{name} {_fmt_sdst(inst.sdata, dst_n, cdna)}"
|
||||
return f"{name} {_fmt_sdst(inst.sdata, dst_n, cdna)}, {sbase_str}, {off_s}" + _mods((inst.glc, " glc"), (getattr(inst, 'dlc', 0), " dlc"))
|
||||
|
||||
R4_TH_LOAD = {1: 'TH_LOAD_NT', 2: 'TH_LOAD_HT', 3: 'TH_LOAD_LU', 4: 'TH_LOAD_RT_WB', 5: 'TH_LOAD_NT_WB'}
|
||||
R4_TH_STORE = {1: 'TH_STORE_NT', 2: 'TH_STORE_HT', 3: 'TH_STORE_ST', 4: 'TH_STORE_RT_WB', 5: 'TH_STORE_NT_WB'}
|
||||
R4_TH_ATOMIC = {1: 'TH_ATOMIC_RETURN', 2: 'TH_ATOMIC_NT', 3: 'TH_ATOMIC_RETURN_NT',
|
||||
4: 'TH_ATOMIC_CASCADE_RT', 5: 'TH_ATOMIC_CASCADE_RETURN', 6: 'TH_ATOMIC_CASCADE_NT', 7: 'TH_ATOMIC_CASCADE_RETURN_NT'}
|
||||
R4_SCOPE = {1: 'SCOPE_SE', 2: 'SCOPE_DEV', 3: 'SCOPE_SYS'}
|
||||
|
||||
def _disasm_flat(inst: FLAT) -> str:
|
||||
name, cdna, r4 = inst.op_name.lower(), _is_cdna(inst), _is_r4(inst)
|
||||
acc = getattr(inst, 'acc', 0)
|
||||
reg_fn = _areg if acc else _vreg
|
||||
if r4: seg = 'flat' if (cls_name:=inst.__class__.__name__) == 'VFLAT' else ('global' if cls_name == 'VGLOBAL' else 'scratch')
|
||||
else: seg = ['flat', 'scratch', 'global'][inst.seg] if inst.seg < 3 else 'flat'
|
||||
instr = f"{seg}_{name.split('_', 1)[1] if '_' in name else name}"
|
||||
# Global/scratch uses 13-bit signed offset (RDNA3/CDNA), 24-bit signed offset (RDNA4)
|
||||
offset = inst.ioffset if r4 else inst.offset # type: ignore[attr-defined]
|
||||
if r4: off_val = offset if offset < (1 << 23) else offset - (1 << 24) # sign extend 24-bit
|
||||
elif seg != 'flat':
|
||||
if cdna:
|
||||
# CDNA: bit 12 is sign bit but not in offset field
|
||||
raw = int.from_bytes(inst.to_bytes(), 'little')
|
||||
off_val = offset | ((raw >> 12) & 1) << 12 # get bit 12
|
||||
else:
|
||||
off_val = offset
|
||||
off_val = off_val if off_val < 4096 else off_val - 8192 # sign extend 13-bit
|
||||
else:
|
||||
off_val = offset
|
||||
# Use get_field_bits: data for stores/atomics, d for loads
|
||||
regs = inst.canonical_op_regs
|
||||
w = regs.get('data', regs.get('d', 1)) if 'store' in name or 'atomic' in name else regs.get('d', 1)
|
||||
off_s = f" offset:{off_val}" if off_val else ""
|
||||
if cdna: mods = f"{off_s}{' sc0' if inst.sc0 else ''}{' nt' if inst.nt else ''}{' sc1' if getattr(inst, 'sc1', 0) else ''}" # type: ignore[attr-defined]
|
||||
elif r4:
|
||||
th_names = R4_TH_ATOMIC if 'atomic' in name else (R4_TH_STORE if 'store' in name else R4_TH_LOAD)
|
||||
mods = off_s + (f" th:{th_names[inst.th]}" if inst.th in th_names else "") + (f" scope:{R4_SCOPE[inst.scope]}" if inst.scope in R4_SCOPE else "")
|
||||
else: mods = f"{off_s}{' glc' if inst.glc else ''}{' slc' if inst.slc else ''}{' dlc' if inst.dlc else ''}"
|
||||
if seg == 'flat': saddr_s = ""
|
||||
elif _unwrap(inst.saddr) in (0x7F, 124): saddr_s = ", off"
|
||||
elif seg == 'scratch': saddr_s = f", {decode_src(inst.saddr, cdna)}"
|
||||
elif _unwrap(inst.saddr) in (SPECIAL_PAIRS_CDNA if cdna else SPECIAL_PAIRS):
|
||||
saddr_s = f", {(SPECIAL_PAIRS_CDNA if cdna else SPECIAL_PAIRS)[_unwrap(inst.saddr)]}"
|
||||
elif t := _ttmp(inst.saddr, 2): saddr_s = f", {t}"
|
||||
else: saddr_s = f", {_sreg(inst.saddr, 2) if _unwrap(inst.saddr) < 106 else decode_src(_unwrap(inst.saddr), cdna)}"
|
||||
if 'addtid' in name: return f"{instr} {reg_fn((inst.vsrc if r4 else inst.data) if 'store' in name else inst.vdst)}{saddr_s}{mods}"
|
||||
# RDNA4: vaddr instead of addr, vsrc instead of data
|
||||
addr = inst.vaddr if r4 else inst.addr # type: ignore[attr-defined]
|
||||
data = inst.vsrc if r4 else inst.data # type: ignore[attr-defined]
|
||||
# load_lds_* instructions: vaddr, saddr (no vdst, data goes to LDS)
|
||||
if 'load_lds' in name:
|
||||
addr_w = 1 if seg == 'scratch' or (_unwrap(inst.saddr) not in (0x7F, 124)) else 2
|
||||
addr_s = "off" if not inst.sve and seg == 'scratch' else _vreg(addr, addr_w)
|
||||
return f"{instr} {addr_s}{saddr_s}{mods}"
|
||||
if seg == 'flat': addr_w = 2 # flat always uses 64-bit vaddr
|
||||
elif cdna: addr_w = 1 if seg == 'scratch' or (_unwrap(inst.saddr) not in (0x7F, 124)) else 2
|
||||
else: addr_w = 1 if seg == 'scratch' or (_unwrap(inst.saddr) not in (0x7F, 124)) else 2
|
||||
addr_s = "off" if not inst.sve and seg == 'scratch' else _vreg(addr, addr_w)
|
||||
data_s, vdst_s = reg_fn(data, w), reg_fn(inst.vdst, w // 2 if 'cmpswap' in name else w)
|
||||
if 'atomic' in name:
|
||||
glc_or_sc0 = inst.sc0 if cdna else (inst.th & 1 if r4 else inst.glc) # type: ignore[attr-defined]
|
||||
sfx = f"{saddr_s if seg != 'flat' else ''}{mods}"
|
||||
return f"{instr} {vdst_s}, {addr_s}, {data_s}{sfx}" if glc_or_sc0 else f"{instr} {addr_s}, {data_s}{sfx}"
|
||||
if 'store' in name: return f"{instr} {addr_s}, {data_s}{saddr_s}{mods}"
|
||||
return f"{instr} {reg_fn(inst.vdst, w)}, {addr_s}{saddr_s}{mods}"
|
||||
|
||||
def _disasm_ds(inst: DS) -> str:
|
||||
name = inst.op_name.lower()
|
||||
acc = getattr(inst, 'acc', 0)
|
||||
reg_fn = _areg if acc else _vreg
|
||||
gds = " gds" if getattr(inst, 'gds', 0) else ""
|
||||
off = f" offset:{inst.offset0 | (inst.offset1 << 8)}" if inst.offset0 or inst.offset1 else ""
|
||||
off2 = (" offset0:" + str(inst.offset0) if inst.offset0 else "") + (" offset1:" + str(inst.offset1) if inst.offset1 else "")
|
||||
# Use get_field_bits: data for stores/writes/atomics, d for loads
|
||||
regs = inst.canonical_op_regs
|
||||
w = regs.get('data', regs.get('d', 1)) if 'store' in name or 'write' in name or ('load' not in name and 'read' not in name) else regs.get('d', 1)
|
||||
d0, d1, dst, addr = reg_fn(inst.data0, w), reg_fn(inst.data1, w), reg_fn(inst.vdst, w), _vreg(inst.addr)
|
||||
|
||||
if name == 'ds_nop': return name
|
||||
if name == 'ds_bvh_stack_rtn_b32': return f"{name} {_vreg(inst.vdst)}, {addr}, {_vreg(inst.data0)}, {_vreg(inst.data1, 4)}{off}{gds}"
|
||||
if 'bvh_stack_push' in name:
|
||||
d1_regs = 8 if 'push8' in name else 4
|
||||
vdst_regs = 2 if 'pop2' in name else 1
|
||||
vdst_s = _vreg(inst.vdst, vdst_regs) if vdst_regs > 1 else _vreg(inst.vdst)
|
||||
return f"{name} {vdst_s}, {addr}, {_vreg(inst.data0)}, {_vreg(inst.data1, d1_regs)}{off}{gds}"
|
||||
if 'gws_sema' in name and 'sema_br' not in name: return f"{name}{off}{gds}"
|
||||
if 'gws_' in name: return f"{name} {addr}{off}{gds}"
|
||||
if name in ('ds_consume', 'ds_append'): return f"{name} {reg_fn(inst.vdst)}{off}{gds}"
|
||||
if 'gs_reg' in name: return f"{name} {reg_fn(inst.vdst, 2)}, {reg_fn(inst.data0)}{off}{gds}"
|
||||
if '2addr' in name:
|
||||
if 'load' in name: return f"{name} {reg_fn(inst.vdst, regs.get('d', 1))}, {addr}{off2}{gds}"
|
||||
if 'store' in name and 'xchg' not in name: return f"{name} {addr}, {d0}, {d1}{off2}{gds}"
|
||||
return f"{name} {reg_fn(inst.vdst, regs.get('d', 1))}, {addr}, {d0}, {d1}{off2}{gds}"
|
||||
if 'write2' in name: return f"{name} {addr}, {d0}, {d1}{off2}{gds}"
|
||||
if 'read2' in name: return f"{name} {reg_fn(inst.vdst, regs.get('d', 1))}, {addr}{off2}{gds}"
|
||||
if 'xchg2' in name: return f"{name} {reg_fn(inst.vdst, regs.get('d', 1))}, {addr}, {d0}, {d1}{off2}{gds}"
|
||||
if 'load' in name or ('read' in name and 'read2' not in name):
|
||||
return f"{name} {reg_fn(inst.vdst)}{off}{gds}" if 'addtid' in name else f"{name} {dst}, {addr}{off}{gds}"
|
||||
if ('store' in name or 'write' in name) and not _has(name, 'cmp', 'xchg', 'write2'):
|
||||
return f"{name} {reg_fn(inst.data0)}{off}{gds}" if 'addtid' in name else f"{name} {addr}, {d0}{off}{gds}"
|
||||
if 'swizzle' in name or name == 'ds_ordered_count': return f"{name} {reg_fn(inst.vdst)}, {addr}{off}{gds}"
|
||||
if 'permute' in name: return f"{name} {reg_fn(inst.vdst)}, {addr}, {reg_fn(inst.data0)}{off}{gds}"
|
||||
if 'condxchg' in name: return f"{name} {reg_fn(inst.vdst, 2)}, {addr}, {reg_fn(inst.data0, 2)}{off}{gds}"
|
||||
if _has(name, 'cmpst', 'mskor', 'wrap'):
|
||||
return f"{name} {dst}, {addr}, {d0}, {d1}{off}{gds}" if '_rtn' in name else f"{name} {addr}, {d0}, {d1}{off}{gds}"
|
||||
return f"{name} {dst}, {addr}, {d0}{off}{gds}" if '_rtn' in name else f"{name} {addr}, {d0}{off}{gds}"
|
||||
|
||||
def _disasm_vop3(inst: VOP3) -> str:
|
||||
name = inst.op_name.lower()
|
||||
bits = inst.canonical_op_bits
|
||||
|
||||
# RDNA4 v_s_* scalar VOP3 instructions - vdst is SGPR (VGPRField adds 256)
|
||||
if name.startswith('v_s_'):
|
||||
s0v = _unwrap(inst.src0)
|
||||
if s0v == 255: src = _lit(inst, inst.src0)
|
||||
elif s0v == 253: src = "src_scc"
|
||||
else: src = _fmt_src(inst.src0, max(1, bits['s0'] // 32))
|
||||
if inst.neg & 1: src = f"-{src}"
|
||||
if inst.abs & 1: src = f"|{src}|"
|
||||
clamp = getattr(inst, 'cm', None) or getattr(inst, 'clmp', 0)
|
||||
vdst_raw = _unwrap(inst.vdst)
|
||||
return f"{name} s{vdst_raw - 256 if vdst_raw >= 256 else vdst_raw}, {src}" + (" clamp" if clamp else "") + _omod(inst.omod)
|
||||
|
||||
# Use get_field_bits for register sizes and 16-bit detection
|
||||
r0, r1, r2 = max(1, bits['s0'] // 32), max(1, bits['s1'] // 32), max(1, bits['s2'] // 32)
|
||||
is16_d, is16_s, is16_s2 = bits['d'] == 16, bits['s0'] == 16, bits['s2'] == 16
|
||||
|
||||
s0 = _vop3_src(inst, inst.src0, inst.neg&1, inst.abs&1, inst.opsel&1, r0, is16_s)
|
||||
s1 = _vop3_src(inst, inst.src1, inst.neg&2, inst.abs&2, inst.opsel&2, r1, is16_s)
|
||||
s2 = _vop3_src(inst, inst.src2, inst.neg&4, inst.abs&4, inst.opsel&4, r2, is16_s2)
|
||||
|
||||
# Format destination
|
||||
if 'readlane' in name:
|
||||
vdst_off = inst.vdst.offset - 256 if inst.vdst.offset >= 256 else inst.vdst.offset
|
||||
dst = _fmt_sdst(vdst_off, 1)
|
||||
elif is16_d: dst = f"{inst.vdst.fmt()}.h" if (inst.opsel & 8) else f"{inst.vdst.fmt()}.l"
|
||||
else: dst = inst.vdst.fmt()
|
||||
|
||||
clamp = getattr(inst, 'cm', None) or getattr(inst, 'clmp', 0)
|
||||
cl, om = " clamp" if clamp else "", _omod(inst.omod)
|
||||
nonvgpr_opsel = ((inst.src0.offset < 256 and (inst.opsel & 1)) or (inst.src1.offset < 256 and (inst.opsel & 2))
|
||||
or (inst.src2.offset < 256 and (inst.opsel & 4)))
|
||||
need_opsel = nonvgpr_opsel or (inst.opsel and not is16_s)
|
||||
|
||||
op_val = inst.op.value if hasattr(inst.op, 'value') else inst.op
|
||||
e64 = "" if name.endswith('_e64') else "_e64"
|
||||
if op_val < 256: # VOPC
|
||||
vdst_off = inst.vdst.offset - 256 if inst.vdst.offset >= 256 else inst.vdst.offset
|
||||
return f"{name}{e64} {s0}, {s1}{cl}" if name.startswith('v_cmpx') else f"{name}{e64} {_fmt_sdst(vdst_off, 1)}, {s0}, {s1}{cl}"
|
||||
if op_val < 384: # VOP2
|
||||
n = inst.num_srcs() or 2
|
||||
os = _opsel_str(inst.opsel, n, need_opsel, is16_d)
|
||||
return f"{name}{e64} {dst}, {s0}, {s1}, {s2}{os}{cl}{om}" if n == 3 else f"{name}{e64} {dst}, {s0}, {s1}{os}{cl}{om}"
|
||||
if op_val < 512: # VOP1
|
||||
if re.match(r'v_cvt_f32_(bf|fp)8', name) and inst.opsel:
|
||||
os = f" byte_sel:{((inst.opsel & 1) << 1) | ((inst.opsel >> 1) & 1)}"
|
||||
else:
|
||||
os = _opsel_str(inst.opsel, 1, need_opsel, is16_d)
|
||||
if 'v_nop' in name or 'v_pipeflush' in name: return f"{name}{e64}"
|
||||
return f"{name}{e64} {dst}, {s0}{os}{cl}{om}"
|
||||
# Native VOP3
|
||||
n = inst.num_srcs() or 2
|
||||
os = f" byte_sel:{inst.opsel >> 2}" if 'cvt_sr' in name and inst.opsel else _opsel_str(inst.opsel, n, need_opsel, is16_d)
|
||||
return f"{name} {dst}, {s0}, {s1}, {s2}{os}{cl}{om}" if n == 3 else f"{name} {dst}, {s0}, {s1}{os}{cl}{om}"
|
||||
|
||||
def _disasm_vop3sd(inst: VOP3SD) -> str:
|
||||
name = inst.op_name.lower()
|
||||
def src(reg, neg):
|
||||
s = _lit(inst, reg.offset) if reg.offset == 255 else ("src_scc" if reg.offset == 253 else (reg.fmt() if reg.sz > 1 else _lit(inst, reg.offset)))
|
||||
return f"neg({s})" if neg and reg.offset == 255 else (f"-{s}" if neg else s)
|
||||
s0, s1, s2 = src(inst.src0, inst.neg & 1), src(inst.src1, inst.neg & 2), src(inst.src2, inst.neg & 4)
|
||||
# VOP3SD: _co_ ops (add/sub) without _ci_ have only 2 sources, all others (mad, div_scale, _co_ci_) have 3 sources
|
||||
has_only_two_srcs = '_co_' in name and '_ci_' not in name and 'mad' not in name
|
||||
srcs = f"{s0}, {s1}" if has_only_two_srcs else f"{s0}, {s1}, {s2}"
|
||||
clamp = getattr(inst, 'cm', None) or getattr(inst, 'clmp', 0)
|
||||
return f"{name} {inst.vdst.fmt()}, {_fmt_sdst(inst.sdst, 1)}, {srcs}{' clamp' if clamp else ''}{_omod(inst.omod)}"
|
||||
|
||||
def _disasm_vopd(inst: VOPD) -> str:
|
||||
lit = inst._literal
|
||||
op_enum = R4_VOPDOp if _is_r4(inst) else VOPDOp
|
||||
nx, ny = op_enum(inst.opx).name.lower(), op_enum(inst.opy).name.lower()
|
||||
def half(n, vd, s0, vs1):
|
||||
vd, vs1 = _vi(vd), _vi(vs1)
|
||||
if 'mov' in n: return f"{n} v{vd}, {_lit(inst, s0)}"
|
||||
if 'fmamk' in n and lit: return f"{n} v{vd}, {_lit(inst, s0)}, 0x{lit:x}, v{vs1}"
|
||||
if 'fmaak' in n and lit: return f"{n} v{vd}, {_lit(inst, s0)}, v{vs1}, 0x{lit:x}"
|
||||
return f"{n} v{vd}, {_lit(inst, s0)}, v{vs1}"
|
||||
return f"{half(nx, inst.vdstx, inst.srcx0, inst.vsrcx1)} :: {half(ny, inst.vdsty, inst.srcy0, inst.vsrcy1)}"
|
||||
|
||||
def _disasm_vop3p(inst: VOP3P) -> str:
|
||||
name = inst.op_name.lower()
|
||||
is_swmmac, n, is_fma_mix = 'swmmac' in name, inst.num_srcs() or 2, 'fma_mix' in name
|
||||
def get_src(reg):
|
||||
return _lit(inst, reg.offset) if reg.offset == 255 else reg.fmt()
|
||||
src0, src1, src2, dst = get_src(inst.src0), get_src(inst.src1), get_src(inst.src2), inst.vdst.fmt()
|
||||
opsel_hi = inst.opsel_hi | (inst.opsel_hi2 << 2)
|
||||
clamp = getattr(inst, 'cm', None) or getattr(inst, 'clmp', 0)
|
||||
if is_fma_mix:
|
||||
def m(s, neg, abs_): return f"-{f'|{s}|' if abs_ else s}" if neg else (f"|{s}|" if abs_ else s)
|
||||
src0, src1, src2 = m(src0, inst.neg & 1, inst.neg_hi & 1), m(src1, inst.neg & 2, inst.neg_hi & 2), m(src2, inst.neg & 4, inst.neg_hi & 4)
|
||||
mods = (([_fmt_bits("op_sel", inst.opsel, n)] if inst.opsel else [])
|
||||
+ ([_fmt_bits("op_sel_hi", opsel_hi, n)] if opsel_hi else []) + (["clamp"] if clamp else []))
|
||||
elif is_swmmac:
|
||||
mods = ([f"index_key:{inst.opsel}"] if inst.opsel else []) + ([_fmt_bits("neg_lo", inst.neg, n)] if inst.neg else []) + \
|
||||
([_fmt_bits("neg_hi", inst.neg_hi, n)] if inst.neg_hi else []) + (["clamp"] if clamp else [])
|
||||
else:
|
||||
opsel_hi_default = 7 if n == 3 else 3
|
||||
mods = (([_fmt_bits("op_sel", inst.opsel, n)] if inst.opsel else [])
|
||||
+ ([_fmt_bits("op_sel_hi", opsel_hi, n)] if opsel_hi != opsel_hi_default else [])
|
||||
+ ([_fmt_bits("neg_lo", inst.neg, n)] if inst.neg else [])
|
||||
+ ([_fmt_bits("neg_hi", inst.neg_hi, n)] if inst.neg_hi else []) + (["clamp"] if clamp else []))
|
||||
mod_s = ' ' + ' '.join(mods) if mods else ''
|
||||
return f"{name} {dst}, {src0}, {src1}, {src2}{mod_s}" if n == 3 else f"{name} {dst}, {src0}, {src1}{mod_s}"
|
||||
|
||||
def _disasm_sop1(inst: SOP1) -> str:
|
||||
name, cdna = inst.op_name.lower(), _is_cdna(inst)
|
||||
# Use get_field_bits for register sizes
|
||||
regs = inst.canonical_op_regs
|
||||
dst_regs, src_regs = regs.get('d', 1), regs.get('s0', 1)
|
||||
src = _lit(inst, inst.ssrc0) if _unwrap(inst.ssrc0) == 255 else _fmt_src(inst.ssrc0, src_regs, cdna)
|
||||
if not cdna:
|
||||
if 'getpc_b64' in name: return f"{name} {_fmt_sdst(inst.sdst, 2)}"
|
||||
if 'setpc_b64' in name or 'rfe_b64' in name: return f"{name} {src}"
|
||||
if 'swappc_b64' in name: return f"{name} {_fmt_sdst(inst.sdst, 2)}, {src}"
|
||||
if 'sendmsg_rtn' in name:
|
||||
v = _unwrap(inst.ssrc0)
|
||||
try: msg_str = MSG(v).name if v != 255 else None # MSG_RTN_ILLEGAL_MSG (255) not supported by LLVM
|
||||
except ValueError: msg_str = None
|
||||
return f"{name} {_fmt_sdst(inst.sdst, dst_regs)}, sendmsg({msg_str})" if msg_str else f"{name} {_fmt_sdst(inst.sdst, dst_regs)}, 0x{v:x}"
|
||||
sop1_src_only = ('S_ALLOC_VGPR', 'S_SLEEP_VAR', 'S_BARRIER_SIGNAL', 'S_BARRIER_SIGNAL_ISFIRST',
|
||||
'S_BARRIER_INIT', 'S_BARRIER_JOIN', 'S_SET_GPR_IDX_IDX', 'S_CBRANCH_JOIN')
|
||||
if inst.op_name in sop1_src_only: return f"{name} {src}"
|
||||
if cdna:
|
||||
if 'getpc_b64' in name: return f"{name} {_fmt_sdst(inst.sdst, 2, cdna)}"
|
||||
if 'setpc_b64' in name or 'rfe_b64' in name: return f"{name} {src}"
|
||||
if 'swappc_b64' in name: return f"{name} {_fmt_sdst(inst.sdst, 2, cdna)}, {src}"
|
||||
return f"{name} {_fmt_sdst(inst.sdst, dst_regs, cdna)}, {src}"
|
||||
|
||||
def _disasm_sop2(inst: SOP2) -> str:
|
||||
cdna, name = _is_cdna(inst), inst.op_name.lower()
|
||||
lit = inst._literal
|
||||
# Use get_field_bits for register sizes
|
||||
regs = inst.canonical_op_regs
|
||||
dn, s0n, s1n = regs['d'], regs['s0'], regs['s1']
|
||||
s0 = _lit(inst, inst.ssrc0) if _unwrap(inst.ssrc0) == 255 else _fmt_src(inst.ssrc0, s0n, cdna)
|
||||
s1 = _lit(inst, inst.ssrc1) if _unwrap(inst.ssrc1) == 255 else _fmt_src(inst.ssrc1, s1n, cdna)
|
||||
dst = _fmt_sdst(inst.sdst, dn, cdna)
|
||||
if 'fmamk' in name and lit is not None: return f"{name} {dst}, {s0}, 0x{lit:x}, {s1}"
|
||||
if 'fmaak' in name and lit is not None: return f"{name} {dst}, {s0}, {s1}, 0x{lit:x}"
|
||||
if name in ('s_cbranch_g_fork', 's_rfe_restore_b64'): return f"{name} {s0}, {s1}" # no destination
|
||||
return f"{name} {dst}, {s0}, {s1}"
|
||||
|
||||
def _disasm_sopc(inst: SOPC) -> str:
|
||||
cdna, regs, name = _is_cdna(inst), inst.canonical_op_regs, inst.op_name.lower()
|
||||
s0 = _lit(inst, inst.ssrc0) if _unwrap(inst.ssrc0) == 255 else _fmt_src(inst.ssrc0, regs['s0'], cdna)
|
||||
if name == 's_set_gpr_idx_on':
|
||||
imm = _unwrap(inst.ssrc1) & 0xf
|
||||
flags = [n for i, n in enumerate(['SRC0', 'SRC1', 'SRC2', 'DST']) if imm & (1 << i)]
|
||||
return f"{name} {s0}, gpr_idx({','.join(flags)})"
|
||||
s1 = _lit(inst, inst.ssrc1) if _unwrap(inst.ssrc1) == 255 else _fmt_src(inst.ssrc1, regs['s1'], cdna)
|
||||
return f"{name} {s0}, {s1}"
|
||||
|
||||
_HWREG_BLACKLIST = {'HW_REG_PC_LO', 'HW_REG_PC_HI', 'HW_REG_IB_DBG1', 'HW_REG_FLUSH_IB', 'HW_REG_SHADER_TBA_LO', 'HW_REG_SHADER_TBA_HI',
|
||||
'HW_REG_SHADER_FLAT_SCRATCH_LO', 'HW_REG_SHADER_FLAT_SCRATCH_HI', 'HW_REG_SHADER_CYCLES'}
|
||||
_HWREG_BLACKLIST_CDNA = {'HW_REG_PC_LO', 'HW_REG_PC_HI', 'HW_REG_IB_DBG1', 'HW_REG_FLUSH_IB', 'HW_REG_SQ_SHADER_TBA_LO', 'HW_REG_SQ_SHADER_TBA_HI',
|
||||
'HW_REG_SQ_SHADER_TMA_LO', 'HW_REG_SQ_SHADER_TMA_HI', 'HW_REG_SQ_PERF_SNAPSHOT_DATA', 'HW_REG_SQ_PERF_SNAPSHOT_DATA1',
|
||||
'HW_REG_SQ_PERF_SNAPSHOT_PC_LO', 'HW_REG_SQ_PERF_SNAPSHOT_PC_HI', 'HW_REG_XCC_ID'}
|
||||
def _disasm_sopk(inst: SOPK) -> str:
|
||||
name, cdna = inst.op_name.lower(), _is_cdna(inst)
|
||||
is_rdna4 = _is_r4(inst)
|
||||
hw = HWREG_CDNA if cdna else (HWREG_RDNA4 if is_rdna4 else HWREG)
|
||||
blacklist = _HWREG_BLACKLIST_CDNA if cdna else _HWREG_BLACKLIST
|
||||
def fmt_hwreg(hid, hoff, hsz):
|
||||
try: hr_name = hw(hid).name.replace("HW_REG_WAVE_", "HW_REG_")
|
||||
except ValueError: return f"0x{inst.simm16:x}"
|
||||
if hr_name in blacklist: return f"0x{inst.simm16:x}"
|
||||
return f"hwreg({hr_name})" if hoff == 0 and hsz == 32 else f"hwreg({hr_name}, {hoff}, {hsz})"
|
||||
if name == 's_setreg_imm32_b32':
|
||||
hid, hoff, hsz = inst.simm16 & 0x3f, (inst.simm16 >> 6) & 0x1f, ((inst.simm16 >> 11) & 0x1f) + 1
|
||||
return f"{name} {fmt_hwreg(hid, hoff, hsz)}, 0x{inst._literal:x}"
|
||||
if name == 's_version': return f"{name} 0x{inst.simm16:x}"
|
||||
if name in ('s_setreg_b32', 's_getreg_b32'):
|
||||
hid, hoff, hsz = inst.simm16 & 0x3f, (inst.simm16 >> 6) & 0x1f, ((inst.simm16 >> 11) & 0x1f) + 1
|
||||
hs = fmt_hwreg(hid, hoff, hsz)
|
||||
return f"{name} {hs}, {_fmt_sdst(inst.sdst, 1, cdna)}" if 'setreg' in name else f"{name} {_fmt_sdst(inst.sdst, 1, cdna)}, {hs}"
|
||||
if name in ('s_subvector_loop_begin', 's_subvector_loop_end'):
|
||||
return f"{name} {_fmt_sdst(inst.sdst, 1)}, 0x{inst.simm16:x}"
|
||||
return f"{name} {_fmt_sdst(inst.sdst, inst.canonical_op_regs['d'], cdna)}, 0x{inst.simm16:x}"
|
||||
|
||||
def _disasm_vinterp(inst: VINTERP) -> str:
|
||||
mods = _mods((inst.waitexp, f"wait_exp:{inst.waitexp}"), (inst.clmp, "clamp"))
|
||||
s0, s1, s2 = _lit(inst, inst.src0, inst.neg & 1), _lit(inst, inst.src1, inst.neg & 2), _lit(inst, inst.src2, inst.neg & 4)
|
||||
return f"{inst.op_name.lower()} {inst.vdst.fmt()}, {s0}, {s1}, {s2}" + (" " + mods if mods else "")
|
||||
|
||||
DISASM_HANDLERS: dict[type, Callable[..., str]] = {
|
||||
VOP1: _disasm_vop1, VOP1_SDST: _disasm_vop1, VOP1_SDST_LIT: _disasm_vop1, VOP1_LIT: _disasm_vop1,
|
||||
VOP2: _disasm_vop2, VOP2_LIT: _disasm_vop2, VOPC: _disasm_vopc, VOPC_LIT: _disasm_vopc,
|
||||
VOP3: _disasm_vop3, VOP3_SDST: _disasm_vop3, VOP3_SDST_LIT: _disasm_vop3, VOP3_LIT: _disasm_vop3,
|
||||
VOP3SD: _disasm_vop3sd, VOP3SD_LIT: _disasm_vop3sd,
|
||||
VOPD: _disasm_vopd, VOPD_LIT: _disasm_vopd, VOP3P: _disasm_vop3p, VOP3P_LIT: _disasm_vop3p,
|
||||
VINTERP: _disasm_vinterp, SOPP: _disasm_sopp, SMEM: _disasm_smem, DS: _disasm_ds, FLAT: _disasm_flat, GLOBAL: _disasm_flat, SCRATCH: _disasm_flat,
|
||||
SOP1: _disasm_sop1, SOP1_LIT: _disasm_sop1, SOP2: _disasm_sop2, SOP2_LIT: _disasm_sop2,
|
||||
SOPC: _disasm_sopc, SOPC_LIT: _disasm_sopc, SOPK: _disasm_sopk, SOPK_LIT: _disasm_sopk,
|
||||
# RDNA4
|
||||
R4_VOP1: _disasm_vop1, R4_VOP1_SDST: _disasm_vop1, R4_VOP1_SDST_LIT: _disasm_vop1, R4_VOP1_LIT: _disasm_vop1,
|
||||
R4_VOP2: _disasm_vop2, R4_VOP2_LIT: _disasm_vop2, R4_VOPC: _disasm_vopc, R4_VOPC_LIT: _disasm_vopc,
|
||||
R4_VOP3: _disasm_vop3, R4_VOP3_SDST: _disasm_vop3, R4_VOP3_SDST_LIT: _disasm_vop3, R4_VOP3_LIT: _disasm_vop3,
|
||||
R4_VOP3SD: _disasm_vop3sd, R4_VOP3SD_LIT: _disasm_vop3sd, R4_VOP3P: _disasm_vop3p, R4_VOP3P_LIT: _disasm_vop3p,
|
||||
R4_FLAT: _disasm_flat, R4_GLOBAL: _disasm_flat, R4_SCRATCH: _disasm_flat,
|
||||
R4_VOPD: _disasm_vopd, R4_VOPD_LIT: _disasm_vopd, R4_VINTERP: _disasm_vinterp, R4_SOPP: _disasm_sopp, R4_SMEM: _disasm_smem, R4_DS: _disasm_ds,
|
||||
R4_SOP1: _disasm_sop1, R4_SOP1_LIT: _disasm_sop1, R4_SOP2: _disasm_sop2, R4_SOP2_LIT: _disasm_sop2,
|
||||
R4_SOPC: _disasm_sopc, R4_SOPC_LIT: _disasm_sopc, R4_SOPK: _disasm_sopk, R4_SOPK_LIT: _disasm_sopk}
|
||||
|
||||
def disasm(inst: Inst) -> str: return DISASM_HANDLERS[type(inst)](inst)
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# CDNA DISASSEMBLER SUPPORT
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
from tinygrad.runtime.autogen.amd.cdna.ins import (VOP1 as CDNA_VOP1, VOP1_LIT as CDNA_VOP1_LIT,
|
||||
VOP1_SDWA as CDNA_VOP1_SDWA, VOP1_DPP16 as CDNA_VOP1_DPP16,
|
||||
VOP2 as CDNA_VOP2, VOP2_LIT as CDNA_VOP2_LIT, VOP2_SDWA as CDNA_VOP2_SDWA, VOP2_DPP16 as CDNA_VOP2_DPP16,
|
||||
VOPC as CDNA_VOPC, VOPC_LIT as CDNA_VOPC_LIT, VOPC_SDWA_SDST as CDNA_VOPC_SDWA_SDST,
|
||||
VOP3 as CDNA_VOP3, VOP3_SDST as CDNA_VOP3_SDST, VOP3SD as CDNA_VOP3SD, VOP3P as CDNA_VOP3P, VOP3P_MFMA as CDNA_VOP3P_MFMA, VOP3PX2 as CDNA_VOP3PX2,
|
||||
SOP1 as CDNA_SOP1, SOP1_LIT as CDNA_SOP1_LIT, SOP2 as CDNA_SOP2, SOP2_LIT as CDNA_SOP2_LIT,
|
||||
SOPC as CDNA_SOPC, SOPC_LIT as CDNA_SOPC_LIT, SOPK as CDNA_SOPK, SOPK_LIT as CDNA_SOPK_LIT,
|
||||
SOPP as CDNA_SOPP, SMEM as CDNA_SMEM, DS as CDNA_DS,
|
||||
FLAT as CDNA_FLAT, GLOBAL as CDNA_GLOBAL, SCRATCH as CDNA_SCRATCH, MUBUF as CDNA_MUBUF)
|
||||
|
||||
def _cdna_src(inst, v, neg, abs_=0, n=1):
|
||||
s = _lit(inst, v) if v == 255 else _fmt_src(v, n, cdna=True)
|
||||
if abs_: s = f"|{s}|"
|
||||
return f"neg({s})" if neg and v == 255 else (f"-{s}" if neg else s)
|
||||
|
||||
_CDNA_VOP3_ALIASES = {'v_fmac_f64': 'v_mul_legacy_f32', 'v_dot2c_f32_bf16': 'v_mac_f32'}
|
||||
|
||||
def _disasm_vop3a(inst) -> str:
|
||||
op_val = inst.op.value if hasattr(inst.op, 'value') else inst.op
|
||||
name = inst.op_name.lower() or f'vop3a_op_{op_val}'
|
||||
n = inst.num_srcs() or _num_srcs(inst)
|
||||
cl, om = " clamp" if inst.clmp else "", _omod(inst.omod)
|
||||
# _sr_ instructions use 4-element op_sel (src2 for byte selection)
|
||||
opsel_n = 3 if '_sr_' in name and n == 2 else n
|
||||
opsel = _opsel_str(inst.opsel, opsel_n, inst.opsel != 0, False)
|
||||
orig_name = name
|
||||
name = _CDNA_VOP3_ALIASES.get(name, name)
|
||||
if name != orig_name:
|
||||
s0, s1 = _cdna_src(inst, inst.src0, inst.neg&1, inst.abs&1, 1), _cdna_src(inst, inst.src1, inst.neg&2, inst.abs&2, 1)
|
||||
s2 = ""
|
||||
dst = _vreg(inst.vdst)
|
||||
else:
|
||||
regs = inst.canonical_op_regs
|
||||
dregs, r0, r1, r2 = regs['d'], regs['s0'], regs['s1'], regs['s2']
|
||||
s0 = _cdna_src(inst, inst.src0, inst.neg&1, inst.abs&1, r0)
|
||||
s1 = _cdna_src(inst, inst.src1, inst.neg&2, inst.abs&2, r1)
|
||||
s2 = _cdna_src(inst, inst.src2, inst.neg&4, inst.abs&4, r2)
|
||||
dst = _vreg(inst.vdst, dregs) if dregs > 1 else _vreg(inst.vdst)
|
||||
if op_val >= 512:
|
||||
return f"{name} {dst}, {s0}, {s1}, {s2}{opsel}{cl}{om}" if n == 3 else f"{name} {dst}, {s0}, {s1}{opsel}{cl}{om}"
|
||||
if op_val < 256:
|
||||
# VOPC: vdst is actually sdst (SGPR pair), but VGPRField adds 256 to the offset
|
||||
sdst_val = _unwrap(inst.vdst)
|
||||
if sdst_val >= 256: sdst_val -= 256
|
||||
sdst = _fmt_sdst(sdst_val, 2, cdna=True)
|
||||
return f"{name} {sdst}, {s0}, {s1}{cl}"
|
||||
if 320 <= op_val < 512:
|
||||
if name in ('v_nop', 'v_clrexcp', 'v_nop_e64', 'v_clrexcp_e64'): return name.replace('_e64', '')
|
||||
return f"{name} {dst}, {s0}{cl}{om}"
|
||||
if name == 'v_cndmask_b32':
|
||||
s2 = _fmt_src(inst.src2, 2, cdna=True)
|
||||
return f"{name} {dst}, {s0}, {s1}, {s2}{cl}{om}"
|
||||
return f"{name} {dst}, {s0}, {s1}, {s2}{opsel}{cl}{om}" if n == 3 else f"{name} {dst}, {s0}, {s1}{opsel}{cl}{om}"
|
||||
|
||||
def _disasm_vop3b(inst) -> str:
|
||||
op_val = inst.op.value if hasattr(inst.op, 'value') else inst.op
|
||||
name, cdna = inst.op_name.lower() or f'vop3b_op_{op_val}', _is_cdna(inst)
|
||||
n = inst.num_srcs() or _num_srcs(inst)
|
||||
regs = inst.canonical_op_regs
|
||||
dregs, r0, r1, r2 = regs['d'], regs['s0'], regs['s1'], regs['s2']
|
||||
s0 = _cdna_src(inst, inst.src0, inst.neg&1, n=r0)
|
||||
s1 = _cdna_src(inst, inst.src1, inst.neg&2, n=r1)
|
||||
s2 = _cdna_src(inst, inst.src2, inst.neg&4, n=r2)
|
||||
# CDNA VOP3_SDST uses vdst field for sdst (but vdst adds 256), RDNA uses separate sdst field
|
||||
sdst_val = getattr(inst, 'sdst', None)
|
||||
if sdst_val is None and hasattr(inst, 'vdst'):
|
||||
sdst_val = _unwrap(inst.vdst)
|
||||
if sdst_val >= 256: sdst_val -= 256 # VGPRField adds 256, remove it for SGPR
|
||||
# For CDNA VOP3_SDST (VOPC->VOP3), vdst is the scalar dest (sdst), there's no vdst output
|
||||
if cdna and 'v_cmp' in name:
|
||||
sdst = _fmt_sdst(sdst_val, 2, cdna=True)
|
||||
cl, om = " clamp" if inst.clmp else "", _omod(inst.omod)
|
||||
return f"{name} {sdst}, {s0}, {s1}{cl}{om}"
|
||||
dst = _vreg(inst.vdst, dregs) if dregs > 1 else _vreg(inst.vdst)
|
||||
sdst = _fmt_sdst(sdst_val, 2, cdna=cdna)
|
||||
cl, om = " clamp" if inst.clmp else "", _omod(inst.omod)
|
||||
if name in ('v_addc_co_u32', 'v_subb_co_u32', 'v_subbrev_co_u32'):
|
||||
s2 = _fmt_src(inst.src2, 2, cdna=cdna)
|
||||
return f"{name} {dst}, {sdst}, {s0}, {s1}, {s2}{cl}{om}" if n == 3 else f"{name} {dst}, {sdst}, {s0}, {s1}{cl}{om}"
|
||||
|
||||
def _disasm_cdna_vop3p(inst) -> str:
|
||||
name, n = inst.op_name.lower(), inst.num_srcs() or 2
|
||||
is_mfma = 'mfma' in name or 'smfmac' in name
|
||||
is_accvgpr = 'accvgpr' in name
|
||||
def get_src(v, sc): return _lit(inst, v) if v == 255 else _fmt_src(v, sc, cdna=True)
|
||||
|
||||
# Handle accvgpr read/write (accumulator register operations)
|
||||
if is_accvgpr:
|
||||
src0_off = _unwrap(inst.src0)
|
||||
vdst_off = _vi(inst.vdst)
|
||||
if 'read' in name:
|
||||
# v_accvgpr_read_b32 vN, aM - reads from accumulator to VGPR
|
||||
return f"{name}_b32 v{vdst_off}, a{src0_off - 256 if src0_off >= 256 else src0_off}"
|
||||
if 'write' in name:
|
||||
# v_accvgpr_write_b32 aM, src - writes to accumulator from source
|
||||
src = _lit(inst, inst.src0) if src0_off == 255 else (f"v{src0_off - 256}" if src0_off >= 256 else decode_src(src0_off, cdna=True))
|
||||
return f"{name}_b32 a{vdst_off}, {src}"
|
||||
|
||||
# Handle v_mfma_ld_scale_b32 - special 2-operand format: v_mfma_ld_scale_b32 src0, src1
|
||||
if 'ld_scale' in name:
|
||||
src0, src1 = get_src(inst.src0, 1), get_src(inst.src1, 1)
|
||||
mods = ([_fmt_bits("op_sel", inst.opsel, 2)] if inst.opsel else []) + \
|
||||
([_fmt_bits("op_sel_hi", inst.opsel_hi, 2)] if inst.opsel_hi != 3 else [])
|
||||
return f"{name} {src0}, {src1}{' ' + ' '.join(mods) if mods else ''}"
|
||||
|
||||
# Handle MFMA instructions with accumulator destinations
|
||||
if is_mfma:
|
||||
regs = inst.canonical_op_regs
|
||||
dregs, r0, r1, r2 = regs['d'], regs['s0'], regs['s1'], regs['s2']
|
||||
# Infer register counts from instruction name if not in operands table (e.g., v_mfma_f32_32x32x4_xf32)
|
||||
if dregs == 1:
|
||||
if '32x32' in name: dregs, r0, r1, r2 = 16, 2, 2, 16
|
||||
elif '16x16' in name: dregs, r0, r1, r2 = 4, 2, 2, 4
|
||||
# MFMA reuses VOP3P fields differently: clmp -> acc_cd (dest is acc), opsel_hi -> acc (src1/src2 are acc)
|
||||
# acc field (bits 60-59): bit 0 = src2 is acc (always for MFMA), bit 1 = src1 is acc
|
||||
acc = inst.opsel_hi # opsel_hi field maps to acc for MFMA
|
||||
acc_cd = inst.clmp # clmp field maps to acc_cd for MFMA (dest is accumulator)
|
||||
is_smfmac = 'smfmac' in name # SMFMAC has different operand semantics
|
||||
# Format sources: src0 is always VGPR, src1/src2 depend on acc bits
|
||||
def mfma_src(v, sc, is_acc):
|
||||
v = _unwrap(v)
|
||||
if v == 255: return _lit(inst, v)
|
||||
if 128 <= v <= 208 or 240 <= v <= 248: return _lit(inst, v)
|
||||
base = v - 256 if v >= 256 else v
|
||||
if is_acc: return _areg(base, sc)
|
||||
return _vreg(base, sc)
|
||||
src0 = get_src(inst.src0, r0) # src0 is always VGPR
|
||||
src1 = mfma_src(inst.src1, r1, acc & 2) # bit 1 = src1 is acc
|
||||
# For SMFMAC, src2 is always a VGPR index (1 register), not accumulator
|
||||
src2 = _vreg(inst.src2) if is_smfmac else mfma_src(inst.src2, r2, acc_cd)
|
||||
dst = _areg(inst.vdst, dregs) if acc_cd else _vreg(inst.vdst, dregs)
|
||||
# MFMA uses neg:[...] not neg_lo:[...], and doesn't support op_sel_hi or clamp
|
||||
# Only f64 MFMA instructions support neg modifier
|
||||
# f8f6f4 MFMA instructions support cbsz/blgp modifiers
|
||||
mods = []
|
||||
if 'f8f6f4' in name:
|
||||
if inst.neg_hi: mods.append(f"cbsz:{inst.neg_hi}")
|
||||
if inst.neg: mods.append(f"blgp:{inst.neg}")
|
||||
elif inst.neg and 'f64' in name:
|
||||
mods.append(_fmt_bits("neg", inst.neg, n))
|
||||
return f"{name} {dst}, {src0}, {src1}, {src2}{' ' + ' '.join(mods) if mods else ''}"
|
||||
|
||||
# Standard VOP3P instructions
|
||||
src0, src1, src2, dst = get_src(inst.src0, 1), get_src(inst.src1, 1), get_src(inst.src2, 1), _vreg(inst.vdst)
|
||||
opsel_hi = inst.opsel_hi # CDNA VOP3P only has 2 bits for opsel_hi (no opsel_hi2)
|
||||
opsel_hi_default = 3 # CDNA default is 0b11 (2 bits), not 0b111 like RDNA
|
||||
mods = (([_fmt_bits("op_sel", inst.opsel, n)] if inst.opsel else [])
|
||||
+ ([_fmt_bits("op_sel_hi", opsel_hi, n)] if opsel_hi != opsel_hi_default else [])
|
||||
+ ([_fmt_bits("neg_lo", inst.neg, n)] if inst.neg else [])
|
||||
+ ([_fmt_bits("neg_hi", inst.neg_hi, n)] if inst.neg_hi else []) + (["clamp"] if inst.clmp else []))
|
||||
mod_s = ' ' + ' '.join(mods) if mods else ''
|
||||
return f"{name} {dst}, {src0}, {src1}, {src2}{mod_s}" if n == 3 else f"{name} {dst}, {src0}, {src1}{mod_s}"
|
||||
|
||||
def _disasm_mubuf(inst) -> str:
|
||||
name = inst.op_name.lower()
|
||||
# Determine vdata register count from instruction name
|
||||
nregs = 4 if 'xyzw' in name else 3 if 'xyz' in name else 2 if 'xy' in name or 'x2' in name or 'f64' in name or 'dwordx2' in name else 1
|
||||
vdata = _vreg(inst.vdata, nregs)
|
||||
vaddr = _vreg(inst.vaddr) if inst.offen or inst.idxen else None
|
||||
srsrc = str(inst.srsrc)
|
||||
soffset_val = _unwrap(inst.soffset)
|
||||
soffset = f"s{soffset_val}" if soffset_val < 128 else "off"
|
||||
offset = f" offset:{inst.offset}" if inst.offset else ""
|
||||
offen = " offen" if inst.offen else ""
|
||||
idxen = " idxen" if inst.idxen else ""
|
||||
lds = " lds" if inst.lds else ""
|
||||
sc0 = " sc0" if inst.sc0 else ""
|
||||
sc1 = " sc1" if inst.sc1 else ""
|
||||
nt = " nt" if inst.nt else ""
|
||||
# Handle special cases
|
||||
if name in ('buffer_wbl2', 'buffer_inv'):
|
||||
return f"{name}{sc0}{sc1}"
|
||||
if vaddr:
|
||||
return f"{name} {vdata}, {vaddr}, {srsrc}, {soffset}{offen}{idxen}{offset}{sc0}{nt}{sc1}{lds}"
|
||||
return f"{name} {vdata}, off, {srsrc}, {soffset}{offset}{sc0}{nt}{sc1}{lds}"
|
||||
|
||||
_SDWA_SEL = {0: 'BYTE_0', 1: 'BYTE_1', 2: 'BYTE_2', 3: 'BYTE_3', 4: 'WORD_0', 5: 'WORD_1', 6: 'DWORD'}
|
||||
|
||||
def _disasm_vop1_sdwa(inst) -> str:
|
||||
name = inst.op_name.lower().replace('_e32', '')
|
||||
regs = inst.canonical_op_regs
|
||||
dst = _vreg(inst.vdst, regs['d'])
|
||||
# When s0=1, vsrc0 is SGPR/constant (VGPRField adds 256, so subtract it back)
|
||||
if inst.s0 == 0: src0 = _vreg(inst.vsrc0, regs['s0'])
|
||||
else:
|
||||
raw = _unwrap(inst.vsrc0) - 256 # VGPRField adds 256
|
||||
src0 = decode_src(raw, cdna=True) # handles SGPRs, constants, specials
|
||||
src0_sel = _SDWA_SEL.get(inst.src0_sel, f'SEL{inst.src0_sel}')
|
||||
mods = []
|
||||
if inst.clmp: mods.append("clamp")
|
||||
if inst.omod == 1: mods.append("mul:2")
|
||||
elif inst.omod == 2: mods.append("mul:4")
|
||||
elif inst.omod == 3: mods.append("div:2")
|
||||
mods.append(f"src0_sel:{src0_sel}")
|
||||
return f"{name}_sdwa {dst}, {src0} {' '.join(mods)}"
|
||||
|
||||
def _decode_dpp(dpp: int) -> str:
|
||||
"""Decode DPP control value to string."""
|
||||
op, arg = decode_dpp16(dpp)
|
||||
if op == "quad_perm": return f"quad_perm:[{','.join(str(x) for x in arg)}]"
|
||||
if op in ("row_mirror", "row_half_mirror"): return op
|
||||
if op == "dpp": return f"dpp:{arg:#x}"
|
||||
return f"{op}:{arg}"
|
||||
|
||||
def _disasm_vop1_dpp(inst) -> str:
|
||||
name = inst.op_name.lower().replace('_e32', '')
|
||||
regs = inst.canonical_op_regs
|
||||
dst, src0 = _vreg(inst.vdst, regs['d']), _vreg(inst.vsrc0, regs['s0'])
|
||||
dpp_str = _decode_dpp(inst.dpp)
|
||||
mods = [dpp_str]
|
||||
if inst.row_mask != 0xf: mods.append(f"row_mask:{inst.row_mask:#x}")
|
||||
if inst.bank_mask != 0xf: mods.append(f"bank_mask:{inst.bank_mask:#x}")
|
||||
if inst.bc: mods.append("bound_ctrl:1")
|
||||
return f"{name}_dpp {dst}, {src0} {' '.join(mods)}"
|
||||
|
||||
def _disasm_vop2_sdwa(inst) -> str:
|
||||
name, cdna = inst.op_name.lower().replace('_e32', ''), _is_cdna(inst)
|
||||
regs = inst.canonical_op_regs
|
||||
dst = _vreg(inst.vdst, regs['d'])
|
||||
# When s0/s1=1, vsrc is SGPR/constant (VGPRField adds 256, so subtract it back)
|
||||
src0 = _vreg(inst.vsrc0, regs['s0']) if inst.s0 == 0 else decode_src(_unwrap(inst.vsrc0) - 256, cdna)
|
||||
src1 = _vreg(inst.vsrc1, regs['s1']) if inst.s1 == 0 else decode_src(_unwrap(inst.vsrc1) - 256, cdna)
|
||||
src0_sel = _SDWA_SEL.get(inst.src0_sel, f'SEL{inst.src0_sel}')
|
||||
src1_sel = _SDWA_SEL.get(inst.src1_sel, f'SEL{inst.src1_sel}')
|
||||
mods = []
|
||||
if inst.clmp: mods.append("clamp")
|
||||
if inst.omod == 1: mods.append("mul:2")
|
||||
elif inst.omod == 2: mods.append("mul:4")
|
||||
elif inst.omod == 3: mods.append("div:2")
|
||||
if inst.src0_sel != 6: mods.append(f"src0_sel:{src0_sel}")
|
||||
if inst.src1_sel != 6: mods.append(f"src1_sel:{src1_sel}")
|
||||
mods_str = ' '.join(mods) if mods else ""
|
||||
# CDNA carry instructions and cndmask need vcc operands
|
||||
if cdna and name in _VOP2_CARRY_OUT: return f"{name}_sdwa {dst}, vcc, {src0}, {src1} {mods_str}".strip()
|
||||
if cdna and name in _VOP2_CARRY_INOUT: return f"{name}_sdwa {dst}, vcc, {src0}, {src1}, vcc {mods_str}".strip()
|
||||
if cdna and name == 'v_cndmask_b32': return f"{name}_sdwa {dst}, {src0}, {src1}, vcc {mods_str}".strip()
|
||||
return f"{name}_sdwa {dst}, {src0}, {src1} {mods_str}".strip()
|
||||
|
||||
def _disasm_vop2_dpp(inst) -> str:
|
||||
name, cdna = inst.op_name.lower().replace('_e32', ''), _is_cdna(inst)
|
||||
regs = inst.canonical_op_regs
|
||||
dst, src0, src1 = _vreg(inst.vdst, regs['d']), _vreg(inst.vsrc0, regs['s0']), _vreg(inst.vsrc1, regs['s1'])
|
||||
dpp_str = _decode_dpp(inst.dpp)
|
||||
mods = [dpp_str]
|
||||
if inst.row_mask != 0xf: mods.append(f"row_mask:{inst.row_mask:#x}")
|
||||
if inst.bank_mask != 0xf: mods.append(f"bank_mask:{inst.bank_mask:#x}")
|
||||
if inst.bc: mods.append("bound_ctrl:1")
|
||||
# CDNA carry instructions and cndmask need vcc operands
|
||||
if cdna and name in _VOP2_CARRY_OUT: return f"{name}_dpp {dst}, vcc, {src0}, {src1} {' '.join(mods)}"
|
||||
if cdna and name in _VOP2_CARRY_INOUT: return f"{name}_dpp {dst}, vcc, {src0}, {src1}, vcc {' '.join(mods)}"
|
||||
if cdna and name == 'v_cndmask_b32': return f"{name}_dpp {dst}, {src0}, {src1}, vcc {' '.join(mods)}"
|
||||
return f"{name}_dpp {dst}, {src0}, {src1} {' '.join(mods)}"
|
||||
|
||||
def _disasm_vopc_sdwa(inst) -> str:
|
||||
name = inst.op_name.lower().replace('_e32', '')
|
||||
regs = inst.canonical_op_regs
|
||||
sdst = _fmt_sdst(inst.sdst, 2, cdna=True)
|
||||
src0 = _vreg(inst.vsrc0, regs['s0']) if getattr(inst, 's0', 0) == 0 else decode_src(_unwrap(inst.vsrc0) - 256, cdna=True)
|
||||
src1 = _vreg(inst.vsrc1, regs['s1']) if getattr(inst, 's1', 0) == 0 else decode_src(_unwrap(inst.vsrc1) - 256, cdna=True)
|
||||
src0_sel = _SDWA_SEL.get(inst.src0_sel, f'SEL{inst.src0_sel}')
|
||||
src1_sel = _SDWA_SEL.get(inst.src1_sel, f'SEL{inst.src1_sel}')
|
||||
mods = []
|
||||
if inst.src0_sel != 6: mods.append(f"src0_sel:{src0_sel}")
|
||||
if inst.src1_sel != 6: mods.append(f"src1_sel:{src1_sel}")
|
||||
return f"{name}_sdwa {sdst}, {src0}, {src1} {' '.join(mods)}".strip()
|
||||
|
||||
def _disasm_vop3px2(inst) -> str:
|
||||
"""VOP3PX2 disassembler for scaled MFMA instructions."""
|
||||
name = inst.op_name.lower()
|
||||
regs = inst.canonical_op_regs
|
||||
dregs, r2 = regs['d'], regs['s2']
|
||||
# F8F6F4 MFMA: CBSZ selects matrix A format, BLGP selects matrix B format
|
||||
# VGPRs: FP8/BF8(0,1)=8, FP6/BF6(2,3)=6, FP4(4)=4
|
||||
vgprs = {0: 8, 1: 8, 2: 6, 3: 6, 4: 4}
|
||||
r0, r1 = vgprs.get(inst.cbsz, 8), vgprs.get(inst.blgp, 8)
|
||||
def mfma_src(v, sc, is_acc):
|
||||
v = _unwrap(v)
|
||||
if v == 255: return _lit(inst, v)
|
||||
base = v - 256 if v >= 256 else v
|
||||
return _areg(base, sc) if is_acc else _vreg(base, sc)
|
||||
src0, src1, src2 = mfma_src(inst.src0, r0, False), mfma_src(inst.src1, r1, inst.acc & 2), mfma_src(inst.src2, r2, inst.acc_cd)
|
||||
dst = _areg(inst.vdst, dregs) if inst.acc_cd else _vreg(inst.vdst, dregs)
|
||||
scale_src0, scale_src1 = _vreg(inst.scale_src0), _vreg(inst.scale_src1)
|
||||
mods = []
|
||||
if inst.opsel: mods.append(_fmt_bits("op_sel", inst.opsel, 3))
|
||||
if inst.opsel_hi != 0: mods.append(_fmt_bits("op_sel_hi", inst.opsel_hi, 3))
|
||||
if inst.neg: mods.append(_fmt_bits("neg", inst.neg, 3))
|
||||
if inst.cbsz: mods.append(f"cbsz:{inst.cbsz}")
|
||||
if inst.blgp: mods.append(f"blgp:{inst.blgp}")
|
||||
return f"{name} {dst}, {src0}, {src1}, {src2}, {scale_src0}, {scale_src1}{' ' + ' '.join(mods) if mods else ''}"
|
||||
|
||||
DISASM_HANDLERS.update({CDNA_VOP1: _disasm_vop1, CDNA_VOP1_LIT: _disasm_vop1,
|
||||
CDNA_VOP1_SDWA: _disasm_vop1_sdwa, CDNA_VOP1_DPP16: _disasm_vop1_dpp,
|
||||
CDNA_VOP2: _disasm_vop2, CDNA_VOP2_LIT: _disasm_vop2,
|
||||
CDNA_VOP2_SDWA: _disasm_vop2_sdwa, CDNA_VOP2_DPP16: _disasm_vop2_dpp,
|
||||
CDNA_VOPC: _disasm_vopc, CDNA_VOPC_LIT: _disasm_vopc, CDNA_VOPC_SDWA_SDST: _disasm_vopc_sdwa,
|
||||
CDNA_SOP1: _disasm_sop1, CDNA_SOP1_LIT: _disasm_sop1, CDNA_SOP2: _disasm_sop2, CDNA_SOP2_LIT: _disasm_sop2,
|
||||
CDNA_SOPC: _disasm_sopc, CDNA_SOPC_LIT: _disasm_sopc, CDNA_SOPK: _disasm_sopk, CDNA_SOPK_LIT: _disasm_sopk, CDNA_SOPP: _disasm_sopp,
|
||||
CDNA_SMEM: _disasm_smem, CDNA_DS: _disasm_ds, CDNA_FLAT: _disasm_flat, CDNA_GLOBAL: _disasm_flat, CDNA_SCRATCH: _disasm_flat,
|
||||
CDNA_VOP3: _disasm_vop3a, CDNA_VOP3_SDST: _disasm_vop3b, CDNA_VOP3SD: _disasm_vop3b,
|
||||
CDNA_VOP3P: _disasm_cdna_vop3p, CDNA_VOP3P_MFMA: _disasm_cdna_vop3p,
|
||||
CDNA_MUBUF: _disasm_mubuf, CDNA_VOP3PX2: _disasm_vop3px2})
|
||||
132
tinygrad_repo/test/amd/helpers.py
Normal file
132
tinygrad_repo/test/amd/helpers.py
Normal file
@@ -0,0 +1,132 @@
|
||||
"""Shared test helpers for AMD tests."""
|
||||
import ctypes
|
||||
from tinygrad.helpers import unwrap
|
||||
from tinygrad.runtime.autogen import llvm
|
||||
from tinygrad.runtime.support.elf import elf_loader
|
||||
|
||||
ARCH_TO_TARGET:dict[str, list[str]] = {
|
||||
"rdna3":["gfx1100", "gfx1151"],
|
||||
"rdna4":["gfx1200", "gfx1201"],
|
||||
"cdna":["gfx950", "gfx942"],
|
||||
}
|
||||
|
||||
TARGET_TO_ARCH:dict[str, str] = {t:arch for arch,targets in ARCH_TO_TARGET.items() for t in targets}
|
||||
|
||||
_DPP16_RANGE_OPS = {0x100: "row_shl", 0x110: "row_shr", 0x120: "row_ror", 0x150: "row_newbcast", 0x160: "row_share", 0x170: "row_xmask"}
|
||||
_DPP16_EXACT_OPS = {0x130: ("wave_shl", 1), 0x134: ("wave_rol", 1), 0x138: ("wave_shr", 1), 0x13c: ("wave_ror", 1),
|
||||
0x140: ("row_mirror", 0), 0x141: ("row_half_mirror", 0), 0x142: ("row_bcast", 15), 0x143: ("row_bcast", 31)}
|
||||
|
||||
def get_target(arch:str) -> str: return ARCH_TO_TARGET[arch][0]
|
||||
|
||||
def decode_dpp16(dpp: int) -> tuple[str, int | tuple[int, int, int, int]]:
|
||||
"""Decode a DPP16 control word into a symbolic operation and argument."""
|
||||
if dpp < 0x100: return "quad_perm", ((dpp >> 0) & 0x3, (dpp >> 2) & 0x3, (dpp >> 4) & 0x3, (dpp >> 6) & 0x3)
|
||||
if dpp in _DPP16_EXACT_OPS: return _DPP16_EXACT_OPS[dpp]
|
||||
if (base := dpp & 0x1f0) in _DPP16_RANGE_OPS: return _DPP16_RANGE_OPS[base], dpp & 0xf
|
||||
return "dpp", dpp
|
||||
|
||||
def get_mattr(arch:str) -> str:
|
||||
return {"rdna3":"+real-true16,+wavefrontsize32", "rdna4":"+real-true16,+wavefrontsize32", "cdna":"+wavefrontsize64"}[arch]
|
||||
|
||||
# LLVM in-process assembler/disassembler (replaces llvm-mc and llvm-objdump subprocesses)
|
||||
_SENTINEL = b'\xde\xad\xbe\xef'
|
||||
_SENTINEL_ASM = '.byte 0xde, 0xad, 0xbe, 0xef'
|
||||
|
||||
def _cerr(): return ctypes.pointer(ctypes.pointer(ctypes.c_char()))
|
||||
def _expect(x, err, ret=None):
|
||||
if x: raise RuntimeError(unwrap(ctypes.cast(err.contents, ctypes.c_char_p).value).decode() if not isinstance(err, str) else err)
|
||||
return ret
|
||||
|
||||
def _init_llvm():
|
||||
for component in ['Target', 'TargetInfo', 'TargetMC', 'AsmParser', 'AsmPrinter', 'Disassembler']:
|
||||
getattr(llvm, f'LLVMInitializeAMDGPU{component}')()
|
||||
|
||||
def _create_target_machine(mcpu:str, mattr:str) -> llvm.LLVMTargetMachineRef:
|
||||
target = _expect(llvm.LLVMGetTargetFromTriple(b'amdgcn-amd-amdhsa', ctypes.pointer(tgt:=llvm.LLVMTargetRef()), err:=_cerr()), err, tgt)
|
||||
return llvm.LLVMCreateTargetMachine(target, b'amdgcn-amd-amdhsa', mcpu.encode(), mattr.encode(),
|
||||
llvm.LLVMCodeGenLevelDefault, llvm.LLVMRelocDefault, llvm.LLVMCodeModelDefault)
|
||||
|
||||
def _emit_obj(asm_text:str, mcpu:str, mattr:str, diag_errors:list[str]|None=None) -> bytes:
|
||||
"""Assemble raw asm text into an ELF object using LLVM in-process."""
|
||||
_init_llvm()
|
||||
tm = _create_target_machine(mcpu, mattr)
|
||||
ctx = llvm.LLVMContextCreate()
|
||||
try:
|
||||
errors = diag_errors if diag_errors is not None else []
|
||||
@llvm.LLVMDiagnosticHandler
|
||||
def handle_diag(diag_ref, _arg):
|
||||
if llvm.LLVMGetDiagInfoSeverity(diag_ref) == llvm.LLVMDSError:
|
||||
errors.append(ctypes.string_at(llvm.LLVMGetDiagInfoDescription(diag_ref)).decode())
|
||||
llvm.LLVMContextSetDiagnosticHandler(ctx, handle_diag, None)
|
||||
mod = llvm.LLVMModuleCreateWithNameInContext(b'asm', ctx)
|
||||
llvm.LLVMSetTarget(mod, b'amdgcn-amd-amdhsa')
|
||||
asm_bytes = asm_text.encode()
|
||||
llvm.LLVMSetModuleInlineAsm2(mod, asm_bytes, len(asm_bytes))
|
||||
buf = llvm.LLVMMemoryBufferRef()
|
||||
_expect(llvm.LLVMTargetMachineEmitToMemoryBuffer(tm, mod, llvm.LLVMObjectFile, err:=_cerr(), ctypes.pointer(buf)), err)
|
||||
obj = ctypes.string_at(llvm.LLVMGetBufferStart(buf), llvm.LLVMGetBufferSize(buf))
|
||||
llvm.LLVMDisposeMemoryBuffer(buf)
|
||||
llvm.LLVMDisposeModule(mod)
|
||||
return obj
|
||||
finally:
|
||||
llvm.LLVMContextDispose(ctx)
|
||||
llvm.LLVMDisposeTargetMachine(tm)
|
||||
|
||||
def _extract_text(obj:bytes) -> bytes:
|
||||
"""Extract .text section from ELF object bytes."""
|
||||
return next(s.content for s in elf_loader(obj)[1] if s.name == ".text")
|
||||
|
||||
def llvm_assemble(instrs:list[str], mcpu:str, mattr:str) -> list[bytes]:
|
||||
"""Assemble instructions in one LLVM emission, return per-instruction bytes."""
|
||||
if not instrs: return []
|
||||
parts = []
|
||||
for instr in instrs:
|
||||
parts.append(instr)
|
||||
parts.append(_SENTINEL_ASM)
|
||||
text = _extract_text(_emit_obj('.text\n' + '\n'.join(parts) + '\n', mcpu, mattr))
|
||||
results, start = [], 0
|
||||
for _ in instrs:
|
||||
idx = text.find(_SENTINEL, start)
|
||||
assert idx != -1, "sentinel not found in .text section"
|
||||
results.append(bytes(text[start:idx]))
|
||||
start = idx + len(_SENTINEL)
|
||||
return results
|
||||
|
||||
def llvm_disasm(code:bytes, mcpu:str, mattr:str) -> list[str]:
|
||||
"""Disassemble raw bytes into instruction strings using LLVM."""
|
||||
_init_llvm()
|
||||
dc = llvm.LLVMCreateDisasmCPUFeatures(b'amdgcn-amd-amdhsa', mcpu.encode(), mattr.encode(), None, 0,
|
||||
llvm.LLVMOpInfoCallback(0), llvm.LLVMSymbolLookupCallback(0))
|
||||
if not dc: raise RuntimeError(f"failed to create disasm context for {mcpu}")
|
||||
llvm.LLVMSetDisasmOptions(dc, 2 | 4) # PrintImmHex | AsmPrinterVariant
|
||||
try:
|
||||
buf = ctypes.create_string_buffer(256)
|
||||
arr = (ctypes.c_uint8 * len(code)).from_buffer_copy(code)
|
||||
results, offset = [], 0
|
||||
while offset < len(code):
|
||||
size = llvm.LLVMDisasmInstruction(dc, ctypes.cast(ctypes.addressof(arr) + offset, ctypes.POINTER(ctypes.c_uint8)),
|
||||
len(code) - offset, 0, buf, 256)
|
||||
if size == 0: break
|
||||
results.append(buf.value.decode().strip())
|
||||
offset += size
|
||||
return results
|
||||
finally:
|
||||
llvm.LLVMDisasmDispose(dc)
|
||||
|
||||
def llvm_filter_valid_asm(tests:list[tuple[str, bytes]], mcpu:str, mattr:str) -> list[tuple[str, bytes]]:
|
||||
"""Filter out tests where original ASM isn't valid on target, and where LLVM roundtrip doesn't match."""
|
||||
if not tests: return []
|
||||
# Assemble all instructions at once with sentinels and diagnostic handler to detect failures
|
||||
parts, diag_errors = [], [] # type: ignore[var-annotated]
|
||||
for asm, _ in tests:
|
||||
parts.append(asm)
|
||||
parts.append(_SENTINEL_ASM)
|
||||
text = _extract_text(_emit_obj('.text\n' + '\n'.join(parts) + '\n', mcpu, mattr, diag_errors))
|
||||
results, start = [], 0
|
||||
for _ in tests:
|
||||
idx = text.find(_SENTINEL, start)
|
||||
assert idx != -1, "sentinel not found in .text section"
|
||||
results.append(bytes(text[start:idx]))
|
||||
start = idx + len(_SENTINEL)
|
||||
# Invalid instructions produce 0 bytes; also filter where LLVM roundtrip doesn't match original
|
||||
return [(asm, data) for (asm, data), chunk in zip(tests, results) if len(chunk) > 0 and chunk == data]
|
||||
1
tinygrad_repo/test/amd/hw/__init__.py
Normal file
1
tinygrad_repo/test/amd/hw/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Hardware-validated emulator tests for RDNA3 instructions."""
|
||||
287
tinygrad_repo/test/amd/hw/helpers.py
Normal file
287
tinygrad_repo/test/amd/hw/helpers.py
Normal file
@@ -0,0 +1,287 @@
|
||||
"""Test infrastructure for hardware-validated RDNA3 emulator tests.
|
||||
|
||||
Uses run_asm() with memory output, so tests can run on both emulator and real hardware.
|
||||
Set USE_HW=1 to run on both emulator and hardware, comparing results.
|
||||
"""
|
||||
import ctypes, math, os, struct
|
||||
from tinygrad.runtime.autogen.amd.rdna3.ins import *
|
||||
|
||||
from test.mockgpu.amd.emu import run_asm
|
||||
from tinygrad.renderer.amd.dsl import NULL, SCC, VCC_LO, VCC_HI, EXEC_LO, EXEC_HI, M0
|
||||
|
||||
def _i32(f: float) -> int: return struct.unpack('<I', struct.pack('<f', f))[0]
|
||||
def _f32(i: int) -> float: return struct.unpack('<f', struct.pack('<I', i & 0xFFFFFFFF))[0]
|
||||
|
||||
# f16 conversion helpers
|
||||
def f16(i: int) -> float: return struct.unpack('<e', struct.pack('<H', i & 0xFFFF))[0]
|
||||
def f32_to_f16(f: float) -> int:
|
||||
f = float(f)
|
||||
if math.isnan(f): return 0x7e00
|
||||
if math.isinf(f): return 0x7c00 if f > 0 else 0xfc00
|
||||
try: return struct.unpack('<H', struct.pack('<e', f))[0]
|
||||
except OverflowError: return 0x7c00 if f > 0 else 0xfc00
|
||||
|
||||
# For backwards compatibility with tests using SrcEnum.NULL etc.
|
||||
class SrcEnum:
|
||||
NULL = NULL
|
||||
VCC_LO = VCC_LO
|
||||
VCC_HI = VCC_HI
|
||||
EXEC_LO = EXEC_LO
|
||||
EXEC_HI = EXEC_HI
|
||||
SCC = SCC
|
||||
M0 = M0
|
||||
POS_HALF = 0.5
|
||||
NEG_HALF = -0.5
|
||||
POS_ONE = 1.0
|
||||
NEG_ONE = -1.0
|
||||
POS_TWO = 2.0
|
||||
NEG_TWO = -2.0
|
||||
POS_FOUR = 4.0
|
||||
NEG_FOUR = -4.0
|
||||
|
||||
VCC = VCC_LO # For VOP3SD sdst field (VCC_LO is exported from dsl)
|
||||
USE_HW = os.environ.get("USE_HW", "0") == "1"
|
||||
FLOAT_TOLERANCE = 1e-5
|
||||
|
||||
def get_gpu_target() -> tuple[int, int, int]:
|
||||
"""Get the GPU target as (major, minor, stepping) tuple."""
|
||||
if not USE_HW: return (0, 0, 0)
|
||||
from tinygrad.device import Device
|
||||
return Device["AMD"].target # type: ignore[attr-defined]
|
||||
|
||||
def skip_unless_gfx(min_major: int, min_minor: int = 0, reason: str = ""):
|
||||
"""Skip test if GPU target is below the minimum required version."""
|
||||
import unittest
|
||||
def decorator(test_func):
|
||||
if not USE_HW: return test_func
|
||||
target = get_gpu_target()
|
||||
if target[0] < min_major or (target[0] == min_major and target[1] < min_minor):
|
||||
return unittest.skip(reason or f"requires gfx{min_major}{min_minor}0+")(test_func)
|
||||
return test_func
|
||||
return decorator
|
||||
|
||||
# Output buffer layout: vgpr[N_VGPRS][n_lanes], sgpr[N_SGPRS], vcc, scc, exec
|
||||
N_VGPRS, N_SGPRS, WAVE_SIZE = 16, 16, 32
|
||||
SGPR_BYTES = N_SGPRS * 4 # 16 regs * 4 bytes = 64
|
||||
_VGPR_REGION = N_VGPRS * WAVE_SIZE * 4 # minimum vgpr region size (tests may use as scratch)
|
||||
def _out_bytes(n_lanes: int) -> int: return max(N_VGPRS * n_lanes * 4, _VGPR_REGION) + SGPR_BYTES + 12
|
||||
OUT_BYTES = _out_bytes(WAVE_SIZE) # default for single-wave (backward compat)
|
||||
|
||||
# Float conversion helpers
|
||||
def f2i(f: float) -> int: return _i32(f)
|
||||
def i2f(i: int) -> float: return _f32(i)
|
||||
def f2i64(f: float) -> int: return struct.unpack('<Q', struct.pack('<d', f))[0]
|
||||
def i642f(i: int) -> float: return struct.unpack('<d', struct.pack('<Q', i))[0]
|
||||
|
||||
def assemble(instructions: list) -> bytes:
|
||||
return b''.join(inst.to_bytes() for inst in instructions)
|
||||
|
||||
# Simple WaveState class for test output parsing (mirrors test/mockgpu/amd/emu.py interface for tests)
|
||||
class WaveState:
|
||||
def __init__(self, n_lanes: int = 32):
|
||||
self.vgpr = [[0] * 256 for _ in range(n_lanes)] # vgpr[lane][reg]
|
||||
self.sgpr = [0] * 128
|
||||
self.vcc = 0
|
||||
self.scc = 0
|
||||
|
||||
def get_prologue_epilogue(n_lanes: int) -> tuple[list, list]:
|
||||
"""Generate prologue and epilogue instructions for state capture."""
|
||||
prologue = [
|
||||
s_mov_b32(s[80], s[0]),
|
||||
s_mov_b32(s[81], s[1]),
|
||||
v_mov_b32_e32(v[255], v[0]),
|
||||
]
|
||||
for i in range(N_VGPRS):
|
||||
prologue.append(v_mov_b32_e32(v[i], 0))
|
||||
for i in range(N_SGPRS):
|
||||
prologue.append(s_mov_b32(s[i], 0))
|
||||
prologue.append(s_mov_b32(VCC_LO, 0))
|
||||
|
||||
epilogue = [
|
||||
s_mov_b32(s[90], VCC_LO),
|
||||
s_cselect_b32(s[91], 1, 0),
|
||||
# Save EXEC early (before we modify it for VGPR stores)
|
||||
s_mov_b32(s[95], EXEC_LO),
|
||||
# Restore EXEC to all active lanes for VGPR stores (test may have modified EXEC)
|
||||
s_mov_b32(EXEC_LO, (1 << min(n_lanes, WAVE_SIZE)) - 1),
|
||||
s_load_b64(s[92:93], s[80:81], 0, soffset=NULL),
|
||||
s_waitcnt(0), # simm16=0 waits for all
|
||||
v_lshlrev_b32_e32(v[240], 2, v[255]),
|
||||
]
|
||||
vgpr_bytes = N_VGPRS * n_lanes * 4
|
||||
for i in range(N_VGPRS):
|
||||
epilogue.append(global_store_b32(addr=v[240], data=v[i], saddr=s[92:93], offset=i * n_lanes * 4))
|
||||
epilogue.append(v_mov_b32_e32(v[241], 0))
|
||||
epilogue.append(v_cmp_eq_u32_e32(v[255], v[241]))
|
||||
epilogue.append(s_and_saveexec_b32(s[94], VCC_LO))
|
||||
# Scalar stores: only thread 0. Use v[240]=vgpr_bytes as base offset so immediate offsets stay small.
|
||||
epilogue.append(v_mov_b32_e32(v[240], vgpr_bytes))
|
||||
for i in range(N_SGPRS):
|
||||
epilogue.append(v_mov_b32_e32(v[243], s[i]))
|
||||
epilogue.append(global_store_b32(addr=v[240], data=v[243], saddr=s[92:93], offset=i * 4))
|
||||
epilogue.append(v_mov_b32_e32(v[243], s[90]))
|
||||
epilogue.append(global_store_b32(addr=v[240], data=v[243], saddr=s[92:93], offset=SGPR_BYTES))
|
||||
epilogue.append(v_mov_b32_e32(v[243], s[91]))
|
||||
epilogue.append(global_store_b32(addr=v[240], data=v[243], saddr=s[92:93], offset=SGPR_BYTES + 4))
|
||||
# Store EXEC (saved earlier in s[95])
|
||||
epilogue.append(v_mov_b32_e32(v[243], s[95]))
|
||||
epilogue.append(global_store_b32(addr=v[240], data=v[243], saddr=s[92:93], offset=SGPR_BYTES + 8))
|
||||
epilogue.append(s_mov_b32(EXEC_LO, s[94]))
|
||||
epilogue.append(s_endpgm())
|
||||
return prologue, epilogue
|
||||
|
||||
def parse_output(out_buf: bytes, n_lanes: int) -> WaveState:
|
||||
"""Parse output buffer into WaveState."""
|
||||
vgpr_bytes = N_VGPRS * n_lanes * 4
|
||||
st = WaveState(n_lanes)
|
||||
for i in range(N_VGPRS):
|
||||
for lane in range(n_lanes):
|
||||
off = i * n_lanes * 4 + lane * 4
|
||||
st.vgpr[lane][i] = struct.unpack_from('<I', out_buf, off)[0]
|
||||
for i in range(N_SGPRS):
|
||||
st.sgpr[i] = struct.unpack_from('<I', out_buf, vgpr_bytes + i * 4)[0]
|
||||
st.vcc = struct.unpack_from('<I', out_buf, vgpr_bytes + SGPR_BYTES)[0]
|
||||
st.scc = struct.unpack_from('<I', out_buf, vgpr_bytes + SGPR_BYTES + 4)[0]
|
||||
# Store EXEC in its proper location (index 126)
|
||||
st.sgpr[EXEC_LO.offset] = struct.unpack_from('<I', out_buf, vgpr_bytes + SGPR_BYTES + 8)[0]
|
||||
return st
|
||||
|
||||
def run_program_emu(instructions: list, n_lanes: int = 1) -> WaveState:
|
||||
"""Run instructions via emulator run_asm, dump state to memory, return WaveState."""
|
||||
buf_sz = _out_bytes(n_lanes)
|
||||
out_buf = (ctypes.c_uint8 * buf_sz)(*([0] * buf_sz))
|
||||
out_addr = ctypes.addressof(out_buf)
|
||||
|
||||
prologue, epilogue = get_prologue_epilogue(n_lanes)
|
||||
code = assemble(prologue + instructions + epilogue)
|
||||
|
||||
args = (ctypes.c_uint64 * 1)(out_addr)
|
||||
args_ptr = ctypes.addressof(args)
|
||||
kernel_buf = (ctypes.c_char * len(code)).from_buffer_copy(code)
|
||||
lib_ptr = ctypes.addressof(kernel_buf)
|
||||
|
||||
# rsrc2: USER_SGPR_COUNT=2, ENABLE_SGPR_WORKGROUP_ID_X/Y/Z=1, LDS_SIZE=128 (64KB)
|
||||
rsrc2 = 0x19c | (128 << 15)
|
||||
scratch_size = 0x10000 # 64KB per lane, matches .amdhsa_private_segment_fixed_size in run_program_hw
|
||||
result = run_asm(lib_ptr, len(code), 1, 1, 1, n_lanes, 1, 1, args_ptr, rsrc2, scratch_size)
|
||||
assert result == 0, f"run_asm failed with {result}"
|
||||
|
||||
return parse_output(bytes(out_buf), n_lanes)
|
||||
|
||||
def run_program_hw(instructions: list, n_lanes: int = 1) -> WaveState:
|
||||
"""Run instructions on real AMD hardware via HIPCompiler and AMDProgram."""
|
||||
from tinygrad.device import Device
|
||||
from tinygrad.runtime.ops_amd import AMDProgram
|
||||
from tinygrad.runtime.support.compiler_amd import HIPCompiler
|
||||
from tinygrad.helpers import flat_mv
|
||||
|
||||
dev = Device["AMD"]
|
||||
compiler = HIPCompiler(dev.arch) # type: ignore[attr-defined]
|
||||
|
||||
prologue, epilogue = get_prologue_epilogue(n_lanes)
|
||||
code = assemble(prologue + instructions + epilogue)
|
||||
|
||||
byte_str = ', '.join(f'0x{b:02x}' for b in code)
|
||||
asm_src = f""".text
|
||||
.globl test
|
||||
.p2align 8
|
||||
.type test,@function
|
||||
test:
|
||||
.byte {byte_str}
|
||||
|
||||
.rodata
|
||||
.p2align 6
|
||||
.amdhsa_kernel test
|
||||
.amdhsa_next_free_vgpr 256
|
||||
.amdhsa_next_free_sgpr 96
|
||||
.amdhsa_wavefront_size32 1
|
||||
.amdhsa_user_sgpr_kernarg_segment_ptr 1
|
||||
.amdhsa_kernarg_size 8
|
||||
.amdhsa_group_segment_fixed_size 65536
|
||||
.amdhsa_private_segment_fixed_size 65536
|
||||
.amdhsa_enable_private_segment 1
|
||||
.end_amdhsa_kernel
|
||||
|
||||
.amdgpu_metadata
|
||||
---
|
||||
amdhsa.version:
|
||||
- 1
|
||||
- 0
|
||||
amdhsa.kernels:
|
||||
- .name: test
|
||||
.symbol: test.kd
|
||||
.kernarg_segment_size: 8
|
||||
.group_segment_fixed_size: 65536
|
||||
.private_segment_fixed_size: 65536
|
||||
.kernarg_segment_align: 8
|
||||
.wavefront_size: 32
|
||||
.sgpr_count: 96
|
||||
.vgpr_count: 256
|
||||
.max_flat_workgroup_size: 1024
|
||||
...
|
||||
.end_amdgpu_metadata
|
||||
"""
|
||||
|
||||
lib = compiler.compile(asm_src)
|
||||
prg = AMDProgram(dev, "test", lib) # type: ignore[arg-type]
|
||||
|
||||
buf_sz = _out_bytes(n_lanes)
|
||||
out_gpu = dev.allocator.alloc(buf_sz)
|
||||
assert out_gpu.va_addr % 16 == 0, f"buffer not 16-byte aligned: 0x{out_gpu.va_addr:x}"
|
||||
prg(out_gpu, global_size=(1, 1, 1), local_size=(n_lanes, 1, 1), wait=True)
|
||||
|
||||
out_buf = bytearray(buf_sz)
|
||||
dev.allocator._copyout(flat_mv(memoryview(out_buf)), out_gpu)
|
||||
|
||||
return parse_output(bytes(out_buf), n_lanes)
|
||||
|
||||
def compare_wave_states(emu_st: WaveState, hw_st: WaveState, n_lanes: int, n_vgprs: int = N_VGPRS, ulp_tolerance: int = 0) -> list[str]:
|
||||
"""Compare two WaveStates and return list of differences.
|
||||
|
||||
Args:
|
||||
ulp_tolerance: Allow up to this many ULPs difference for float comparisons (0 = exact match required)
|
||||
"""
|
||||
import math
|
||||
diffs = []
|
||||
for i in range(n_vgprs):
|
||||
for lane in range(n_lanes):
|
||||
emu_val = emu_st.vgpr[lane][i]
|
||||
hw_val = hw_st.vgpr[lane][i]
|
||||
if emu_val != hw_val:
|
||||
emu_f, hw_f = _f32(emu_val), _f32(hw_val)
|
||||
if math.isnan(emu_f) and math.isnan(hw_f):
|
||||
continue
|
||||
# Check ULP difference for floats (only for same-sign values)
|
||||
if ulp_tolerance > 0 and (emu_val < 0x80000000) == (hw_val < 0x80000000):
|
||||
ulp_diff = abs(int(emu_val) - int(hw_val))
|
||||
if ulp_diff <= ulp_tolerance:
|
||||
continue
|
||||
diffs.append(f"v[{i}] lane {lane}: emu=0x{emu_val:08x} ({emu_f:.6g}) hw=0x{hw_val:08x} ({hw_f:.6g})")
|
||||
for i in range(N_SGPRS):
|
||||
emu_val = emu_st.sgpr[i]
|
||||
hw_val = hw_st.sgpr[i]
|
||||
if emu_val != hw_val:
|
||||
diffs.append(f"s[{i}]: emu=0x{emu_val:08x} hw=0x{hw_val:08x}")
|
||||
if emu_st.vcc != hw_st.vcc:
|
||||
diffs.append(f"vcc: emu=0x{emu_st.vcc:08x} hw=0x{hw_st.vcc:08x}")
|
||||
if emu_st.scc != hw_st.scc:
|
||||
diffs.append(f"scc: emu={emu_st.scc} hw={hw_st.scc}")
|
||||
return diffs
|
||||
|
||||
def run_program(instructions: list, n_lanes: int = 1, ulp_tolerance: int = 0) -> WaveState:
|
||||
"""Run instructions and return WaveState.
|
||||
|
||||
If USE_HW=1, runs on both emulator and hardware, compares results, and raises if they differ.
|
||||
Otherwise, runs only on emulator.
|
||||
|
||||
Args:
|
||||
ulp_tolerance: Allow up to this many ULPs difference for float comparisons (0 = exact match required)
|
||||
"""
|
||||
emu_st = run_program_emu(instructions, n_lanes)
|
||||
if USE_HW:
|
||||
hw_st = run_program_hw(instructions, n_lanes)
|
||||
diffs = compare_wave_states(emu_st, hw_st, n_lanes, ulp_tolerance=ulp_tolerance)
|
||||
if diffs:
|
||||
raise AssertionError("Emulator vs Hardware mismatch:\n" + "\n".join(diffs))
|
||||
return hw_st
|
||||
return emu_st
|
||||
20
tinygrad_repo/test/amd/hw/test_cdna_sdwa.py
Normal file
20
tinygrad_repo/test/amd/hw/test_cdna_sdwa.py
Normal file
@@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env python3
|
||||
import unittest
|
||||
import tinygrad.runtime.autogen.amd.cdna.ins as cdna
|
||||
from test.amd.hw.test_cdna_vop3 import run_cdna
|
||||
|
||||
class TestCDNASDWA(unittest.TestCase):
|
||||
def test_v_add_co_u32_e32_writes_vcc(self):
|
||||
out = run_cdna([
|
||||
cdna.s_mov_b32(cdna.s[0], 0xffffffff),
|
||||
cdna.v_mov_b32_e32(cdna.v[0], cdna.s[0]),
|
||||
cdna.v_mov_b32_e32(cdna.v[13], 1),
|
||||
cdna.v_add_co_u32_e32(cdna.v[0], cdna.SDWA, cdna.v[13], vsrc0=cdna.v[0], dst_sel=6, src0_sel=6),
|
||||
cdna.v_mov_b32_e32(cdna.v[2], cdna.VCC_LO),
|
||||
cdna.v_lshlrev_b32_e32(cdna.v[2], 31, cdna.v[2]),
|
||||
cdna.v_or_b32_e32(cdna.v[2], cdna.v[2], cdna.v[0]),
|
||||
])
|
||||
self.assertEqual(out, 0x80000000)
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
130
tinygrad_repo/test/amd/hw/test_cdna_vop3.py
Normal file
130
tinygrad_repo/test/amd/hw/test_cdna_vop3.py
Normal file
@@ -0,0 +1,130 @@
|
||||
"""CDNA VOP3 instruction coverage.
|
||||
|
||||
Exercises generated CDNA pcode end-to-end in the emulator and compares against
|
||||
gfx950 hardware when USE_HW=1.
|
||||
"""
|
||||
import ctypes, struct, unittest
|
||||
import tinygrad.runtime.autogen.amd.cdna.ins as cdna
|
||||
from tinygrad.helpers import flat_mv
|
||||
from tinygrad.renderer.amd.dsl import NULL
|
||||
from test.amd.hw.helpers import USE_HW, assemble
|
||||
from test.mockgpu.amd.emu import run_asm
|
||||
|
||||
LANES = 1
|
||||
|
||||
def _code(instructions: list, out_reg: int = 2, out_addr: int | None = None) -> bytes:
|
||||
load_out_addr = [
|
||||
cdna.s_mov_b32(cdna.s[92], out_addr & 0xffffffff),
|
||||
cdna.s_mov_b32(cdna.s[93], out_addr >> 32),
|
||||
] if out_addr is not None else [
|
||||
cdna.s_load_dwordx2(cdna.s[92:93], cdna.s[80:81], 0, soffset=NULL),
|
||||
cdna.s_waitcnt(0),
|
||||
]
|
||||
return assemble([
|
||||
cdna.s_mov_b32(cdna.s[80], cdna.s[0]),
|
||||
cdna.s_mov_b32(cdna.s[81], cdna.s[1]),
|
||||
cdna.v_mov_b32_e32(cdna.v[255], cdna.v[0]),
|
||||
*instructions,
|
||||
*load_out_addr,
|
||||
cdna.v_lshlrev_b32_e32(cdna.v[240], 2, cdna.v[255]),
|
||||
cdna.global_store_dword(addr=cdna.v[240], data=cdna.v[out_reg], saddr=cdna.s[92:93], offset=0),
|
||||
cdna.s_endpgm(),
|
||||
])
|
||||
|
||||
def _run_emu(instructions: list, out_reg: int = 2) -> int:
|
||||
out_buf = (ctypes.c_uint32 * LANES)(*([0] * LANES))
|
||||
args = (ctypes.c_uint64 * 1)(ctypes.addressof(out_buf))
|
||||
code = _code(instructions, out_reg)
|
||||
kernel_buf = (ctypes.c_char * len(code)).from_buffer_copy(code)
|
||||
result = run_asm(ctypes.addressof(kernel_buf), len(code), 1, 1, 1, LANES, 1, 1, ctypes.addressof(args),
|
||||
0x19c | (128 << 15), 0x10000, arch="cdna")
|
||||
assert result == 0, f"run_asm failed with {result}"
|
||||
return out_buf[0]
|
||||
|
||||
def _run_hw(instructions: list, out_reg: int = 2) -> int:
|
||||
from tinygrad.device import Device
|
||||
from tinygrad.runtime.ops_amd import AMDProgram
|
||||
from tinygrad.runtime.support.compiler_amd import HIPCompiler
|
||||
|
||||
dev = Device["AMD"]
|
||||
if dev.arch != "gfx950": raise unittest.SkipTest("requires gfx950 hardware")
|
||||
out_gpu = dev.allocator.alloc(LANES * 4)
|
||||
code = _code(instructions, out_reg, out_gpu.va_addr)
|
||||
byte_str = ", ".join(f"0x{b:02x}" for b in code)
|
||||
asm_src = f""".text
|
||||
.globl test
|
||||
.p2align 8
|
||||
.type test,@function
|
||||
test:
|
||||
.byte {byte_str}
|
||||
|
||||
.rodata
|
||||
.p2align 6
|
||||
.amdhsa_kernel test
|
||||
.amdhsa_next_free_vgpr 256
|
||||
.amdhsa_next_free_sgpr 96
|
||||
.amdhsa_accum_offset 256
|
||||
.amdhsa_kernarg_size 0
|
||||
.end_amdhsa_kernel
|
||||
|
||||
.amdgpu_metadata
|
||||
---
|
||||
amdhsa.version:
|
||||
- 1
|
||||
- 0
|
||||
amdhsa.kernels:
|
||||
- .name: test
|
||||
.symbol: test.kd
|
||||
.kernarg_segment_size: 0
|
||||
.group_segment_fixed_size: 0
|
||||
.private_segment_fixed_size: 0
|
||||
.kernarg_segment_align: 8
|
||||
.wavefront_size: 64
|
||||
.sgpr_count: 96
|
||||
.vgpr_count: 256
|
||||
.max_flat_workgroup_size: 1024
|
||||
...
|
||||
.end_amdgpu_metadata
|
||||
"""
|
||||
prg = AMDProgram(dev, "test", HIPCompiler(dev.arch).compile(asm_src))
|
||||
prg(global_size=(1, 1, 1), local_size=(LANES, 1, 1), wait=True)
|
||||
out = bytearray(LANES * 4)
|
||||
dev.allocator._copyout(flat_mv(memoryview(out)), out_gpu)
|
||||
return struct.unpack("<I", out)[0]
|
||||
|
||||
def run_cdna(instructions: list, out_reg: int = 2) -> int:
|
||||
emu = _run_emu(instructions, out_reg)
|
||||
if not USE_HW: return emu
|
||||
hw = _run_hw(instructions, out_reg)
|
||||
if emu != hw: raise AssertionError(f"Emulator vs Hardware mismatch: emu=0x{emu:08x} hw=0x{hw:08x}")
|
||||
return hw
|
||||
|
||||
class TestCDNAVOP3(unittest.TestCase):
|
||||
def test_cvt_pk_fp8_f32_preserves_upper_half(self):
|
||||
"""V_CVT_PK_FP8_F32 with OPSEL[3]=0 writes only D[15:0]."""
|
||||
out = run_cdna([
|
||||
cdna.s_mov_b32(cdna.s[0], 0xdeadbeef),
|
||||
cdna.v_mov_b32_e32(cdna.v[2], cdna.s[0]),
|
||||
cdna.v_mov_b32_e32(cdna.v[0], 1.0),
|
||||
cdna.v_mov_b32_e32(cdna.v[1], 2.0),
|
||||
cdna.v_cvt_pk_fp8_f32(cdna.v[2], cdna.v[0], cdna.v[1]),
|
||||
])
|
||||
self.assertEqual(out, 0xdead4038)
|
||||
|
||||
def test_cvt_pk_bf8_f32_overflow_and_inf(self):
|
||||
"""V_CVT_PK_BF8_F32 converts finite overflow and infinities to E5M2 infinities."""
|
||||
for name, bits, expected in [
|
||||
("finite_overflow", 0x47700000, 0x7c),
|
||||
("pos_inf", 0x7f800000, 0x7c),
|
||||
("neg_inf", 0xff800000, 0xfc),
|
||||
]:
|
||||
with self.subTest(name=name):
|
||||
out = run_cdna([
|
||||
cdna.s_mov_b32(cdna.s[0], 0xdeadbeef),
|
||||
cdna.v_mov_b32_e32(cdna.v[2], cdna.s[0]),
|
||||
cdna.s_mov_b32(cdna.s[0], bits),
|
||||
cdna.v_mov_b32_e32(cdna.v[0], cdna.s[0]),
|
||||
cdna.v_mov_b32_e32(cdna.v[1], 1.0),
|
||||
cdna.v_cvt_pk_bf8_f32(cdna.v[2], cdna.v[0], cdna.v[1]),
|
||||
])
|
||||
self.assertEqual(out, 0xdead3c00 | expected)
|
||||
187
tinygrad_repo/test/amd/hw/test_dpp.py
Normal file
187
tinygrad_repo/test/amd/hw/test_dpp.py
Normal file
@@ -0,0 +1,187 @@
|
||||
"""Tests for DPP16 source swizzles.
|
||||
|
||||
These instructions trap in the default wave32 hw helper, so this file uses a
|
||||
minimal wave64 lane-store harness and compares emulator vs hardware directly
|
||||
when USE_HW=1.
|
||||
"""
|
||||
import ctypes, unittest
|
||||
from tinygrad.runtime.autogen.amd.rdna3.ins import *
|
||||
from tinygrad.helpers import flat_mv
|
||||
from test.amd.hw.helpers import USE_HW, assemble
|
||||
from test.mockgpu.amd.emu import run_asm
|
||||
|
||||
WAVE64 = 64
|
||||
|
||||
def _wave64_code(instructions: list, out_reg: int = 1) -> bytes:
|
||||
return assemble([
|
||||
s_mov_b32(s[80], s[0]),
|
||||
s_mov_b32(s[81], s[1]),
|
||||
v_mov_b32_e32(v[255], v[0]),
|
||||
*instructions,
|
||||
s_load_b64(s[92:93], s[80:81], 0, soffset=NULL),
|
||||
s_waitcnt(0),
|
||||
v_lshlrev_b32_e32(v[240], 2, v[255]),
|
||||
global_store_b32(addr=v[240], data=v[out_reg], saddr=s[92:93], offset=0),
|
||||
s_endpgm(),
|
||||
])
|
||||
|
||||
def _run_wave64_emu(instructions: list, out_reg: int = 1) -> list[int]:
|
||||
out_buf = (ctypes.c_uint32 * WAVE64)(*([0] * WAVE64))
|
||||
args = (ctypes.c_uint64 * 1)(ctypes.addressof(out_buf))
|
||||
code = _wave64_code(instructions, out_reg)
|
||||
kernel_buf = (ctypes.c_char * len(code)).from_buffer_copy(code)
|
||||
rsrc2 = 0x19c | (128 << 15)
|
||||
scratch_size = 0x10000
|
||||
result = run_asm(ctypes.addressof(kernel_buf), len(code), 1, 1, 1, WAVE64, 1, 1, ctypes.addressof(args), rsrc2, scratch_size)
|
||||
assert result == 0, f"run_asm failed with {result}"
|
||||
return list(out_buf)
|
||||
|
||||
def _run_wave64_hw(instructions: list, out_reg: int = 1) -> list[int]:
|
||||
from tinygrad.device import Device
|
||||
from tinygrad.runtime.ops_amd import AMDProgram
|
||||
from tinygrad.runtime.support.compiler_amd import HIPCompiler
|
||||
|
||||
dev = Device["AMD"]
|
||||
compiler = HIPCompiler(dev.arch) # type: ignore[attr-defined]
|
||||
code = _wave64_code(instructions, out_reg)
|
||||
byte_str = ', '.join(f'0x{b:02x}' for b in code)
|
||||
asm_src = f""".text
|
||||
.globl test
|
||||
.p2align 8
|
||||
.type test,@function
|
||||
test:
|
||||
.byte {byte_str}
|
||||
|
||||
.rodata
|
||||
.p2align 6
|
||||
.amdhsa_kernel test
|
||||
.amdhsa_next_free_vgpr 256
|
||||
.amdhsa_next_free_sgpr 96
|
||||
.amdhsa_user_sgpr_kernarg_segment_ptr 1
|
||||
.amdhsa_kernarg_size 8
|
||||
.amdhsa_group_segment_fixed_size 65536
|
||||
.amdhsa_private_segment_fixed_size 65536
|
||||
.amdhsa_enable_private_segment 1
|
||||
.end_amdhsa_kernel
|
||||
|
||||
.amdgpu_metadata
|
||||
---
|
||||
amdhsa.version:
|
||||
- 1
|
||||
- 0
|
||||
amdhsa.kernels:
|
||||
- .name: test
|
||||
.symbol: test.kd
|
||||
.kernarg_segment_size: 8
|
||||
.group_segment_fixed_size: 65536
|
||||
.private_segment_fixed_size: 65536
|
||||
.kernarg_segment_align: 8
|
||||
.wavefront_size: 64
|
||||
.sgpr_count: 96
|
||||
.vgpr_count: 256
|
||||
.max_flat_workgroup_size: 1024
|
||||
...
|
||||
.end_amdgpu_metadata
|
||||
"""
|
||||
lib = compiler.compile(asm_src)
|
||||
prg = AMDProgram(dev, "test", lib) # type: ignore[arg-type]
|
||||
out_gpu = dev.allocator.alloc(WAVE64 * 4)
|
||||
prg(out_gpu, global_size=(1, 1, 1), local_size=(WAVE64, 1, 1), wait=True)
|
||||
out = bytearray(WAVE64 * 4)
|
||||
dev.allocator._copyout(flat_mv(memoryview(out)), out_gpu)
|
||||
return [int.from_bytes(out[i*4:(i+1)*4], 'little') for i in range(WAVE64)]
|
||||
|
||||
def run_wave64(instructions: list, out_reg: int = 1) -> list[int]:
|
||||
emu = _run_wave64_emu(instructions, out_reg)
|
||||
if not USE_HW: return emu
|
||||
hw = _run_wave64_hw(instructions, out_reg)
|
||||
if emu != hw:
|
||||
diffs = [f"lane {i}: emu=0x{e:08x} hw=0x{h:08x}" for i, (e, h) in enumerate(zip(emu, hw)) if e != h]
|
||||
raise AssertionError("Emulator vs Hardware mismatch:\n" + '\n'.join(diffs[:16]))
|
||||
return hw
|
||||
|
||||
class TestDPP16(unittest.TestCase):
|
||||
def _run_copy(self, dpp: int, *, row_mask: int = 0xf, bank_mask: int = 0xf, bc: int = 1, dst_seed: int | None = None) -> list[int]:
|
||||
instructions = [
|
||||
v_mul_u32_u24_e32(v[0], 10, v[255]),
|
||||
v_add_nc_u32_e32(v[0], 3, v[0]),
|
||||
]
|
||||
if dst_seed is not None: instructions.append(v_mov_b32_e32(v[1], dst_seed))
|
||||
instructions += [v_mov_b32_e32(v[2], 0), v_or_b32_e32(v[1], DPP, v[2], vsrc0=v[0], dpp=dpp, row_mask=row_mask, bank_mask=bank_mask, bc=bc)]
|
||||
return run_wave64(instructions)
|
||||
|
||||
def test_quad_perm_reverse(self):
|
||||
out = self._run_copy(0x1b)
|
||||
self.assertEqual(out[0], 33)
|
||||
self.assertEqual(out[1], 23)
|
||||
self.assertEqual(out[2], 13)
|
||||
self.assertEqual(out[3], 3)
|
||||
self.assertEqual(out[4], 73)
|
||||
|
||||
def test_row_shl(self):
|
||||
out = self._run_copy(0x101)
|
||||
self.assertEqual(out[0], 13)
|
||||
self.assertEqual(out[7], 83)
|
||||
self.assertEqual(out[14], 153)
|
||||
self.assertEqual(out[15], 0)
|
||||
self.assertEqual(out[16], 173)
|
||||
|
||||
def test_row_shr(self):
|
||||
out = self._run_copy(0x111)
|
||||
self.assertEqual(out[0], 0)
|
||||
self.assertEqual(out[1], 3)
|
||||
self.assertEqual(out[8], 73)
|
||||
self.assertEqual(out[15], 143)
|
||||
self.assertEqual(out[16], 0)
|
||||
self.assertEqual(out[17], 163)
|
||||
|
||||
def test_row_ror(self):
|
||||
out = self._run_copy(0x121)
|
||||
self.assertEqual(out[0], 153)
|
||||
self.assertEqual(out[1], 3)
|
||||
self.assertEqual(out[15], 143)
|
||||
self.assertEqual(out[16], 313)
|
||||
|
||||
def test_row_mirror(self):
|
||||
out = self._run_copy(0x140)
|
||||
self.assertEqual(out[0], 153)
|
||||
self.assertEqual(out[5], 103)
|
||||
self.assertEqual(out[8], 73)
|
||||
self.assertEqual(out[16], 313)
|
||||
|
||||
def test_row_half_mirror(self):
|
||||
out = self._run_copy(0x141)
|
||||
self.assertEqual(out[0], 73)
|
||||
self.assertEqual(out[7], 3)
|
||||
self.assertEqual(out[8], 153)
|
||||
self.assertEqual(out[15], 83)
|
||||
self.assertEqual(out[16], 233)
|
||||
|
||||
def test_row_mask(self):
|
||||
out = self._run_copy(0x101, row_mask=0x5, dst_seed=0xDEADBEEF)
|
||||
self.assertEqual(out[0], 13)
|
||||
self.assertEqual(out[15], 0)
|
||||
self.assertEqual(out[16], 0xDEADBEEF)
|
||||
self.assertEqual(out[32], 333)
|
||||
self.assertEqual(out[47], 0)
|
||||
self.assertEqual(out[48], 0xDEADBEEF)
|
||||
|
||||
def test_bank_mask(self):
|
||||
out = self._run_copy(0x101, bank_mask=0x5, dst_seed=0xDEADBEEF)
|
||||
self.assertEqual(out[0], 13)
|
||||
self.assertEqual(out[3], 43)
|
||||
self.assertEqual(out[4], 0xDEADBEEF)
|
||||
self.assertEqual(out[8], 93)
|
||||
self.assertEqual(out[12], 0xDEADBEEF)
|
||||
|
||||
class TestVOPCDPP16(unittest.TestCase):
|
||||
def test_row_bcast15_materializes_vcc(self):
|
||||
out = run_wave64([
|
||||
v_mov_b32_e32(v[0], v[255]),
|
||||
v_cmp_eq_u32_e32(DPP, v[0], vsrc0=v[0], dpp=0x142, row_mask=0xf, bank_mask=0xf, bc=1),
|
||||
v_mov_b32_e32(v[2], 0),
|
||||
v_mov_b32_e32(v[3], 1),
|
||||
v_cndmask_b32_e32(v[1], v[2], v[3]),
|
||||
])
|
||||
for lane in (0, 16, 32, 48): self.assertEqual(out[lane], 1)
|
||||
for lane in (1, 15, 31, 47, 63): self.assertEqual(out[lane], 0)
|
||||
960
tinygrad_repo/test/amd/hw/test_ds.py
Normal file
960
tinygrad_repo/test/amd/hw/test_ds.py
Normal file
@@ -0,0 +1,960 @@
|
||||
"""Tests for DS instructions - data share (LDS) operations.
|
||||
|
||||
Includes: ds_store_b32, ds_load_b32, ds_store_2addr_*, ds_load_2addr_*,
|
||||
ds_add_*, ds_max_*, ds_min_*, ds_and_*, ds_or_*, ds_xor_*,
|
||||
ds_inc_*, ds_dec_*, ds_cmpstore_*, ds_storexchg_*
|
||||
"""
|
||||
import unittest
|
||||
from test.amd.hw.helpers import *
|
||||
|
||||
class TestDS2Addr(unittest.TestCase):
|
||||
"""Tests for DS_*_2ADDR instructions."""
|
||||
|
||||
def test_ds_store_load_2addr_b32(self):
|
||||
"""DS_STORE_2ADDR_B32 and DS_LOAD_2ADDR_B32 with offset * 4."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[0], 0xAAAAAAAA),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
s_mov_b32(s[0], 0xBBBBBBBB),
|
||||
v_mov_b32_e32(v[1], s[0]),
|
||||
DS(DSOp.DS_STORE_2ADDR_B32, addr=v[10], data0=v[0], data1=v[1], vdst=v[0], offset0=0, offset1=1),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
DS(DSOp.DS_LOAD_2ADDR_B32, addr=v[10], vdst=v[2:3], offset0=0, offset1=1),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 0xAAAAAAAA)
|
||||
self.assertEqual(st.vgpr[0][3], 0xBBBBBBBB)
|
||||
|
||||
def test_ds_store_load_2addr_b64(self):
|
||||
"""DS_STORE_2ADDR_B64 and DS_LOAD_2ADDR_B64."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[0], 0xDEADBEEF),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
s_mov_b32(s[0], 0xCAFEBABE),
|
||||
v_mov_b32_e32(v[1], s[0]),
|
||||
s_mov_b32(s[0], 0x12345678),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
s_mov_b32(s[0], 0x9ABCDEF0),
|
||||
v_mov_b32_e32(v[3], s[0]),
|
||||
DS(DSOp.DS_STORE_2ADDR_B64, addr=v[10], data0=v[0:1], data1=v[2:3], vdst=v[0], offset0=0, offset1=2),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
DS(DSOp.DS_LOAD_2ADDR_B64, addr=v[10], vdst=v[4:7], offset0=0, offset1=2),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][4], 0xDEADBEEF)
|
||||
self.assertEqual(st.vgpr[0][5], 0xCAFEBABE)
|
||||
self.assertEqual(st.vgpr[0][6], 0x12345678)
|
||||
self.assertEqual(st.vgpr[0][7], 0x9ABCDEF0)
|
||||
|
||||
|
||||
class TestDS2AddrMore(unittest.TestCase):
|
||||
"""Additional DS_*_2ADDR tests."""
|
||||
|
||||
def test_ds_store_load_2addr_b32_nonzero_offsets(self):
|
||||
"""DS_STORE_2ADDR_B32 with non-zero offsets (offset*4 scaling)."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[2], 0x11111111),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
s_mov_b32(s[2], 0x22222222),
|
||||
v_mov_b32_e32(v[1], s[2]),
|
||||
DS(DSOp.DS_STORE_2ADDR_B32, addr=v[10], data0=v[0], data1=v[1], vdst=v[0], offset0=2, offset1=5),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
DS(DSOp.DS_LOAD_2ADDR_B32, addr=v[10], vdst=v[2:3], offset0=2, offset1=5),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 0x11111111, "v2 should have value from offset 8 (2*4)")
|
||||
self.assertEqual(st.vgpr[0][3], 0x22222222, "v3 should have value from offset 20 (5*4)")
|
||||
|
||||
def test_ds_2addr_b64_no_overlap(self):
|
||||
"""DS_LOAD_2ADDR_B64 with adjacent offsets should not overlap."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[2], 0x11111111),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
ds_store_b32(addr=v[10], data0=v[0], offset0=0),
|
||||
s_mov_b32(s[2], 0x22222222),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
ds_store_b32(addr=v[10], data0=v[0], offset0=4),
|
||||
s_mov_b32(s[2], 0x33333333),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
ds_store_b32(addr=v[10], data0=v[0], offset0=8),
|
||||
s_mov_b32(s[2], 0x44444444),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
ds_store_b32(addr=v[10], data0=v[0], offset0=12),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
DS(DSOp.DS_LOAD_2ADDR_B64, addr=v[10], vdst=v[4:7], offset0=0, offset1=1),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][4], 0x11111111, "v4 should be 0x11111111")
|
||||
self.assertEqual(st.vgpr[0][5], 0x22222222, "v5 should be 0x22222222")
|
||||
self.assertEqual(st.vgpr[0][6], 0x33333333, "v6 should be 0x33333333")
|
||||
self.assertEqual(st.vgpr[0][7], 0x44444444, "v7 should be 0x44444444")
|
||||
|
||||
def test_ds_load_2addr_b32_no_overwrite(self):
|
||||
"""DS_LOAD_2ADDR_B32 should only write 2 VGPRs."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[2], 0xAAAAAAAA),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
s_mov_b32(s[2], 0xBBBBBBBB),
|
||||
v_mov_b32_e32(v[1], s[2]),
|
||||
DS(DSOp.DS_STORE_2ADDR_B32, addr=v[10], data0=v[0], data1=v[1], vdst=v[0], offset0=0, offset1=1),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[2], 0xDEADBEEF),
|
||||
v_mov_b32_e32(v[4], s[2]), # Sentinel
|
||||
DS(DSOp.DS_LOAD_2ADDR_B32, addr=v[10], vdst=v[2:3], offset0=0, offset1=1),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 0xAAAAAAAA)
|
||||
self.assertEqual(st.vgpr[0][3], 0xBBBBBBBB)
|
||||
self.assertEqual(st.vgpr[0][4], 0xDEADBEEF, "v4 should be untouched")
|
||||
|
||||
def test_ds_load_2addr_b64_addr_overlaps_vdst(self):
|
||||
"""DS_LOAD_2ADDR_B64 where addr register overlaps vdst range.
|
||||
|
||||
Hardware reads the address before writing any results, so addr=v[4]
|
||||
with vdst=v[4:7] must load all 4 dwords using the original v[4] value.
|
||||
"""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[2], 0xAAAAAAAA),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
ds_store_b32(addr=v[10], data0=v[0], offset0=0),
|
||||
s_mov_b32(s[2], 0xBBBBBBBB),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
ds_store_b32(addr=v[10], data0=v[0], offset0=4),
|
||||
s_mov_b32(s[2], 0xCCCCCCCC),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
ds_store_b32(addr=v[10], data0=v[0], offset0=8),
|
||||
s_mov_b32(s[2], 0xDDDDDDDD),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
ds_store_b32(addr=v[10], data0=v[0], offset0=12),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
# addr=v[4] overlaps vdst=v[4:7]
|
||||
v_mov_b32_e32(v[4], 0),
|
||||
DS(DSOp.DS_LOAD_2ADDR_B64, addr=v[4], vdst=v[4:7], offset0=0, offset1=1),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][4], 0xAAAAAAAA, "v4 = LDS[0:4]")
|
||||
self.assertEqual(st.vgpr[0][5], 0xBBBBBBBB, "v5 = LDS[4:8]")
|
||||
self.assertEqual(st.vgpr[0][6], 0xCCCCCCCC, "v6 = LDS[8:12]")
|
||||
self.assertEqual(st.vgpr[0][7], 0xDDDDDDDD, "v7 = LDS[12:16]")
|
||||
|
||||
def test_ds_load_2addr_b32_addr_overlaps_vdst(self):
|
||||
"""DS_LOAD_2ADDR_B32 where addr register overlaps vdst range."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[2], 0xAAAAAAAA),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
ds_store_b32(addr=v[10], data0=v[0], offset0=0),
|
||||
s_mov_b32(s[2], 0xBBBBBBBB),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
ds_store_b32(addr=v[10], data0=v[0], offset0=4),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
# addr=v[2] overlaps vdst=v[2:3]
|
||||
v_mov_b32_e32(v[2], 0),
|
||||
DS(DSOp.DS_LOAD_2ADDR_B32, addr=v[2], vdst=v[2:3], offset0=0, offset1=1),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 0xAAAAAAAA, "v2 = LDS[0:4]")
|
||||
self.assertEqual(st.vgpr[0][3], 0xBBBBBBBB, "v3 = LDS[4:8]")
|
||||
|
||||
def test_ds_load_b64_no_overwrite(self):
|
||||
"""DS_LOAD_B64 should only write 2 VGPRs."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[2], 0xDEADBEEF),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
s_mov_b32(s[2], 0xCAFEBABE),
|
||||
v_mov_b32_e32(v[1], s[2]),
|
||||
ds_store_b64(addr=v[10], data0=v[0:1], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[2], 0x12345678),
|
||||
v_mov_b32_e32(v[4], s[2]), # Sentinel
|
||||
ds_load_b64(addr=v[10], vdst=v[2:3], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 0xDEADBEEF)
|
||||
self.assertEqual(st.vgpr[0][3], 0xCAFEBABE)
|
||||
self.assertEqual(st.vgpr[0][4], 0x12345678, "v4 should be untouched")
|
||||
|
||||
|
||||
class TestDSB96(unittest.TestCase):
|
||||
"""Tests for DS_STORE_B96 and DS_LOAD_B96 (96-bit / 3 dwords)."""
|
||||
|
||||
def test_ds_store_load_b96(self):
|
||||
"""DS_STORE_B96 stores 3 VGPRs, DS_LOAD_B96 loads them back."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[0], 0x11111111),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
s_mov_b32(s[0], 0x22222222),
|
||||
v_mov_b32_e32(v[1], s[0]),
|
||||
s_mov_b32(s[0], 0x33333333),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
ds_store_b96(addr=v[10], data0=v[0:2]),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
ds_load_b96(addr=v[10], vdst=v[4:6]),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][4], 0x11111111, "v4 should have first dword")
|
||||
self.assertEqual(st.vgpr[0][5], 0x22222222, "v5 should have second dword")
|
||||
self.assertEqual(st.vgpr[0][6], 0x33333333, "v6 should have third dword")
|
||||
|
||||
def test_ds_store_b96_with_offset(self):
|
||||
"""DS_STORE_B96 with non-zero offset."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[0], 0xAAAAAAAA),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
s_mov_b32(s[0], 0xBBBBBBBB),
|
||||
v_mov_b32_e32(v[1], s[0]),
|
||||
s_mov_b32(s[0], 0xCCCCCCCC),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
DS(DSOp.DS_STORE_B96, addr=v[10], data0=v[0:2], offset0=12),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
DS(DSOp.DS_LOAD_B96, addr=v[10], vdst=v[4:6], offset0=12),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][4], 0xAAAAAAAA)
|
||||
self.assertEqual(st.vgpr[0][5], 0xBBBBBBBB)
|
||||
self.assertEqual(st.vgpr[0][6], 0xCCCCCCCC)
|
||||
|
||||
|
||||
class TestDSB128(unittest.TestCase):
|
||||
"""Tests for DS_STORE_B128 and DS_LOAD_B128 (128-bit / 4 dwords)."""
|
||||
|
||||
def test_ds_store_load_b128(self):
|
||||
"""DS_STORE_B128 stores 4 VGPRs, DS_LOAD_B128 loads them back."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[0], 0x11111111),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
s_mov_b32(s[0], 0x22222222),
|
||||
v_mov_b32_e32(v[1], s[0]),
|
||||
s_mov_b32(s[0], 0x33333333),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
s_mov_b32(s[0], 0x44444444),
|
||||
v_mov_b32_e32(v[3], s[0]),
|
||||
ds_store_b128(addr=v[10], data0=v[0:3]),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
ds_load_b128(addr=v[10], vdst=v[4:7]),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][4], 0x11111111, "v4 should have first dword")
|
||||
self.assertEqual(st.vgpr[0][5], 0x22222222, "v5 should have second dword")
|
||||
self.assertEqual(st.vgpr[0][6], 0x33333333, "v6 should have third dword")
|
||||
self.assertEqual(st.vgpr[0][7], 0x44444444, "v7 should have fourth dword")
|
||||
|
||||
def test_ds_store_b128_with_offset(self):
|
||||
"""DS_STORE_B128 with non-zero offset."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[0], 0xAAAAAAAA),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
s_mov_b32(s[0], 0xBBBBBBBB),
|
||||
v_mov_b32_e32(v[1], s[0]),
|
||||
s_mov_b32(s[0], 0xCCCCCCCC),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
s_mov_b32(s[0], 0xDDDDDDDD),
|
||||
v_mov_b32_e32(v[3], s[0]),
|
||||
DS(DSOp.DS_STORE_B128, addr=v[10], data0=v[0:3], offset0=16),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
DS(DSOp.DS_LOAD_B128, addr=v[10], vdst=v[4:7], offset0=16),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][4], 0xAAAAAAAA)
|
||||
self.assertEqual(st.vgpr[0][5], 0xBBBBBBBB)
|
||||
self.assertEqual(st.vgpr[0][6], 0xCCCCCCCC)
|
||||
self.assertEqual(st.vgpr[0][7], 0xDDDDDDDD)
|
||||
|
||||
|
||||
class TestDSAtomic(unittest.TestCase):
|
||||
"""Tests for DS atomic operations."""
|
||||
|
||||
def test_ds_max_rtn_u32(self):
|
||||
"""DS_MAX_RTN_U32: atomically store max and return old value."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[2], 100),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
ds_store_b32(addr=v[10], data0=v[0], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[2], 200),
|
||||
v_mov_b32_e32(v[1], s[2]),
|
||||
ds_max_rtn_u32(addr=v[10], data0=v[1], vdst=v[2], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
ds_load_b32(addr=v[10], vdst=v[3], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 100, "v2 should have old value (100)")
|
||||
self.assertEqual(st.vgpr[0][3], 200, "v3 should have max(100, 200) = 200")
|
||||
|
||||
def test_ds_min_rtn_u32(self):
|
||||
"""DS_MIN_RTN_U32: atomically store min and return old value."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[2], 200),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
ds_store_b32(addr=v[10], data0=v[0], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[2], 100),
|
||||
v_mov_b32_e32(v[1], s[2]),
|
||||
ds_min_rtn_u32(addr=v[10], data0=v[1], vdst=v[2], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
ds_load_b32(addr=v[10], vdst=v[3], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 200)
|
||||
self.assertEqual(st.vgpr[0][3], 100)
|
||||
|
||||
def test_ds_and_rtn_b32(self):
|
||||
"""DS_AND_RTN_B32: atomically AND and return old value."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[2], 0xFF00FF00),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
ds_store_b32(addr=v[10], data0=v[0], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[2], 0xFFFF0000),
|
||||
v_mov_b32_e32(v[1], s[2]),
|
||||
ds_and_rtn_b32(addr=v[10], data0=v[1], vdst=v[2], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
ds_load_b32(addr=v[10], vdst=v[3], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 0xFF00FF00)
|
||||
self.assertEqual(st.vgpr[0][3], 0xFF000000)
|
||||
|
||||
def test_ds_or_rtn_b32(self):
|
||||
"""DS_OR_RTN_B32: atomically OR and return old value."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[2], 0x00FF0000),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
ds_store_b32(addr=v[10], data0=v[0], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[2], 0x000000FF),
|
||||
v_mov_b32_e32(v[1], s[2]),
|
||||
ds_or_rtn_b32(addr=v[10], data0=v[1], vdst=v[2], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
ds_load_b32(addr=v[10], vdst=v[3], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 0x00FF0000)
|
||||
self.assertEqual(st.vgpr[0][3], 0x00FF00FF)
|
||||
|
||||
def test_ds_xor_rtn_b32(self):
|
||||
"""DS_XOR_RTN_B32: atomically XOR and return old value."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[2], 0xAAAAAAAA),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
ds_store_b32(addr=v[10], data0=v[0], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[2], 0xFFFFFFFF),
|
||||
v_mov_b32_e32(v[1], s[2]),
|
||||
ds_xor_rtn_b32(addr=v[10], data0=v[1], vdst=v[2], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
ds_load_b32(addr=v[10], vdst=v[3], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 0xAAAAAAAA)
|
||||
self.assertEqual(st.vgpr[0][3], 0x55555555)
|
||||
|
||||
def test_ds_inc_rtn_u32(self):
|
||||
"""DS_INC_RTN_U32: increment with wrap."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[2], 5),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
ds_store_b32(addr=v[10], data0=v[0], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[2], 10), # limit
|
||||
v_mov_b32_e32(v[1], s[2]),
|
||||
ds_inc_rtn_u32(addr=v[10], data0=v[1], vdst=v[2], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
ds_load_b32(addr=v[10], vdst=v[3], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 5)
|
||||
self.assertEqual(st.vgpr[0][3], 6)
|
||||
|
||||
def test_ds_dec_rtn_u32(self):
|
||||
"""DS_DEC_RTN_U32: decrement with wrap."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[2], 5),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
ds_store_b32(addr=v[10], data0=v[0], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[2], 10), # limit
|
||||
v_mov_b32_e32(v[1], s[2]),
|
||||
ds_dec_rtn_u32(addr=v[10], data0=v[1], vdst=v[2], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
ds_load_b32(addr=v[10], vdst=v[3], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 5)
|
||||
self.assertEqual(st.vgpr[0][3], 4)
|
||||
|
||||
def test_ds_cmpstore_b32_match(self):
|
||||
"""DS_CMPSTORE_B32: conditional store when compare matches."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[2], 100),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
ds_store_b32(addr=v[10], data0=v[0], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[2], 200),
|
||||
v_mov_b32_e32(v[1], s[2]), # new value
|
||||
s_mov_b32(s[2], 100),
|
||||
v_mov_b32_e32(v[2], s[2]), # compare = 100 (matches)
|
||||
ds_cmpstore_b32(addr=v[10], data0=v[1], data1=v[2], vdst=v[3], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
ds_load_b32(addr=v[10], vdst=v[4], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][4], 200)
|
||||
|
||||
def test_ds_cmpstore_b32_no_match(self):
|
||||
"""DS_CMPSTORE_B32: no store when compare doesn't match."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[2], 100),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
ds_store_b32(addr=v[10], data0=v[0], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[2], 200),
|
||||
v_mov_b32_e32(v[1], s[2]), # new value
|
||||
s_mov_b32(s[2], 50),
|
||||
v_mov_b32_e32(v[2], s[2]), # compare = 50 (doesn't match)
|
||||
ds_cmpstore_b32(addr=v[10], data0=v[1], data1=v[2], vdst=v[3], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
ds_load_b32(addr=v[10], vdst=v[4], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][4], 100)
|
||||
|
||||
def test_ds_max_u32_no_rtn(self):
|
||||
"""DS_MAX_U32 (no RTN): atomically store max, no return value."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[2], 100),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
ds_store_b32(addr=v[10], data0=v[0], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[2], 200),
|
||||
v_mov_b32_e32(v[1], s[2]),
|
||||
ds_max_u32(addr=v[10], data0=v[1], vdst=v[2], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
ds_load_b32(addr=v[10], vdst=v[3], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][3], 200, "v3 should have max(100, 200) = 200")
|
||||
|
||||
def test_ds_add_u32_no_rtn_preserves_vdst(self):
|
||||
"""DS_ADD_U32 (no RTN) should NOT write to vdst."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[2], 0xDEADBEEF),
|
||||
v_mov_b32_e32(v[2], s[2]), # sentinel
|
||||
s_mov_b32(s[2], 100),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
ds_store_b32(addr=v[10], data0=v[0], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[2], 50),
|
||||
v_mov_b32_e32(v[1], s[2]),
|
||||
ds_add_u32(addr=v[10], data0=v[1], vdst=v[2], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
ds_load_b32(addr=v[10], vdst=v[3], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 0xDEADBEEF, "v2 should preserve sentinel")
|
||||
self.assertEqual(st.vgpr[0][3], 150, "v3 should have 100 + 50 = 150")
|
||||
|
||||
def test_ds_add_rtn_u32_writes_vdst(self):
|
||||
"""DS_ADD_RTN_U32 should write old value to vdst."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[2], 0xDEADBEEF),
|
||||
v_mov_b32_e32(v[2], s[2]), # sentinel
|
||||
s_mov_b32(s[2], 100),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
ds_store_b32(addr=v[10], data0=v[0], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[2], 50),
|
||||
v_mov_b32_e32(v[1], s[2]),
|
||||
ds_add_rtn_u32(addr=v[10], data0=v[1], vdst=v[2], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
ds_load_b32(addr=v[10], vdst=v[3], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 100, "v2 should have old value (100)")
|
||||
self.assertEqual(st.vgpr[0][3], 150, "v3 should have 100 + 50 = 150")
|
||||
|
||||
def test_ds_dec_rtn_u32_wrap(self):
|
||||
"""DS_DEC_RTN_U32: decrement wraps when value is 0 or > limit."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[2], 0), # Start at 0
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
ds_store_b32(addr=v[10], data0=v[0], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[2], 10), # limit
|
||||
v_mov_b32_e32(v[1], s[2]),
|
||||
ds_dec_rtn_u32(addr=v[10], data0=v[1], vdst=v[2], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
ds_load_b32(addr=v[10], vdst=v[3], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 0, "v2 should have old value (0)")
|
||||
# When mem == 0 or mem > limit, result = limit
|
||||
self.assertEqual(st.vgpr[0][3], 10, "v3 should wrap to limit (10)")
|
||||
|
||||
|
||||
class TestDSStorexchg(unittest.TestCase):
|
||||
"""Tests for DS_STOREXCHG instructions."""
|
||||
|
||||
def test_ds_storexchg_rtn_b32(self):
|
||||
"""DS_STOREXCHG_RTN_B32: exchange value and return old."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[0], 0xAAAAAAAA),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
ds_store_b32(addr=v[10], data0=v[0], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[0], 0xBBBBBBBB),
|
||||
v_mov_b32_e32(v[1], s[0]),
|
||||
DS(DSOp.DS_STOREXCHG_RTN_B32, addr=v[10], data0=v[1], vdst=v[2], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
ds_load_b32(addr=v[10], vdst=v[3], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 0xAAAAAAAA)
|
||||
self.assertEqual(st.vgpr[0][3], 0xBBBBBBBB)
|
||||
|
||||
|
||||
class TestDSRegisterWidth(unittest.TestCase):
|
||||
"""Regression tests: DS loads should only write correct number of VGPRs."""
|
||||
|
||||
def test_ds_load_b32_no_overwrite(self):
|
||||
"""DS_LOAD_B32 should only write 1 VGPR."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
s_mov_b32(s[0], 0xDEADBEEF),
|
||||
v_mov_b32_e32(v[1], s[0]),
|
||||
s_mov_b32(s[0], 0x11111111),
|
||||
v_mov_b32_e32(v[2], s[0]), # sentinel
|
||||
ds_store_b32(addr=v[0], data0=v[1], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
ds_load_b32(addr=v[0], vdst=v[1], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][1], 0xDEADBEEF)
|
||||
self.assertEqual(st.vgpr[0][2], 0x11111111, "v2 should be untouched")
|
||||
|
||||
|
||||
class TestDS2AddrStride64(unittest.TestCase):
|
||||
"""Tests for DS_*_2ADDR_STRIDE64 (offset * 256 for B32, offset * 512 for B64)."""
|
||||
|
||||
def test_ds_store_load_2addr_stride64_b32(self):
|
||||
"""DS_STORE_2ADDR_STRIDE64_B32: stores at ADDR + offset*256."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[0], 0xAAAAAAAA),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
s_mov_b32(s[0], 0xBBBBBBBB),
|
||||
v_mov_b32_e32(v[1], s[0]),
|
||||
DS(DSOp.DS_STORE_2ADDR_STRIDE64_B32, addr=v[10], data0=v[0], data1=v[1], vdst=v[0], offset0=1, offset1=2),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
DS(DSOp.DS_LOAD_2ADDR_STRIDE64_B32, addr=v[10], vdst=v[2:3], offset0=1, offset1=2),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 0xAAAAAAAA, "v2 from addr 256")
|
||||
self.assertEqual(st.vgpr[0][3], 0xBBBBBBBB, "v3 from addr 512")
|
||||
|
||||
def test_ds_store_load_2addr_stride64_b64(self):
|
||||
"""DS_STORE_2ADDR_STRIDE64_B64: stores at ADDR + offset*512."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[0], 0xDEADBEEF),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
s_mov_b32(s[0], 0xCAFEBABE),
|
||||
v_mov_b32_e32(v[1], s[0]),
|
||||
s_mov_b32(s[0], 0x12345678),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
s_mov_b32(s[0], 0x9ABCDEF0),
|
||||
v_mov_b32_e32(v[3], s[0]),
|
||||
DS(DSOp.DS_STORE_2ADDR_STRIDE64_B64, addr=v[10], data0=v[0:1], data1=v[2:3], vdst=v[0], offset0=1, offset1=2),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
DS(DSOp.DS_LOAD_2ADDR_STRIDE64_B64, addr=v[10], vdst=v[4:7], offset0=1, offset1=2),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][4], 0xDEADBEEF)
|
||||
self.assertEqual(st.vgpr[0][5], 0xCAFEBABE)
|
||||
self.assertEqual(st.vgpr[0][6], 0x12345678)
|
||||
self.assertEqual(st.vgpr[0][7], 0x9ABCDEF0)
|
||||
|
||||
def test_ds_storexchg_2addr_rtn_b32(self):
|
||||
"""DS_STOREXCHG_2ADDR_RTN_B32: exchange at two addresses."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[0], 0x11111111),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
s_mov_b32(s[0], 0x22222222),
|
||||
v_mov_b32_e32(v[1], s[0]),
|
||||
DS(DSOp.DS_STORE_2ADDR_B32, addr=v[10], data0=v[0], data1=v[1], vdst=v[0], offset0=0, offset1=1),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[0], 0xAAAAAAAA),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
s_mov_b32(s[0], 0xBBBBBBBB),
|
||||
v_mov_b32_e32(v[3], s[0]),
|
||||
DS(DSOp.DS_STOREXCHG_2ADDR_RTN_B32, addr=v[10], data0=v[2], data1=v[3], vdst=v[4:5], offset0=0, offset1=1),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
DS(DSOp.DS_LOAD_2ADDR_B32, addr=v[10], vdst=v[6:7], offset0=0, offset1=1),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][4], 0x11111111, "old val 0")
|
||||
self.assertEqual(st.vgpr[0][5], 0x22222222, "old val 1")
|
||||
self.assertEqual(st.vgpr[0][6], 0xAAAAAAAA, "new val 0")
|
||||
self.assertEqual(st.vgpr[0][7], 0xBBBBBBBB, "new val 1")
|
||||
|
||||
def test_ds_storexchg_rtn_b64(self):
|
||||
"""DS_STOREXCHG_RTN_B64: exchange 64-bit value and return old."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[0], 0xDEADBEEF),
|
||||
v_mov_b32_e32(v[0], s[0]), # initial low
|
||||
s_mov_b32(s[0], 0xCAFEBABE),
|
||||
v_mov_b32_e32(v[1], s[0]), # initial high
|
||||
DS(DSOp.DS_STORE_B64, addr=v[10], data0=v[0:1], vdst=v[0], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[0], 0x12345678),
|
||||
v_mov_b32_e32(v[2], s[0]), # new low
|
||||
s_mov_b32(s[0], 0x9ABCDEF0),
|
||||
v_mov_b32_e32(v[3], s[0]), # new high
|
||||
DS(DSOp.DS_STOREXCHG_RTN_B64, addr=v[10], data0=v[2:3], vdst=v[4:5], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
DS(DSOp.DS_LOAD_B64, addr=v[10], vdst=v[6:7], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][4], 0xDEADBEEF, "v4 should have old low dword")
|
||||
self.assertEqual(st.vgpr[0][5], 0xCAFEBABE, "v5 should have old high dword")
|
||||
self.assertEqual(st.vgpr[0][6], 0x12345678, "v6 should have new low dword")
|
||||
self.assertEqual(st.vgpr[0][7], 0x9ABCDEF0, "v7 should have new high dword")
|
||||
|
||||
def test_ds_store_load_2addr_stride64_b64_roundtrip(self):
|
||||
"""DS_STORE_2ADDR_STRIDE64_B64 followed by DS_LOAD_2ADDR_STRIDE64_B64 works correctly."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[0], 0x11111111),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
s_mov_b32(s[0], 0x22222222),
|
||||
v_mov_b32_e32(v[1], s[0]),
|
||||
DS(DSOp.DS_STORE_2ADDR_STRIDE64_B64, addr=v[10], data0=v[0:1], data1=v[0:1], vdst=v[0], offset0=1, offset1=2),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
DS(DSOp.DS_LOAD_2ADDR_STRIDE64_B64, addr=v[10], vdst=v[2:5], offset0=1, offset1=2),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 0x11111111, "v2 should have val1 low")
|
||||
self.assertEqual(st.vgpr[0][3], 0x22222222, "v3 should have val1 high")
|
||||
self.assertEqual(st.vgpr[0][4], 0x11111111, "v4 should have val2 low")
|
||||
self.assertEqual(st.vgpr[0][5], 0x22222222, "v5 should have val2 high")
|
||||
|
||||
def test_ds_storexchg_2addr_stride64_rtn_b32(self):
|
||||
"""DS_STOREXCHG_2ADDR_STRIDE64_RTN_B32: exchange at two addresses (offset*256)."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[0], 0x11111111),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
s_mov_b32(s[0], 0x22222222),
|
||||
v_mov_b32_e32(v[1], s[0]),
|
||||
DS(DSOp.DS_STORE_2ADDR_STRIDE64_B32, addr=v[10], data0=v[0], data1=v[1], vdst=v[0], offset0=1, offset1=2),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[0], 0xAAAAAAAA),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
s_mov_b32(s[0], 0xBBBBBBBB),
|
||||
v_mov_b32_e32(v[3], s[0]),
|
||||
DS(DSOp.DS_STOREXCHG_2ADDR_STRIDE64_RTN_B32, addr=v[10], data0=v[2], data1=v[3], vdst=v[4:5], offset0=1, offset1=2),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
DS(DSOp.DS_LOAD_2ADDR_STRIDE64_B32, addr=v[10], vdst=v[6:7], offset0=1, offset1=2),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][4], 0x11111111, "v4 should have old value")
|
||||
self.assertEqual(st.vgpr[0][5], 0x22222222, "v5 should have old value")
|
||||
self.assertEqual(st.vgpr[0][6], 0xAAAAAAAA, "v6 should have new value")
|
||||
self.assertEqual(st.vgpr[0][7], 0xBBBBBBBB, "v7 should have new value")
|
||||
|
||||
def test_ds_storexchg_2addr_stride64_rtn_b64_returns_old(self):
|
||||
"""DS_STOREXCHG_2ADDR_STRIDE64_RTN_B64: returns old values correctly."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[0], 0x11111111),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
s_mov_b32(s[0], 0x22222222),
|
||||
v_mov_b32_e32(v[1], s[0]),
|
||||
DS(DSOp.DS_STORE_2ADDR_STRIDE64_B64, addr=v[10], data0=v[0:1], data1=v[0:1], vdst=v[0], offset0=1, offset1=2),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[0], 0xAAAAAAAA),
|
||||
v_mov_b32_e32(v[6], s[0]),
|
||||
s_mov_b32(s[0], 0xBBBBBBBB),
|
||||
v_mov_b32_e32(v[7], s[0]),
|
||||
DS(DSOp.DS_STOREXCHG_2ADDR_STRIDE64_RTN_B64, addr=v[10], data0=v[6:7], data1=v[6:7], vdst=v[8:11], offset0=1, offset1=2),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][8], 0x11111111, "v8 should have old val1 low")
|
||||
self.assertEqual(st.vgpr[0][9], 0x22222222, "v9 should have old val1 high")
|
||||
self.assertEqual(st.vgpr[0][10], 0x11111111, "v10 should have old val2 low")
|
||||
self.assertEqual(st.vgpr[0][11], 0x22222222, "v11 should have old val2 high")
|
||||
|
||||
|
||||
class TestAtomicOrdering(unittest.TestCase):
|
||||
"""Tests for atomic operation return values and ordering."""
|
||||
|
||||
def test_ds_add_rtn_sequence(self):
|
||||
"""DS_ADD_RTN returns correct old values in sequence."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
v_mov_b32_e32(v[0], 100),
|
||||
DS(DSOp.DS_STORE_B32, addr=v[10], data0=v[0], vdst=v[0], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[1], 25),
|
||||
DS(DSOp.DS_ADD_RTN_U32, addr=v[10], data0=v[1], vdst=v[2], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
DS(DSOp.DS_ADD_RTN_U32, addr=v[10], data0=v[1], vdst=v[3], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
DS(DSOp.DS_LOAD_B32, addr=v[10], vdst=v[4], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 100, "First add should return 100")
|
||||
self.assertEqual(st.vgpr[0][3], 125, "Second add should return 125")
|
||||
self.assertEqual(st.vgpr[0][4], 150, "Final value should be 150")
|
||||
|
||||
|
||||
class TestDsPermute(unittest.TestCase):
|
||||
"""Tests for DS_PERMUTE_B32 and DS_BPERMUTE_B32 instructions."""
|
||||
|
||||
def test_ds_permute_b32_identity(self):
|
||||
"""DS_PERMUTE_B32 with identity permutation (lane 0 sends to lane 0)."""
|
||||
# For simplicity, test with single lane
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], 0), # addr = 0 (lane 0)
|
||||
v_mov_b32_e32(v[1], 0xDEADBEEF), # data
|
||||
ds_permute_b32(v[2], v[0], v[1]),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
# Lane 0 sends to lane 0, so lane 0 gets 0xDEADBEEF
|
||||
self.assertEqual(st.vgpr[0][2], 0xDEADBEEF)
|
||||
|
||||
def test_ds_bpermute_b32_identity(self):
|
||||
"""DS_BPERMUTE_B32 with identity permutation (each lane reads from itself)."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], 0), # addr = 0 (read from lane 0)
|
||||
v_mov_b32_e32(v[1], 0xCAFEBABE), # data in lane 0
|
||||
ds_bpermute_b32(v[2], v[0], v[1]),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
# Lane 0 reads from lane 0's v[1]
|
||||
self.assertEqual(st.vgpr[0][2], 0xCAFEBABE)
|
||||
|
||||
def test_ds_permute_b32_broadcast(self):
|
||||
"""DS_PERMUTE_B32 broadcast - all lanes send to lane 0."""
|
||||
# With 4 lanes, all sending to lane 0, highest lane wins
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], 0), # All lanes send to addr 0 (lane 0)
|
||||
v_mov_b32_e32(v[1], 0x11111111), # All lanes send same data
|
||||
ds_permute_b32(v[2], v[0], v[1]),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=4)
|
||||
# Lane 0 receives data (highest numbered active lane wins)
|
||||
self.assertEqual(st.vgpr[0][2], 0x11111111)
|
||||
|
||||
def test_ds_bpermute_b32_xor_swap(self):
|
||||
"""DS_BPERMUTE_B32 with XOR-1 pattern — each lane reads from lane^1.
|
||||
|
||||
This is the pattern used by warp_shfl_xor in flash attention for reduce_max/reduce_sum.
|
||||
Each lane has a unique value (lane_id + 100), and reads from the adjacent lane.
|
||||
"""
|
||||
instructions = [
|
||||
# v[0] = (lane_id ^ 1) * 4 (byte offset for bpermute)
|
||||
v_xor_b32_e32(v[0], 1, v[255]),
|
||||
v_lshlrev_b32_e32(v[0], 2, v[0]),
|
||||
# v[1] = lane_id + 100 (unique per-lane value)
|
||||
s_mov_b32(s[0], 100),
|
||||
v_add_nc_u32_e32(v[1], s[0], v[255]),
|
||||
# ds_bpermute: v[2] = v[1] from lane (lane_id ^ 1)
|
||||
ds_bpermute_b32(vdst=v[2], addr=v[0], data0=v[1]),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=32)
|
||||
for lane in range(32):
|
||||
src_lane = lane ^ 1
|
||||
expected = src_lane + 100
|
||||
self.assertEqual(st.vgpr[lane][2], expected, f"lane {lane}: expected v[1] from lane {src_lane} = {expected}, got {st.vgpr[lane][2]}")
|
||||
class TestDSSubDword(unittest.TestCase):
|
||||
"""Tests for sub-dword DS operations (ds_store_b16, ds_store_b16_d16_hi)."""
|
||||
|
||||
def test_ds_store_b16_and_d16_hi(self):
|
||||
"""DS_STORE_B16 stores low 16 bits, DS_STORE_B16_D16_HI stores high 16 bits to adjacent LDS half-words."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
v_mov_b32_e32(v[1], 0xBEEF1234),
|
||||
DS(DSOp.DS_STORE_B16, addr=v[0], data0=v[1], offset0=0),
|
||||
DS(DSOp.DS_STORE_B16_D16_HI, addr=v[0], data0=v[1], offset0=2),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
ds_load_b32(vdst=v[2], addr=v[0], offset0=0),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 0xBEEF1234, "lo=0x1234 at byte 0, hi=0xBEEF at byte 2")
|
||||
|
||||
|
||||
class TestDSLargeOffset(unittest.TestCase):
|
||||
"""Tests for DS instructions with offsets > 255 (offset1 > 0).
|
||||
|
||||
The DS offset is a 16-bit value encoded as (offset1 << 8) | offset0.
|
||||
These tests verify that offset1 is used correctly, not just offset0.
|
||||
"""
|
||||
|
||||
def test_ds_store_load_b32_offset_256(self):
|
||||
"""DS_STORE_B32/DS_LOAD_B32 with offset=256 (offset0=0, offset1=1)."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[0], 0xDEADBEEF),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
ds_store_b32(addr=v[10], data0=v[0], offset0=0, offset1=1), # offset = 256
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
ds_load_b32(addr=v[10], vdst=v[1], offset0=0, offset1=1), # offset = 256
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][1], 0xDEADBEEF)
|
||||
|
||||
def test_ds_store_load_b32_offset_300(self):
|
||||
"""DS_STORE_B32/DS_LOAD_B32 with offset=300 (offset0=44, offset1=1)."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[0], 0xCAFEBABE),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
ds_store_b32(addr=v[10], data0=v[0], offset0=44, offset1=1), # offset = 300
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
ds_load_b32(addr=v[10], vdst=v[1], offset0=44, offset1=1), # offset = 300
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][1], 0xCAFEBABE)
|
||||
|
||||
def test_ds_store_load_b64_offset_512(self):
|
||||
"""DS_STORE_B64/DS_LOAD_B64 with offset=512 (offset0=0, offset1=2)."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[0], 0x11111111),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
s_mov_b32(s[0], 0x22222222),
|
||||
v_mov_b32_e32(v[1], s[0]),
|
||||
ds_store_b64(addr=v[10], data0=v[0:1], offset0=0, offset1=2), # offset = 512
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
ds_load_b64(addr=v[10], vdst=v[2:3], offset0=0, offset1=2), # offset = 512
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 0x11111111)
|
||||
self.assertEqual(st.vgpr[0][3], 0x22222222)
|
||||
|
||||
def test_ds_large_offset_distinct_from_small(self):
|
||||
"""Verify offset=256 and offset=0 address different LDS locations."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[0], 0xAAAAAAAA),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
s_mov_b32(s[0], 0xBBBBBBBB),
|
||||
v_mov_b32_e32(v[1], s[0]),
|
||||
# Store 0xAAAAAAAA at offset=0, 0xBBBBBBBB at offset=256
|
||||
ds_store_b32(addr=v[10], data0=v[0], offset0=0, offset1=0), # offset = 0
|
||||
ds_store_b32(addr=v[10], data0=v[1], offset0=0, offset1=1), # offset = 256
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
# Read back both
|
||||
ds_load_b32(addr=v[10], vdst=v[2], offset0=0, offset1=0), # offset = 0
|
||||
ds_load_b32(addr=v[10], vdst=v[3], offset0=0, offset1=1), # offset = 256
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 0xAAAAAAAA, "offset=0 should read 0xAAAAAAAA")
|
||||
self.assertEqual(st.vgpr[0][3], 0xBBBBBBBB, "offset=256 should read 0xBBBBBBBB")
|
||||
|
||||
def test_ds_store_load_b32_offset_448(self):
|
||||
"""DS_STORE_B32/DS_LOAD_B32 with offset=448 (offset0=192, offset1=1) - matches matmul B tile."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[0], 0x12345678),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
ds_store_b32(addr=v[10], data0=v[0], offset0=192, offset1=1), # offset = 448
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
ds_load_b32(addr=v[10], vdst=v[1], offset0=192, offset1=1), # offset = 448
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][1], 0x12345678)
|
||||
|
||||
def test_ds_load_b64_offset_392(self):
|
||||
"""DS_LOAD_B64 with offset=392 (offset0=136, offset1=1) - matches matmul B tile load."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], 0),
|
||||
s_mov_b32(s[0], 0xAABBCCDD),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
s_mov_b32(s[0], 0x11223344),
|
||||
v_mov_b32_e32(v[1], s[0]),
|
||||
ds_store_b64(addr=v[10], data0=v[0:1], offset0=136, offset1=1), # offset = 392
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
ds_load_b64(addr=v[10], vdst=v[2:3], offset0=136, offset1=1), # offset = 392
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 0xAABBCCDD)
|
||||
self.assertEqual(st.vgpr[0][3], 0x11223344)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
363
tinygrad_repo/test/amd/hw/test_flat.py
Normal file
363
tinygrad_repo/test/amd/hw/test_flat.py
Normal file
@@ -0,0 +1,363 @@
|
||||
"""Tests for FLAT instructions - flat memory operations.
|
||||
|
||||
Includes: flat_load_*, flat_store_*, flat_atomic_*
|
||||
"""
|
||||
import unittest
|
||||
from test.amd.hw.helpers import *
|
||||
|
||||
class TestFlatAtomic(unittest.TestCase):
|
||||
"""Tests for FLAT atomic instructions."""
|
||||
|
||||
def _make_test(self, setup_instrs, atomic_instr, check_fn, test_offset=2000):
|
||||
"""Helper to create atomic test instructions."""
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
v_mov_b32_e32(v[1], s[3]),
|
||||
] + setup_instrs + [atomic_instr, s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
v_mov_b32_e32(v[1], 0),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
check_fn(st)
|
||||
|
||||
def test_flat_atomic_add_u32(self):
|
||||
"""FLAT_ATOMIC_ADD_U32 adds to memory and returns old value."""
|
||||
TEST_OFFSET = 2000
|
||||
setup = [
|
||||
s_mov_b32(s[0], 100),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
global_store_b32(addr=v[0:1], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[0], 50),
|
||||
v_mov_b32_e32(v[3], s[0]),
|
||||
]
|
||||
atomic = FLAT(FLATOp.FLAT_ATOMIC_ADD_U32, addr=v[0:1], data=v[3], vdst=v[4], saddr=SrcEnum.NULL, offset=TEST_OFFSET, glc=1)
|
||||
def check(st):
|
||||
self.assertEqual(st.vgpr[0][4], 100)
|
||||
self._make_test(setup, atomic, check, TEST_OFFSET)
|
||||
|
||||
def test_flat_atomic_swap_b32(self):
|
||||
"""FLAT_ATOMIC_SWAP_B32 swaps memory value and returns old value."""
|
||||
TEST_OFFSET = 2000
|
||||
setup = [
|
||||
s_mov_b32(s[0], 0xAAAAAAAA),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
global_store_b32(addr=v[0:1], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[0], 0xBBBBBBBB),
|
||||
v_mov_b32_e32(v[3], s[0]),
|
||||
]
|
||||
atomic = FLAT(FLATOp.FLAT_ATOMIC_SWAP_B32, addr=v[0:1], data=v[3], vdst=v[4], saddr=SrcEnum.NULL, offset=TEST_OFFSET, glc=1)
|
||||
def check(st):
|
||||
self.assertEqual(st.vgpr[0][4], 0xAAAAAAAA)
|
||||
self._make_test(setup, atomic, check, TEST_OFFSET)
|
||||
|
||||
def test_flat_atomic_and_b32(self):
|
||||
"""FLAT_ATOMIC_AND_B32 ANDs with memory and returns old value."""
|
||||
TEST_OFFSET = 2000
|
||||
setup = [
|
||||
s_mov_b32(s[0], 0xFF00FF00),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
global_store_b32(addr=v[0:1], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[0], 0xFFFF0000),
|
||||
v_mov_b32_e32(v[3], s[0]),
|
||||
]
|
||||
atomic = FLAT(FLATOp.FLAT_ATOMIC_AND_B32, addr=v[0:1], data=v[3], vdst=v[4], saddr=SrcEnum.NULL, offset=TEST_OFFSET, glc=1)
|
||||
def check(st):
|
||||
self.assertEqual(st.vgpr[0][4], 0xFF00FF00)
|
||||
self._make_test(setup, atomic, check, TEST_OFFSET)
|
||||
|
||||
def test_flat_atomic_or_b32(self):
|
||||
"""FLAT_ATOMIC_OR_B32 ORs with memory and returns old value."""
|
||||
TEST_OFFSET = 2000
|
||||
setup = [
|
||||
s_mov_b32(s[0], 0x00FF0000),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
global_store_b32(addr=v[0:1], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[0], 0x0000FF00),
|
||||
v_mov_b32_e32(v[3], s[0]),
|
||||
]
|
||||
atomic = FLAT(FLATOp.FLAT_ATOMIC_OR_B32, addr=v[0:1], data=v[3], vdst=v[4], saddr=SrcEnum.NULL, offset=TEST_OFFSET, glc=1)
|
||||
def check(st):
|
||||
self.assertEqual(st.vgpr[0][4], 0x00FF0000)
|
||||
self._make_test(setup, atomic, check, TEST_OFFSET)
|
||||
|
||||
def test_flat_atomic_inc_u32(self):
|
||||
"""FLAT_ATOMIC_INC_U32 increments and returns old value."""
|
||||
TEST_OFFSET = 2000
|
||||
setup = [
|
||||
s_mov_b32(s[0], 10),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
global_store_b32(addr=v[0:1], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[0], 100), # threshold
|
||||
v_mov_b32_e32(v[3], s[0]),
|
||||
]
|
||||
atomic = FLAT(FLATOp.FLAT_ATOMIC_INC_U32, addr=v[0:1], data=v[3], vdst=v[4], saddr=SrcEnum.NULL, offset=TEST_OFFSET, glc=1)
|
||||
def check(st):
|
||||
self.assertEqual(st.vgpr[0][4], 10)
|
||||
self._make_test(setup, atomic, check, TEST_OFFSET)
|
||||
|
||||
def test_flat_atomic_dec_u32(self):
|
||||
"""FLAT_ATOMIC_DEC_U32 decrements and returns old value."""
|
||||
TEST_OFFSET = 2000
|
||||
setup = [
|
||||
s_mov_b32(s[0], 10),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
global_store_b32(addr=v[0:1], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[0], 100),
|
||||
v_mov_b32_e32(v[3], s[0]),
|
||||
]
|
||||
atomic = FLAT(FLATOp.FLAT_ATOMIC_DEC_U32, addr=v[0:1], data=v[3], vdst=v[4], saddr=SrcEnum.NULL, offset=TEST_OFFSET, glc=1)
|
||||
def check(st):
|
||||
self.assertEqual(st.vgpr[0][4], 10)
|
||||
self._make_test(setup, atomic, check, TEST_OFFSET)
|
||||
|
||||
def test_flat_atomic_sub_u32(self):
|
||||
"""FLAT_ATOMIC_SUB_U32 subtracts from memory and returns old value."""
|
||||
TEST_OFFSET = 2000
|
||||
setup = [
|
||||
s_mov_b32(s[0], 100),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
global_store_b32(addr=v[0:1], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[0], 30),
|
||||
v_mov_b32_e32(v[3], s[0]), # sub 30
|
||||
]
|
||||
atomic = FLAT(FLATOp.FLAT_ATOMIC_SUB_U32, addr=v[0:1], data=v[3], vdst=v[4], saddr=SrcEnum.NULL, offset=TEST_OFFSET, glc=1)
|
||||
def check(st):
|
||||
self.assertEqual(st.vgpr[0][4], 100, "v4 should have old value (100)")
|
||||
self._make_test(setup, atomic, check, TEST_OFFSET)
|
||||
|
||||
def test_flat_atomic_xor_b32(self):
|
||||
"""FLAT_ATOMIC_XOR_B32 XORs with memory and returns old value."""
|
||||
TEST_OFFSET = 2000
|
||||
setup = [
|
||||
s_mov_b32(s[0], 0xAAAAAAAA),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
global_store_b32(addr=v[0:1], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[0], 0xFFFFFFFF),
|
||||
v_mov_b32_e32(v[3], s[0]), # XOR mask
|
||||
]
|
||||
atomic = FLAT(FLATOp.FLAT_ATOMIC_XOR_B32, addr=v[0:1], data=v[3], vdst=v[4], saddr=SrcEnum.NULL, offset=TEST_OFFSET, glc=1)
|
||||
def check(st):
|
||||
self.assertEqual(st.vgpr[0][4], 0xAAAAAAAA, "v4 should have old value")
|
||||
self._make_test(setup, atomic, check, TEST_OFFSET)
|
||||
|
||||
def test_flat_atomic_min_u32(self):
|
||||
"""FLAT_ATOMIC_MIN_U32 stores min and returns old value."""
|
||||
TEST_OFFSET = 2000
|
||||
setup = [
|
||||
s_mov_b32(s[0], 100),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
global_store_b32(addr=v[0:1], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[0], 50),
|
||||
v_mov_b32_e32(v[3], s[0]), # compare value (smaller)
|
||||
]
|
||||
atomic = FLAT(FLATOp.FLAT_ATOMIC_MIN_U32, addr=v[0:1], data=v[3], vdst=v[4], saddr=SrcEnum.NULL, offset=TEST_OFFSET, glc=1)
|
||||
def check(st):
|
||||
self.assertEqual(st.vgpr[0][4], 100, "v4 should have old value (100)")
|
||||
self._make_test(setup, atomic, check, TEST_OFFSET)
|
||||
|
||||
def test_flat_atomic_max_u32(self):
|
||||
"""FLAT_ATOMIC_MAX_U32 stores max and returns old value."""
|
||||
TEST_OFFSET = 2000
|
||||
setup = [
|
||||
s_mov_b32(s[0], 50),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
global_store_b32(addr=v[0:1], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[0], 100),
|
||||
v_mov_b32_e32(v[3], s[0]), # compare value (larger)
|
||||
]
|
||||
atomic = FLAT(FLATOp.FLAT_ATOMIC_MAX_U32, addr=v[0:1], data=v[3], vdst=v[4], saddr=SrcEnum.NULL, offset=TEST_OFFSET, glc=1)
|
||||
def check(st):
|
||||
self.assertEqual(st.vgpr[0][4], 50, "v4 should have old value (50)")
|
||||
self._make_test(setup, atomic, check, TEST_OFFSET)
|
||||
|
||||
def test_flat_atomic_inc_u64_returns_old_value(self):
|
||||
"""FLAT_ATOMIC_INC_U64 should return full 64-bit old value."""
|
||||
TEST_OFFSET = 2000
|
||||
setup = [
|
||||
# Store initial 64-bit value: 0xCAFEBABE_DEADBEEF
|
||||
s_mov_b32(s[0], 0xDEADBEEF),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
s_mov_b32(s[0], 0xCAFEBABE),
|
||||
v_mov_b32_e32(v[3], s[0]),
|
||||
global_store_b64(addr=v[0:1], data=v[2:3], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
# Threshold: 0xFFFFFFFF_FFFFFFFF
|
||||
s_mov_b32(s[0], 0xFFFFFFFF),
|
||||
v_mov_b32_e32(v[4], s[0]),
|
||||
v_mov_b32_e32(v[5], s[0]),
|
||||
]
|
||||
atomic = FLAT(FLATOp.FLAT_ATOMIC_INC_U64, addr=v[0:1], data=v[4:5], vdst=v[6:7], saddr=SrcEnum.NULL, offset=TEST_OFFSET, glc=1)
|
||||
def check(st):
|
||||
self.assertEqual(st.vgpr[0][6], 0xDEADBEEF, "v6 should have old value low dword")
|
||||
self.assertEqual(st.vgpr[0][7], 0xCAFEBABE, "v7 should have old value high dword")
|
||||
self._make_test(setup, atomic, check, TEST_OFFSET)
|
||||
|
||||
def test_flat_atomic_add_u64(self):
|
||||
"""FLAT_ATOMIC_ADD_U64 adds 64-bit value and returns old value."""
|
||||
TEST_OFFSET = 2000
|
||||
setup = [
|
||||
s_mov_b32(s[0], 0x11111111),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
s_mov_b32(s[0], 0x22222222),
|
||||
v_mov_b32_e32(v[3], s[0]),
|
||||
global_store_b64(addr=v[0:1], data=v[2:3], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[0], 0x00000001), # add 1
|
||||
v_mov_b32_e32(v[4], s[0]),
|
||||
s_mov_b32(s[0], 0x00000000),
|
||||
v_mov_b32_e32(v[5], s[0]),
|
||||
]
|
||||
atomic = FLAT(FLATOp.FLAT_ATOMIC_ADD_U64, addr=v[0:1], data=v[4:5], vdst=v[6:7], saddr=SrcEnum.NULL, offset=TEST_OFFSET, glc=1)
|
||||
def check(st):
|
||||
self.assertEqual(st.vgpr[0][6], 0x11111111, "v6 should have old value low")
|
||||
self.assertEqual(st.vgpr[0][7], 0x22222222, "v7 should have old value high")
|
||||
self._make_test(setup, atomic, check, TEST_OFFSET)
|
||||
|
||||
def test_flat_atomic_swap_b64(self):
|
||||
"""FLAT_ATOMIC_SWAP_B64 swaps 64-bit value and returns old value."""
|
||||
TEST_OFFSET = 2000
|
||||
setup = [
|
||||
s_mov_b32(s[0], 0xAAAAAAAA),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
s_mov_b32(s[0], 0xBBBBBBBB),
|
||||
v_mov_b32_e32(v[3], s[0]),
|
||||
global_store_b64(addr=v[0:1], data=v[2:3], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[0], 0xCCCCCCCC),
|
||||
v_mov_b32_e32(v[4], s[0]),
|
||||
s_mov_b32(s[0], 0xDDDDDDDD),
|
||||
v_mov_b32_e32(v[5], s[0]),
|
||||
]
|
||||
atomic = FLAT(FLATOp.FLAT_ATOMIC_SWAP_B64, addr=v[0:1], data=v[4:5], vdst=v[6:7], saddr=SrcEnum.NULL, offset=TEST_OFFSET, glc=1)
|
||||
def check(st):
|
||||
self.assertEqual(st.vgpr[0][6], 0xAAAAAAAA, "v6 should have old value low")
|
||||
self.assertEqual(st.vgpr[0][7], 0xBBBBBBBB, "v7 should have old value high")
|
||||
self._make_test(setup, atomic, check, TEST_OFFSET)
|
||||
|
||||
|
||||
class TestFlatLoad(unittest.TestCase):
|
||||
"""Tests for FLAT load instructions."""
|
||||
|
||||
def test_flat_load_b32(self):
|
||||
"""FLAT_LOAD_B32 loads 32-bit value correctly."""
|
||||
TEST_OFFSET = 2000
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
v_mov_b32_e32(v[1], s[3]),
|
||||
s_mov_b32(s[0], 0xDEADBEEF),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
global_store_b32(addr=v[0:1], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
FLAT(FLATOp.FLAT_LOAD_B32, addr=v[0:1], vdst=v[4], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
v_mov_b32_e32(v[1], 0),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][4], 0xDEADBEEF)
|
||||
|
||||
def test_flat_load_b64(self):
|
||||
"""FLAT_LOAD_B64 loads 64-bit value correctly."""
|
||||
TEST_OFFSET = 2000
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
v_mov_b32_e32(v[1], s[3]),
|
||||
s_mov_b32(s[0], 0xDEADBEEF),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
s_mov_b32(s[0], 0xCAFEBABE),
|
||||
v_mov_b32_e32(v[3], s[0]),
|
||||
global_store_b64(addr=v[0:1], data=v[2:3], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
FLAT(FLATOp.FLAT_LOAD_B64, addr=v[0:1], vdst=v[4:5], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
v_mov_b32_e32(v[1], 0),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][4], 0xDEADBEEF)
|
||||
self.assertEqual(st.vgpr[0][5], 0xCAFEBABE)
|
||||
|
||||
def test_flat_load_b96(self):
|
||||
"""FLAT_LOAD_B96 loads 96-bit (3 dword) value correctly."""
|
||||
TEST_OFFSET = 2000
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
v_mov_b32_e32(v[1], s[3]),
|
||||
s_mov_b32(s[0], 0x11111111),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
s_mov_b32(s[0], 0x22222222),
|
||||
v_mov_b32_e32(v[3], s[0]),
|
||||
s_mov_b32(s[0], 0x33333333),
|
||||
v_mov_b32_e32(v[4], s[0]),
|
||||
global_store_b96(addr=v[0:1], data=v[2:4], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
FLAT(FLATOp.FLAT_LOAD_B96, addr=v[0:1], vdst=v[5:7], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
v_mov_b32_e32(v[1], 0),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][5], 0x11111111)
|
||||
self.assertEqual(st.vgpr[0][6], 0x22222222)
|
||||
self.assertEqual(st.vgpr[0][7], 0x33333333)
|
||||
|
||||
def test_flat_load_b128(self):
|
||||
"""FLAT_LOAD_B128 loads 128-bit value correctly."""
|
||||
TEST_OFFSET = 2000
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
v_mov_b32_e32(v[1], s[3]),
|
||||
s_mov_b32(s[0], 0x11111111),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
s_mov_b32(s[0], 0x22222222),
|
||||
v_mov_b32_e32(v[3], s[0]),
|
||||
s_mov_b32(s[0], 0x33333333),
|
||||
v_mov_b32_e32(v[4], s[0]),
|
||||
s_mov_b32(s[0], 0x44444444),
|
||||
v_mov_b32_e32(v[5], s[0]),
|
||||
global_store_b128(addr=v[0:1], data=v[2:5], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
FLAT(FLATOp.FLAT_LOAD_B128, addr=v[0:1], vdst=v[6:9], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
v_mov_b32_e32(v[1], 0),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][6], 0x11111111)
|
||||
self.assertEqual(st.vgpr[0][7], 0x22222222)
|
||||
self.assertEqual(st.vgpr[0][8], 0x33333333)
|
||||
self.assertEqual(st.vgpr[0][9], 0x44444444)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
679
tinygrad_repo/test/amd/hw/test_global.py
Normal file
679
tinygrad_repo/test/amd/hw/test_global.py
Normal file
@@ -0,0 +1,679 @@
|
||||
"""Tests for GLOBAL instructions - global memory operations.
|
||||
|
||||
Includes: global_load_*, global_store_*, global_atomic_*, global_load_d16_*
|
||||
"""
|
||||
import unittest
|
||||
from test.amd.hw.helpers import *
|
||||
|
||||
class TestGlobalAtomic(unittest.TestCase):
|
||||
"""Tests for GLOBAL atomic instructions."""
|
||||
|
||||
def _make_test(self, setup_instrs, atomic_instr, check_fn, test_offset=2000):
|
||||
"""Helper to create atomic test instructions."""
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
v_mov_b32_e32(v[1], s[3]),
|
||||
] + setup_instrs + [atomic_instr, s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
v_mov_b32_e32(v[1], 0),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
check_fn(st)
|
||||
|
||||
def test_global_atomic_add_u32(self):
|
||||
"""GLOBAL_ATOMIC_ADD_U32 adds to memory and returns old value."""
|
||||
TEST_OFFSET = 2000
|
||||
setup = [
|
||||
s_mov_b32(s[0], 100),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
global_store_b32(addr=v[0:1], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[0], 50),
|
||||
v_mov_b32_e32(v[3], s[0]),
|
||||
]
|
||||
atomic = GLOBAL(GLOBALOp.GLOBAL_ATOMIC_ADD_U32, addr=v[0:1], data=v[3], vdst=v[4], saddr=SrcEnum.NULL, offset=TEST_OFFSET, glc=1)
|
||||
def check(st):
|
||||
self.assertEqual(st.vgpr[0][4], 100)
|
||||
self._make_test(setup, atomic, check, TEST_OFFSET)
|
||||
|
||||
def test_global_atomic_add_u64(self):
|
||||
"""GLOBAL_ATOMIC_ADD_U64 adds 64-bit value and returns old value."""
|
||||
TEST_OFFSET = 2000
|
||||
setup = [
|
||||
s_mov_b32(s[0], 0xFFFFFFFF),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
s_mov_b32(s[0], 0x00000000),
|
||||
v_mov_b32_e32(v[3], s[0]),
|
||||
global_store_b64(addr=v[0:1], data=v[2:3], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[0], 0x00000001),
|
||||
v_mov_b32_e32(v[4], s[0]),
|
||||
s_mov_b32(s[0], 0x00000000),
|
||||
v_mov_b32_e32(v[5], s[0]),
|
||||
]
|
||||
atomic = GLOBAL(GLOBALOp.GLOBAL_ATOMIC_ADD_U64, addr=v[0:1], data=v[4:5], vdst=v[6:7], saddr=SrcEnum.NULL, offset=TEST_OFFSET, glc=1)
|
||||
def check(st):
|
||||
self.assertEqual(st.vgpr[0][6], 0xFFFFFFFF)
|
||||
self.assertEqual(st.vgpr[0][7], 0x00000000)
|
||||
self._make_test(setup, atomic, check, TEST_OFFSET)
|
||||
|
||||
|
||||
class TestGlobalLoad(unittest.TestCase):
|
||||
"""Tests for GLOBAL load instructions."""
|
||||
|
||||
def test_global_load_b96(self):
|
||||
"""GLOBAL_LOAD_B96 loads 96-bit value correctly."""
|
||||
TEST_OFFSET = 2000
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
v_mov_b32_e32(v[1], s[3]),
|
||||
s_mov_b32(s[0], 0xAAAAAAAA),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
s_mov_b32(s[0], 0xBBBBBBBB),
|
||||
v_mov_b32_e32(v[3], s[0]),
|
||||
s_mov_b32(s[0], 0xCCCCCCCC),
|
||||
v_mov_b32_e32(v[4], s[0]),
|
||||
global_store_b96(addr=v[0:1], data=v[2:4], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
GLOBAL(GLOBALOp.GLOBAL_LOAD_B96, addr=v[0:1], vdst=v[5:7], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
v_mov_b32_e32(v[1], 0),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][5], 0xAAAAAAAA)
|
||||
self.assertEqual(st.vgpr[0][6], 0xBBBBBBBB)
|
||||
self.assertEqual(st.vgpr[0][7], 0xCCCCCCCC)
|
||||
|
||||
def test_global_load_b128(self):
|
||||
"""GLOBAL_LOAD_B128 loads 128-bit value correctly."""
|
||||
TEST_OFFSET = 2000
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
v_mov_b32_e32(v[1], s[3]),
|
||||
s_mov_b32(s[0], 0xDEADBEEF),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
s_mov_b32(s[0], 0xCAFEBABE),
|
||||
v_mov_b32_e32(v[3], s[0]),
|
||||
s_mov_b32(s[0], 0x12345678),
|
||||
v_mov_b32_e32(v[4], s[0]),
|
||||
s_mov_b32(s[0], 0x9ABCDEF0),
|
||||
v_mov_b32_e32(v[5], s[0]),
|
||||
global_store_b128(addr=v[0:1], data=v[2:5], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
GLOBAL(GLOBALOp.GLOBAL_LOAD_B128, addr=v[0:1], vdst=v[6:9], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
v_mov_b32_e32(v[1], 0),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][6], 0xDEADBEEF)
|
||||
self.assertEqual(st.vgpr[0][7], 0xCAFEBABE)
|
||||
self.assertEqual(st.vgpr[0][8], 0x12345678)
|
||||
self.assertEqual(st.vgpr[0][9], 0x9ABCDEF0)
|
||||
|
||||
|
||||
class TestGlobalStore(unittest.TestCase):
|
||||
"""Tests for GLOBAL store instructions."""
|
||||
|
||||
def test_global_store_b8_basic(self):
|
||||
"""GLOBAL_STORE_B8 stores a single byte from VDATA[7:0]."""
|
||||
TEST_OFFSET = 256
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
# First store 0xDEADBEEF to memory
|
||||
s_mov_b32(s[4], 0xDEADBEEF),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
# Now store single byte 0x42 to same address (should only change byte 0)
|
||||
v_mov_b32_e32(v[2], 0x42),
|
||||
global_store_b8(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
# Read back and check
|
||||
GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[0], vdst=v[3], data=v[3], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[3]),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
# Only byte 0 should change from 0xEF to 0x42
|
||||
self.assertEqual(st.vgpr[0][0], 0xDEADBE42, "Only byte 0 should be modified")
|
||||
|
||||
def test_global_store_b8_byte1(self):
|
||||
"""GLOBAL_STORE_B8 at offset+1 stores to byte 1."""
|
||||
TEST_OFFSET = 256
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[4], 0xDEADBEEF),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[2], 0x42),
|
||||
global_store_b8(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+1),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[0], vdst=v[3], data=v[3], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[3]),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][0], 0xDEAD42EF, "Only byte 1 should be modified")
|
||||
|
||||
def test_global_store_b16_basic(self):
|
||||
"""GLOBAL_STORE_B16 stores a 16-bit value from VDATA[15:0]."""
|
||||
TEST_OFFSET = 256
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[4], 0xDEADBEEF),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[4], 0xCAFE),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
global_store_b16(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[0], vdst=v[3], data=v[3], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[3]),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][0], 0xDEADCAFE, "Only lower 16 bits should be modified")
|
||||
|
||||
def test_global_store_b16_high_half(self):
|
||||
"""GLOBAL_STORE_B16 at offset+2 stores to high 16 bits."""
|
||||
TEST_OFFSET = 256
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[4], 0xDEADBEEF),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[4], 0xCAFE),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
global_store_b16(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+2),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[0], vdst=v[3], data=v[3], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[3]),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][0], 0xCAFEBEEF, "Only upper 16 bits should be modified")
|
||||
|
||||
def test_global_store_b16_byte_offset_1(self):
|
||||
"""GLOBAL_STORE_B16 at byte offset 1 stores bytes 1-2 within the same word."""
|
||||
TEST_OFFSET = 256
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[4], 0xDDCCBBAA),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
# Store 0xBEEF at byte offset 1 (bytes 1-2)
|
||||
s_mov_b32(s[4], 0xBEEF),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
global_store_b16(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+1),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[0], vdst=v[3], data=v[3], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[3]),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
# Bytes 1-2 should be 0xBEEF (0xEF at byte 1, 0xBE at byte 2)
|
||||
# Original: 0xDDCCBBAA -> bytes [AA, BB, CC, DD]
|
||||
# After: 0xDDBEEFAA -> bytes [AA, EF, BE, DD]
|
||||
self.assertEqual(st.vgpr[0][0], 0xDDBEEFAA, "Bytes 1-2 should be 0xBEEF")
|
||||
|
||||
def test_global_store_b16_cross_word_boundary(self):
|
||||
"""GLOBAL_STORE_B16 at byte offset 3 crosses word boundary (byte 3 of word N, byte 0 of word N+1)."""
|
||||
TEST_OFFSET = 256
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
# Initialize two consecutive words
|
||||
s_mov_b32(s[4], 0xDDCCBBAA),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
s_mov_b32(s[4], 0x44332211),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+4),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
# Store 0xBEEF at byte offset 3 (crosses word boundary)
|
||||
# Low byte (0xEF) goes to byte 3 of first word
|
||||
# High byte (0xBE) goes to byte 0 of second word
|
||||
s_mov_b32(s[4], 0xBEEF),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
global_store_b16(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+3),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
# Load back both words
|
||||
GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[0], vdst=v[3], data=v[3], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[0], vdst=v[4], data=v[4], saddr=s[2:3], offset=TEST_OFFSET+4),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[3]),
|
||||
v_mov_b32_e32(v[1], v[4]),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
# First word: 0xDDCCBBAA -> 0xEFCCBBAA (byte 3 becomes 0xEF)
|
||||
# Second word: 0x44332211 -> 0x443322BE (byte 0 becomes 0xBE)
|
||||
self.assertEqual(st.vgpr[0][0], 0xEFCCBBAA, "Byte 3 of first word should be 0xEF")
|
||||
self.assertEqual(st.vgpr[0][1], 0x443322BE, "Byte 0 of second word should be 0xBE")
|
||||
|
||||
def test_global_store_b64_basic(self):
|
||||
"""GLOBAL_STORE_B64 stores 8 bytes from v[n:n+1] to memory."""
|
||||
TEST_OFFSET = 256
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[4], 0xDEADBEEF),
|
||||
s_mov_b32(s[5], 0xCAFEBABE),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
v_mov_b32_e32(v[3], s[5]),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
global_store_b64(addr=v[0], data=v[2:3], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
GLOBAL(GLOBALOp.GLOBAL_LOAD_B64, addr=v[0], vdst=v[4:5], data=v[4:5], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[4]),
|
||||
v_mov_b32_e32(v[1], v[5]),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][0], 0xDEADBEEF)
|
||||
self.assertEqual(st.vgpr[0][1], 0xCAFEBABE)
|
||||
|
||||
|
||||
class TestD16HiLoads(unittest.TestCase):
|
||||
"""Tests for D16_HI load instructions that load into high 16 bits."""
|
||||
|
||||
def test_global_load_d16_hi_b16_preserves_low_bits(self):
|
||||
"""GLOBAL_LOAD_D16_HI_B16 must preserve low 16 bits of destination."""
|
||||
TEST_OFFSET = 256
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
v_mov_b32_e32(v[1], s[3]),
|
||||
s_mov_b32(s[4], 0xCAFE),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
global_store_b16(addr=v[0:1], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[4], 0x0000BEEF),
|
||||
v_mov_b32_e32(v[3], s[4]),
|
||||
GLOBAL(GLOBALOp.GLOBAL_LOAD_D16_HI_B16, addr=v[0:1], vdst=v[3], data=v[3], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[3]),
|
||||
v_mov_b32_e32(v[1], 0),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
result = st.vgpr[0][0]
|
||||
self.assertEqual(result, 0xCAFEBEEF, f"Expected 0xCAFEBEEF, got 0x{result:08x}")
|
||||
|
||||
def test_global_load_d16_hi_b16_data_differs_from_vdst(self):
|
||||
"""GLOBAL_LOAD_D16_HI_B16 where data field differs from vdst."""
|
||||
TEST_OFFSET = 256
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[4], 0xCAFE),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
v_mov_b32_e32(v[3], 0),
|
||||
global_store_b16(addr=v[3], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[4], 0x0000DEAD),
|
||||
v_mov_b32_e32(v[0], s[4]), # data field - should NOT affect result
|
||||
v_mov_b32_e32(v[1], 0), # vdst - low bits should be preserved
|
||||
GLOBAL(GLOBALOp.GLOBAL_LOAD_D16_HI_B16, addr=v[1], vdst=v[1], data=v[0], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[1]),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
result = st.vgpr[0][0]
|
||||
self.assertEqual(result, 0xCAFE0000, f"Expected 0xCAFE0000, got 0x{result:08x}")
|
||||
|
||||
def test_global_load_d16_hi_u8_data_differs_from_vdst(self):
|
||||
"""GLOBAL_LOAD_D16_HI_U8 where data field differs from vdst."""
|
||||
TEST_OFFSET = 256
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[4], 0xAB),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
v_mov_b32_e32(v[3], 0),
|
||||
global_store_b8(addr=v[3], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[4], 0x0000DEAD),
|
||||
v_mov_b32_e32(v[4], s[4]), # data field
|
||||
s_mov_b32(s[4], 0x0000BEEF),
|
||||
v_mov_b32_e32(v[5], s[4]), # vdst
|
||||
v_mov_b32_e32(v[3], 0),
|
||||
GLOBAL(GLOBALOp.GLOBAL_LOAD_D16_HI_U8, addr=v[3], vdst=v[5], data=v[4], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[5]),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
result = st.vgpr[0][0]
|
||||
self.assertEqual(result, 0x00ABBEEF, f"Expected 0x00ABBEEF, got 0x{result:08x}")
|
||||
|
||||
def test_global_load_d16_hi_b16_same_addr_and_dst_zero_addr(self):
|
||||
"""GLOBAL_LOAD_D16_HI_B16 with same register for addr and vdst, addr value=0."""
|
||||
TEST_OFFSET = 256
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[4], 0xCAFE),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
v_mov_b32_e32(v[3], 0),
|
||||
global_store_b16(addr=v[3], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[1], 0),
|
||||
GLOBAL(GLOBALOp.GLOBAL_LOAD_D16_HI_B16, addr=v[1], vdst=v[1], data=v[1], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[1]),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
result = st.vgpr[0][0]
|
||||
self.assertEqual(result, 0xCAFE0000, f"Expected 0xCAFE0000, got 0x{result:08x}")
|
||||
|
||||
def test_global_load_d16_hi_b16_tril_exact_pattern(self):
|
||||
"""Exact pattern from tril() failure: data=v0 differs from vdst=v1."""
|
||||
TEST_OFFSET = 256
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[4], 0x01010101),
|
||||
v_mov_b32_e32(v[10], s[4]),
|
||||
v_mov_b32_e32(v[3], 0),
|
||||
global_store_b32(addr=v[3], data=v[10], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
global_store_b32(addr=v[3], data=v[10], saddr=s[2:3], offset=TEST_OFFSET+4),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
# Set v[0] to 0x0101 (simulating prior u16 load result)
|
||||
s_mov_b32(s[4], 0x0101),
|
||||
v_mov_b32_e32(v[0], s[4]),
|
||||
# Set v[1] to 0
|
||||
v_mov_b32_e32(v[1], 0),
|
||||
# Load using v[1] as addr AND vdst, but v[0] as data
|
||||
GLOBAL(GLOBALOp.GLOBAL_LOAD_D16_HI_B16, addr=v[1], vdst=v[1], data=v[0], saddr=s[2:3], offset=TEST_OFFSET+6),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[1]),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
result = st.vgpr[0][0]
|
||||
# Expected: hi=0x0101 (loaded), lo=0x0000 (from v1) -> 0x01010000
|
||||
self.assertEqual(result, 0x01010000, f"Expected 0x01010000, got 0x{result:08x}")
|
||||
|
||||
def test_global_load_d16_hi_i8_data_differs_from_vdst(self):
|
||||
"""GLOBAL_LOAD_D16_HI_I8 where data field differs from vdst."""
|
||||
TEST_OFFSET = 256
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[4], 0x80), # negative signed byte = -128
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
v_mov_b32_e32(v[3], 0),
|
||||
global_store_b8(addr=v[3], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[4], 0x0000DEAD),
|
||||
v_mov_b32_e32(v[4], s[4]), # data field
|
||||
s_mov_b32(s[4], 0x0000BEEF),
|
||||
v_mov_b32_e32(v[5], s[4]), # vdst
|
||||
v_mov_b32_e32(v[3], 0),
|
||||
GLOBAL(GLOBALOp.GLOBAL_LOAD_D16_HI_I8, addr=v[3], vdst=v[5], data=v[4], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[5]),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
result = st.vgpr[0][0]
|
||||
# 0x80 sign-extended = 0xFF80, lo=0xBEEF -> 0xFF80BEEF
|
||||
self.assertEqual(result, 0xFF80BEEF, f"Expected 0xFF80BEEF, got 0x{result:08x}")
|
||||
|
||||
def test_global_store_b64_tril_pattern(self):
|
||||
"""Test the exact pattern from tril() kernel that was failing."""
|
||||
TEST_OFFSET = 256
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[4], 0x01010101),
|
||||
v_mov_b32_e32(v[10], s[4]),
|
||||
v_mov_b32_e32(v[11], s[4]),
|
||||
s_mov_b32(s[4], 0x01),
|
||||
v_mov_b32_e32(v[12], s[4]),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
global_store_b64(addr=v[0], data=v[10:11], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
global_store_b8(addr=v[0], data=v[12], saddr=s[2:3], offset=TEST_OFFSET+8),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
|
||||
v_mov_b32_e32(v[2], 0),
|
||||
v_mov_b32_e32(v[1], 0),
|
||||
GLOBAL(GLOBALOp.GLOBAL_LOAD_U16, addr=v[2], vdst=v[0], data=v[0], saddr=s[2:3], offset=TEST_OFFSET+3),
|
||||
GLOBAL(GLOBALOp.GLOBAL_LOAD_D16_HI_B16, addr=v[1], vdst=v[1], data=v[1], saddr=s[2:3], offset=TEST_OFFSET+6),
|
||||
GLOBAL(GLOBALOp.GLOBAL_LOAD_U8, addr=v[2], vdst=v[3], data=v[3], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
GLOBAL(GLOBALOp.GLOBAL_LOAD_U8, addr=v[2], vdst=v[4], data=v[4], saddr=s[2:3], offset=TEST_OFFSET+8),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
|
||||
v_and_b32_e32(v[5], 0xffff, v[0]),
|
||||
v_lshlrev_b32_e32(v[0], 24, v[0]),
|
||||
v_lshrrev_b32_e32(v[5], 8, v[5]),
|
||||
v_or_b32_e32(v[0], v[3], v[0]),
|
||||
v_or_b32_e32(v[1], v[5], v[1]),
|
||||
|
||||
global_store_b64(addr=v[2], data=v[0:1], saddr=s[2:3], offset=TEST_OFFSET+16),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
|
||||
GLOBAL(GLOBALOp.GLOBAL_LOAD_B64, addr=v[2], vdst=v[6:7], data=v[6:7], saddr=s[2:3], offset=TEST_OFFSET+16),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[6]),
|
||||
v_mov_b32_e32(v[1], v[7]),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
|
||||
v0 = st.vgpr[0][0]
|
||||
v1 = st.vgpr[0][1]
|
||||
self.assertEqual(v0, 0x01000001, f"v0: expected 0x01000001, got 0x{v0:08x}")
|
||||
self.assertEqual(v1, 0x01010001, f"v1: expected 0x01010001, got 0x{v1:08x}")
|
||||
|
||||
byte5 = (v1 >> 8) & 0xff
|
||||
self.assertEqual(byte5, 0x00, f"byte5: expected 0x00, got 0x{byte5:02x}")
|
||||
|
||||
|
||||
class TestGlobalOffset(unittest.TestCase):
|
||||
"""Tests for GLOBAL instructions with different offsets.
|
||||
|
||||
These tests verify that instruction deduplication correctly handles different offset values.
|
||||
If offset is made dynamic incorrectly, instructions with different offsets may load/store wrong data.
|
||||
"""
|
||||
|
||||
def test_global_load_different_offsets(self):
|
||||
"""Load from two different offsets and verify correct values."""
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
v_mov_b32_e32(v[1], s[3]),
|
||||
# Store 0xAAAAAAAA at offset 100
|
||||
s_mov_b32(s[0], 0xAAAAAAAA),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
global_store_b32(addr=v[0:1], data=v[2], saddr=SrcEnum.NULL, offset=100),
|
||||
# Store 0xBBBBBBBB at offset 200
|
||||
s_mov_b32(s[0], 0xBBBBBBBB),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
global_store_b32(addr=v[0:1], data=v[2], saddr=SrcEnum.NULL, offset=200),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
# Load from offset 100 -> should get 0xAAAAAAAA
|
||||
GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[0:1], vdst=v[3], saddr=SrcEnum.NULL, offset=100),
|
||||
# Load from offset 200 -> should get 0xBBBBBBBB
|
||||
GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[0:1], vdst=v[4], saddr=SrcEnum.NULL, offset=200),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[3]),
|
||||
v_mov_b32_e32(v[1], v[4]),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][0], 0xAAAAAAAA, f"offset 100: expected 0xAAAAAAAA, got 0x{st.vgpr[0][0]:08x}")
|
||||
self.assertEqual(st.vgpr[0][1], 0xBBBBBBBB, f"offset 200: expected 0xBBBBBBBB, got 0x{st.vgpr[0][1]:08x}")
|
||||
|
||||
def test_global_store_different_offsets(self):
|
||||
"""Store to two different offsets and verify correct values."""
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
v_mov_b32_e32(v[1], s[3]),
|
||||
# Store 0x11111111 at offset 300
|
||||
s_mov_b32(s[0], 0x11111111),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
global_store_b32(addr=v[0:1], data=v[2], saddr=SrcEnum.NULL, offset=300),
|
||||
# Store 0x22222222 at offset 400
|
||||
s_mov_b32(s[0], 0x22222222),
|
||||
v_mov_b32_e32(v[3], s[0]),
|
||||
global_store_b32(addr=v[0:1], data=v[3], saddr=SrcEnum.NULL, offset=400),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
# Load back to verify
|
||||
GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[0:1], vdst=v[4], saddr=SrcEnum.NULL, offset=300),
|
||||
GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[0:1], vdst=v[5], saddr=SrcEnum.NULL, offset=400),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[4]),
|
||||
v_mov_b32_e32(v[1], v[5]),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][0], 0x11111111, f"offset 300: expected 0x11111111, got 0x{st.vgpr[0][0]:08x}")
|
||||
self.assertEqual(st.vgpr[0][1], 0x22222222, f"offset 400: expected 0x22222222, got 0x{st.vgpr[0][1]:08x}")
|
||||
|
||||
def test_global_negative_offset_no_saddr(self):
|
||||
"""Test negative offset without saddr (VGPR pair for address).
|
||||
Store 0xAAAA at offset 100, 0xBBBB at offset 200.
|
||||
Load with offset -100 from vaddr pointing to base+200 -> should get 0xAAAA (at 100).
|
||||
Load with offset -100 from vaddr pointing to base+300 -> should get 0xBBBB (at 200)."""
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
v_mov_b32_e32(v[1], s[3]),
|
||||
# Store 0xAAAAAAAA at offset 100, 0xBBBBBBBB at offset 200
|
||||
s_mov_b32(s[0], 0xAAAAAAAA),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
global_store_b32(addr=v[0:1], data=v[2], saddr=SrcEnum.NULL, offset=100),
|
||||
s_mov_b32(s[0], 0xBBBBBBBB),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
global_store_b32(addr=v[0:1], data=v[2], saddr=SrcEnum.NULL, offset=200),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
# vaddr = base+200, load with offset -100 -> should get value at 100
|
||||
s_add_u32(s[4], s[2], 200),
|
||||
s_addc_u32(s[5], s[3], 0),
|
||||
v_mov_b32_e32(v[4], s[4]),
|
||||
v_mov_b32_e32(v[5], s[5]),
|
||||
GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[4:5], vdst=v[6], saddr=SrcEnum.NULL, offset=-100),
|
||||
# vaddr = base+300, load with offset -100 -> should get value at 200
|
||||
s_add_u32(s[4], s[2], 300),
|
||||
s_addc_u32(s[5], s[3], 0),
|
||||
v_mov_b32_e32(v[4], s[4]),
|
||||
v_mov_b32_e32(v[5], s[5]),
|
||||
GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[4:5], vdst=v[7], saddr=SrcEnum.NULL, offset=-100),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[6]),
|
||||
v_mov_b32_e32(v[1], v[7]),
|
||||
v_mov_b32_e32(v[4], 0),
|
||||
v_mov_b32_e32(v[5], 0),
|
||||
v_mov_b32_e32(v[6], 0),
|
||||
v_mov_b32_e32(v[7], 0),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
s_mov_b32(s[4], 0),
|
||||
s_mov_b32(s[5], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][0], 0xAAAAAAAA, f"offset 200-100=100: expected 0xAAAAAAAA, got 0x{st.vgpr[0][0]:08x}")
|
||||
self.assertEqual(st.vgpr[0][1], 0xBBBBBBBB, f"offset 300-100=200: expected 0xBBBBBBBB, got 0x{st.vgpr[0][1]:08x}")
|
||||
|
||||
def test_global_negative_offset_with_saddr(self):
|
||||
"""Test negative offset with saddr (SGPR pair for base address).
|
||||
Store 0xAAAA at offset 100, 0xBBBB at offset 200.
|
||||
Load with offset -100 from saddr pointing to base+200 -> should get 0xAAAA (at 100).
|
||||
Load with offset -100 from saddr pointing to base+300 -> should get 0xBBBB (at 200)."""
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
# Store 0xAAAAAAAA at offset 100, 0xBBBBBBBB at offset 200
|
||||
s_mov_b32(s[0], 0xAAAAAAAA),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=100),
|
||||
s_mov_b32(s[0], 0xBBBBBBBB),
|
||||
v_mov_b32_e32(v[2], s[0]),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=200),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
# saddr = base+200, load with offset -100 -> should get value at 100
|
||||
s_add_u32(s[4], s[2], 200),
|
||||
s_addc_u32(s[5], s[3], 0),
|
||||
GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[0], vdst=v[6], saddr=s[4:5], offset=-100),
|
||||
# saddr = base+300, load with offset -100 -> should get value at 200
|
||||
s_add_u32(s[4], s[2], 300),
|
||||
s_addc_u32(s[5], s[3], 0),
|
||||
GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[0], vdst=v[7], saddr=s[4:5], offset=-100),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[6]),
|
||||
v_mov_b32_e32(v[1], v[7]),
|
||||
v_mov_b32_e32(v[6], 0),
|
||||
v_mov_b32_e32(v[7], 0),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
s_mov_b32(s[4], 0),
|
||||
s_mov_b32(s[5], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][0], 0xAAAAAAAA, f"offset 200-100=100: expected 0xAAAAAAAA, got 0x{st.vgpr[0][0]:08x}")
|
||||
self.assertEqual(st.vgpr[0][1], 0xBBBBBBBB, f"offset 300-100=200: expected 0xBBBBBBBB, got 0x{st.vgpr[0][1]:08x}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
129
tinygrad_repo/test/amd/hw/test_rdna4_permlane_var.py
Normal file
129
tinygrad_repo/test/amd/hw/test_rdna4_permlane_var.py
Normal file
@@ -0,0 +1,129 @@
|
||||
"""RDNA4 V_PERMLANE16_VAR_B32 / V_PERMLANEX16_VAR_B32 coverage.
|
||||
|
||||
Exercises the generated pcode path end-to-end in the emulator and compares against
|
||||
real RDNA4 hardware when USE_HW=1.
|
||||
"""
|
||||
import ctypes, unittest
|
||||
import tinygrad.runtime.autogen.amd.rdna4.ins as r4
|
||||
from tinygrad.helpers import flat_mv
|
||||
from tinygrad.renderer.amd.dsl import NULL
|
||||
from test.amd.hw.helpers import USE_HW, assemble
|
||||
from test.mockgpu.amd.emu import run_asm
|
||||
|
||||
LANES = 32
|
||||
|
||||
def _code(instructions: list, out_reg: int = 2) -> bytes:
|
||||
return assemble([
|
||||
r4.s_mov_b32(r4.s[80], r4.s[0]),
|
||||
r4.s_mov_b32(r4.s[81], r4.s[1]),
|
||||
r4.v_mov_b32_e32(r4.v[255], r4.v[0]),
|
||||
*instructions,
|
||||
r4.s_load_b64(r4.s[92:93], r4.s[80:81], soffset=NULL),
|
||||
r4.s_wait_kmcnt(simm16=0),
|
||||
r4.v_lshlrev_b32_e32(r4.v[240], 2, r4.v[255]),
|
||||
r4.v_mov_b32_e32(r4.v[241], 0),
|
||||
r4.global_store_b32(vaddr=r4.v[240:241], saddr=r4.s[92:93], vsrc=r4.v[out_reg]),
|
||||
r4.s_endpgm(),
|
||||
])
|
||||
|
||||
def _run_emu(instructions: list, out_reg: int = 2) -> list[int]:
|
||||
out_buf = (ctypes.c_uint32 * LANES)(*([0] * LANES))
|
||||
args = (ctypes.c_uint64 * 1)(ctypes.addressof(out_buf))
|
||||
code = _code(instructions, out_reg)
|
||||
kernel_buf = (ctypes.c_char * len(code)).from_buffer_copy(code)
|
||||
result = run_asm(ctypes.addressof(kernel_buf), len(code), 1, 1, 1, LANES, 1, 1, ctypes.addressof(args), arch='rdna4')
|
||||
assert result == 0, f"run_asm failed with {result}"
|
||||
return list(out_buf)
|
||||
|
||||
def _run_hw(instructions: list, out_reg: int = 2) -> list[int]:
|
||||
from tinygrad.device import Device
|
||||
from tinygrad.runtime.ops_amd import AMDProgram
|
||||
from tinygrad.runtime.support.compiler_amd import HIPCompiler
|
||||
|
||||
dev = Device['AMD']
|
||||
if not dev.arch.startswith('gfx12'): raise unittest.SkipTest('requires RDNA4 hardware')
|
||||
compiler = HIPCompiler(dev.arch)
|
||||
code = _code(instructions, out_reg)
|
||||
byte_str = ', '.join(f'0x{b:02x}' for b in code)
|
||||
asm_src = f""".text
|
||||
.globl test
|
||||
.p2align 8
|
||||
.type test,@function
|
||||
test:
|
||||
.byte {byte_str}
|
||||
|
||||
.rodata
|
||||
.p2align 6
|
||||
.amdhsa_kernel test
|
||||
.amdhsa_next_free_vgpr 256
|
||||
.amdhsa_next_free_sgpr 96
|
||||
.amdhsa_wavefront_size32 1
|
||||
.amdhsa_user_sgpr_kernarg_segment_ptr 1
|
||||
.amdhsa_kernarg_size 8
|
||||
.amdhsa_group_segment_fixed_size 65536
|
||||
.amdhsa_private_segment_fixed_size 65536
|
||||
.amdhsa_enable_private_segment 1
|
||||
.end_amdhsa_kernel
|
||||
|
||||
.amdgpu_metadata
|
||||
---
|
||||
amdhsa.version:
|
||||
- 1
|
||||
- 0
|
||||
amdhsa.kernels:
|
||||
- .name: test
|
||||
.symbol: test.kd
|
||||
.kernarg_segment_size: 8
|
||||
.group_segment_fixed_size: 65536
|
||||
.private_segment_fixed_size: 65536
|
||||
.kernarg_segment_align: 8
|
||||
.wavefront_size: 32
|
||||
.sgpr_count: 96
|
||||
.vgpr_count: 256
|
||||
.max_flat_workgroup_size: 1024
|
||||
...
|
||||
.end_amdgpu_metadata
|
||||
"""
|
||||
lib = compiler.compile(asm_src)
|
||||
prg = AMDProgram(dev, 'test', lib)
|
||||
out_gpu = dev.allocator.alloc(LANES * 4)
|
||||
prg(out_gpu, global_size=(1, 1, 1), local_size=(LANES, 1, 1), wait=True)
|
||||
out = bytearray(LANES * 4)
|
||||
dev.allocator._copyout(flat_mv(memoryview(out)), out_gpu)
|
||||
return [int.from_bytes(out[i*4:(i+1)*4], 'little') for i in range(LANES)]
|
||||
|
||||
def run_rdna4(instructions: list, out_reg: int = 2) -> list[int]:
|
||||
emu = _run_emu(instructions, out_reg)
|
||||
if not USE_HW: return emu
|
||||
hw = _run_hw(instructions, out_reg)
|
||||
if emu != hw:
|
||||
diffs = [f"lane {i}: emu=0x{e:08x} hw=0x{h:08x}" for i, (e, h) in enumerate(zip(emu, hw)) if e != h]
|
||||
raise AssertionError("Emulator vs Hardware mismatch:\n" + '\n'.join(diffs[:16]))
|
||||
return hw
|
||||
|
||||
class TestPermlaneVarRDNA4(unittest.TestCase):
|
||||
def test_v_permlane16_var_b32_reverse(self):
|
||||
out = run_rdna4([
|
||||
r4.v_mov_b32_e32(r4.v[0], r4.v[255]),
|
||||
r4.v_xor_b32_e32(r4.v[1], 15, r4.v[255]),
|
||||
r4.v_permlane16_var_b32(r4.v[2], r4.v[0], r4.v[1]),
|
||||
])
|
||||
self.assertEqual(out[0], 15)
|
||||
self.assertEqual(out[5], 10)
|
||||
self.assertEqual(out[15], 0)
|
||||
self.assertEqual(out[16], 31)
|
||||
self.assertEqual(out[21], 26)
|
||||
self.assertEqual(out[31], 16)
|
||||
|
||||
def test_v_permlanex16_var_b32_cross_row(self):
|
||||
out = run_rdna4([
|
||||
r4.v_mov_b32_e32(r4.v[0], r4.v[255]),
|
||||
r4.v_mov_b32_e32(r4.v[1], r4.v[255]),
|
||||
r4.v_permlanex16_var_b32(r4.v[2], r4.v[0], r4.v[1]),
|
||||
])
|
||||
self.assertEqual(out[0], 16)
|
||||
self.assertEqual(out[5], 21)
|
||||
self.assertEqual(out[15], 31)
|
||||
self.assertEqual(out[16], 0)
|
||||
self.assertEqual(out[21], 5)
|
||||
self.assertEqual(out[31], 15)
|
||||
355
tinygrad_repo/test/amd/hw/test_scratch.py
Normal file
355
tinygrad_repo/test/amd/hw/test_scratch.py
Normal file
@@ -0,0 +1,355 @@
|
||||
"""Tests for SCRATCH instructions - scratch (private) memory operations.
|
||||
|
||||
Includes: scratch_load_*, scratch_store_*
|
||||
"""
|
||||
import unittest
|
||||
from test.amd.hw.helpers import *
|
||||
|
||||
class TestScratchStore(unittest.TestCase):
|
||||
"""Tests for SCRATCH store instructions."""
|
||||
|
||||
def test_scratch_store_b32_basic(self):
|
||||
"""SCRATCH_STORE_B32 stores 32-bit value to scratch memory."""
|
||||
TEST_OFFSET = 256
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[4], 0xDEADBEEF),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
# Store via scratch
|
||||
scratch_store_b32(addr=v[0], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
# Load back via scratch
|
||||
scratch_load_b32(addr=v[0], vdst=v[3], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[3]),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][0], 0xDEADBEEF)
|
||||
|
||||
def test_scratch_store_b64_basic(self):
|
||||
"""SCRATCH_STORE_B64 stores 64-bit value to scratch memory."""
|
||||
TEST_OFFSET = 256
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[4], 0xDEADBEEF),
|
||||
s_mov_b32(s[5], 0xCAFEBABE),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
v_mov_b32_e32(v[3], s[5]),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
scratch_store_b64(addr=v[0], data=v[2:3], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
scratch_load_b64(addr=v[0], vdst=v[4:5], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[4]),
|
||||
v_mov_b32_e32(v[1], v[5]),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][0], 0xDEADBEEF)
|
||||
self.assertEqual(st.vgpr[0][1], 0xCAFEBABE)
|
||||
|
||||
def test_scratch_store_b8_basic(self):
|
||||
"""SCRATCH_STORE_B8 stores single byte to scratch memory."""
|
||||
TEST_OFFSET = 256
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
# First store full word
|
||||
s_mov_b32(s[4], 0xDEADBEEF),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
scratch_store_b32(addr=v[0], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
# Store single byte
|
||||
v_mov_b32_e32(v[2], 0x42),
|
||||
scratch_store_b8(addr=v[0], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
# Load back
|
||||
scratch_load_b32(addr=v[0], vdst=v[3], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[3]),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
# Only byte 0 should change from 0xEF to 0x42
|
||||
self.assertEqual(st.vgpr[0][0], 0xDEADBE42)
|
||||
|
||||
def test_scratch_store_b16_basic(self):
|
||||
"""SCRATCH_STORE_B16 stores 16-bit value to scratch memory."""
|
||||
TEST_OFFSET = 256
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[4], 0xDEADBEEF),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
scratch_store_b32(addr=v[0], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[4], 0xCAFE),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
scratch_store_b16(addr=v[0], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
scratch_load_b32(addr=v[0], vdst=v[3], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[3]),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][0], 0xDEADCAFE)
|
||||
|
||||
|
||||
class TestScratchLoad(unittest.TestCase):
|
||||
"""Tests for SCRATCH load instructions."""
|
||||
|
||||
def test_scratch_load_b96(self):
|
||||
"""SCRATCH_LOAD_B96 loads 96-bit value correctly."""
|
||||
TEST_OFFSET = 256
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
s_mov_b32(s[4], 0xAAAAAAAA),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
s_mov_b32(s[4], 0xBBBBBBBB),
|
||||
v_mov_b32_e32(v[3], s[4]),
|
||||
s_mov_b32(s[4], 0xCCCCCCCC),
|
||||
v_mov_b32_e32(v[4], s[4]),
|
||||
scratch_store_b96(addr=v[0], data=v[2:4], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
scratch_load_b96(addr=v[0], vdst=v[5:7], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[5]),
|
||||
v_mov_b32_e32(v[1], v[6]),
|
||||
v_mov_b32_e32(v[2], v[7]),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][0], 0xAAAAAAAA)
|
||||
self.assertEqual(st.vgpr[0][1], 0xBBBBBBBB)
|
||||
self.assertEqual(st.vgpr[0][2], 0xCCCCCCCC)
|
||||
|
||||
def test_scratch_load_b128(self):
|
||||
"""SCRATCH_LOAD_B128 loads 128-bit value correctly."""
|
||||
TEST_OFFSET = 256
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
s_mov_b32(s[4], 0xDEADBEEF),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
s_mov_b32(s[4], 0xCAFEBABE),
|
||||
v_mov_b32_e32(v[3], s[4]),
|
||||
s_mov_b32(s[4], 0x12345678),
|
||||
v_mov_b32_e32(v[4], s[4]),
|
||||
s_mov_b32(s[4], 0x9ABCDEF0),
|
||||
v_mov_b32_e32(v[5], s[4]),
|
||||
scratch_store_b128(addr=v[0], data=v[2:5], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
scratch_load_b128(addr=v[0], vdst=v[6:9], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[6]),
|
||||
v_mov_b32_e32(v[1], v[7]),
|
||||
v_mov_b32_e32(v[2], v[8]),
|
||||
v_mov_b32_e32(v[3], v[9]),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][0], 0xDEADBEEF)
|
||||
self.assertEqual(st.vgpr[0][1], 0xCAFEBABE)
|
||||
self.assertEqual(st.vgpr[0][2], 0x12345678)
|
||||
self.assertEqual(st.vgpr[0][3], 0x9ABCDEF0)
|
||||
|
||||
def test_scratch_load_u8(self):
|
||||
"""SCRATCH_LOAD_U8 loads unsigned byte with zero extension."""
|
||||
TEST_OFFSET = 256
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
s_mov_b32(s[4], 0xDEADBEAB),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
scratch_store_b32(addr=v[0], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
scratch_load_u8(addr=v[0], vdst=v[3], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[3]),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][0], 0xAB)
|
||||
|
||||
def test_scratch_load_i8(self):
|
||||
"""SCRATCH_LOAD_I8 loads signed byte with sign extension."""
|
||||
TEST_OFFSET = 256
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
s_mov_b32(s[4], 0x80), # -128 as signed byte
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
scratch_store_b8(addr=v[0], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
scratch_load_i8(addr=v[0], vdst=v[3], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[3]),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][0], 0xFFFFFF80)
|
||||
|
||||
def test_scratch_load_u16(self):
|
||||
"""SCRATCH_LOAD_U16 loads unsigned 16-bit with zero extension."""
|
||||
TEST_OFFSET = 256
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
s_mov_b32(s[4], 0xDEADCAFE),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
scratch_store_b32(addr=v[0], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
scratch_load_u16(addr=v[0], vdst=v[3], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[3]),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][0], 0xCAFE)
|
||||
|
||||
def test_scratch_load_i16(self):
|
||||
"""SCRATCH_LOAD_I16 loads signed 16-bit with sign extension."""
|
||||
TEST_OFFSET = 256
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
s_mov_b32(s[4], 0x8000), # -32768 as signed 16-bit
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
scratch_store_b16(addr=v[0], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
scratch_load_i16(addr=v[0], vdst=v[3], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[3]),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][0], 0xFFFF8000)
|
||||
|
||||
|
||||
class TestScratchSVE(unittest.TestCase):
|
||||
"""Tests for SCRATCH SVE (Scratch VGPR Enable) bit behavior."""
|
||||
|
||||
def test_scratch_sve_zero_ignores_vaddr(self):
|
||||
"""With SVE=0, VADDR should be ignored in address calculation."""
|
||||
TEST_OFFSET = 256
|
||||
# Store a marker value at offset 256 (where SVE=0 should go)
|
||||
# Then set v[0] to a non-zero value (100) and store via scratch with SVE=0
|
||||
# If SVE=0 is handled correctly, the VADDR (100) should be IGNORED,
|
||||
# and the store should go to offset 256, not 256+100=356
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
# First, store 0xAAAAAAAA at offset 256 with v[0]=0
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
s_mov_b32(s[4], 0xAAAAAAAA),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
scratch_store_b32(addr=v[0], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET, sve=0),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
# Now set v[0] to 100 (non-zero) and store 0xBBBBBBBB with SVE=0
|
||||
# With SVE=0, v[0] should be IGNORED, so this should overwrite offset 256
|
||||
v_mov_b32_e32(v[0], 100),
|
||||
s_mov_b32(s[4], 0xBBBBBBBB),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
scratch_store_b32(addr=v[0], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET, sve=0),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
# Load back from offset 256 (with v[0]=0) - should get 0xBBBBBBBB
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
scratch_load_b32(addr=v[0], vdst=v[3], saddr=SrcEnum.NULL, offset=TEST_OFFSET, sve=0),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[3]),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
# If SVE=0 works correctly, v[0] should be 0xBBBBBBBB (the second store overwrote the first)
|
||||
# If SVE=0 is wrong (VADDR used), v[0] would be 0xAAAAAAAA (stores went to different locations)
|
||||
self.assertEqual(st.vgpr[0][0], 0xBBBBBBBB, "SVE=0 should ignore VADDR, both stores should go to same location")
|
||||
|
||||
def test_scratch_sve_one_uses_vaddr(self):
|
||||
"""With SVE=1, VADDR should be used as offset in address calculation."""
|
||||
TEST_OFFSET = 256
|
||||
# Store at offset 256 with v[0]=0, then store at offset 256 with v[0]=100 and SVE=1
|
||||
# With SVE=1, the second store should go to 256+100=356, not 256
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
# First, store 0xAAAAAAAA at offset 256 with v[0]=0
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
s_mov_b32(s[4], 0xAAAAAAAA),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
scratch_store_b32(addr=v[0], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET, sve=1),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
# Now set v[0] to 100 and store 0xBBBBBBBB with SVE=1
|
||||
# With SVE=1, v[0] IS used, so this should go to offset 256+100=356
|
||||
v_mov_b32_e32(v[0], 100),
|
||||
s_mov_b32(s[4], 0xBBBBBBBB),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
scratch_store_b32(addr=v[0], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET, sve=1),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
# Load back from offset 256 (with v[0]=0) - should still be 0xAAAAAAAA
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
scratch_load_b32(addr=v[0], vdst=v[3], saddr=SrcEnum.NULL, offset=TEST_OFFSET, sve=1),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[3]),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
# If SVE=1 works correctly, v[0] should be 0xAAAAAAAA (stores went to different locations)
|
||||
self.assertEqual(st.vgpr[0][0], 0xAAAAAAAA, "SVE=1 should use VADDR, stores should go to different locations")
|
||||
|
||||
|
||||
class TestScratchMultiLane(unittest.TestCase):
|
||||
"""Tests for SCRATCH operations with multiple lanes."""
|
||||
|
||||
def test_scratch_store_load_multi_lane(self):
|
||||
"""SCRATCH store/load works correctly with multiple lanes (private per-lane memory)."""
|
||||
TEST_OFFSET = 256
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
# Each lane stores its lane ID
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
v_mov_b32_e32(v[2], v[255]), # v[255] has packed workitem IDs, low 10 bits = x
|
||||
v_and_b32_e32(v[2], 0x3FF, v[2]), # extract lane ID
|
||||
scratch_store_b32(addr=v[0], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
# Load back
|
||||
scratch_load_b32(addr=v[0], vdst=v[3], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], v[3]),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=4)
|
||||
# Each lane should have loaded its own lane ID
|
||||
for lane in range(4):
|
||||
self.assertEqual(st.vgpr[lane][0], lane, f"Lane {lane} should have value {lane}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
448
tinygrad_repo/test/amd/hw/test_smem.py
Normal file
448
tinygrad_repo/test/amd/hw/test_smem.py
Normal file
@@ -0,0 +1,448 @@
|
||||
"""Tests for SMEM instructions - scalar memory operations.
|
||||
|
||||
Includes: s_load_b32, s_load_b64, s_load_b128, s_load_b256, s_load_b512
|
||||
Tests both immediate and register offset addressing modes.
|
||||
"""
|
||||
import unittest
|
||||
from test.amd.hw.helpers import *
|
||||
|
||||
# Use offset into output buffer for test data (output buffer is 2124 bytes)
|
||||
TEST_OFFSET = 2000
|
||||
|
||||
# Cache invalidation sequence for scalar loads after vector stores
|
||||
# s_wait_idle waits for all outstanding memory operations including cache flushes
|
||||
CACHE_INV = [s_gl1_inv(), s_dcache_inv(), s_wait_idle()]
|
||||
|
||||
class TestSLoadRegisterOffset(unittest.TestCase):
|
||||
"""Tests for s_load with register offset (soffset field).
|
||||
|
||||
Bug: s_load_b32(s[dst], s[base:base+1], s[off]) ignores the register offset
|
||||
and only uses the immediate offset field. This causes incorrect memory loads
|
||||
when the offset comes from a register.
|
||||
"""
|
||||
|
||||
def test_s_load_b32_register_offset_basic(self):
|
||||
"""s_load_b32 with register offset should load from base + reg_offset."""
|
||||
instructions = [
|
||||
# Load output buffer pointer from args
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
# Store test values to output buffer: 0xAAAAAAAA at offset, 0xBBBBBBBB at offset+4
|
||||
s_mov_b32(s[4], 0xAAAAAAAA),
|
||||
s_mov_b32(s[5], 0xBBBBBBBB),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
v_mov_b32_e32(v[3], s[5]),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
global_store_b32(addr=v[0], data=v[3], saddr=s[2:3], offset=TEST_OFFSET+4),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
*CACHE_INV,
|
||||
# Now test s_load with register offset
|
||||
# Put offset value in s[4]: offset = 4 bytes (1 dword)
|
||||
s_mov_b32(s[4], 4),
|
||||
# Load from out_ptr + TEST_OFFSET + s[4] (should load 0xBBBBBBBB)
|
||||
s_load_b32(s[5], s[2:3], s[4], offset=TEST_OFFSET),
|
||||
s_waitcnt(0),
|
||||
# Zero out pointer regs (different addresses in emu vs hw)
|
||||
s_mov_b32(s[2], 0), s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.sgpr[5], 0xBBBBBBBB,
|
||||
f"s_load with reg offset 4 should load 0xBBBBBBBB: s[5]=0x{st.sgpr[5]:08x}")
|
||||
|
||||
def test_s_load_b32_register_offset_different_from_immediate(self):
|
||||
"""s_load_b32 with register offset loads different data than immediate offset 0."""
|
||||
instructions = [
|
||||
# Load output buffer pointer from args
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
# Store test values: 0xAAAAAAAA at offset, 0xBBBBBBBB at offset+4
|
||||
s_mov_b32(s[4], 0xAAAAAAAA),
|
||||
s_mov_b32(s[5], 0xBBBBBBBB),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
v_mov_b32_e32(v[3], s[5]),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
global_store_b32(addr=v[0], data=v[3], saddr=s[2:3], offset=TEST_OFFSET+4),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
*CACHE_INV,
|
||||
# Load with immediate offset 0
|
||||
s_load_b32(s[5], s[2:3], NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt(0),
|
||||
# Load with register offset 4
|
||||
s_mov_b32(s[4], 4),
|
||||
s_load_b32(s[6], s[2:3], s[4], offset=TEST_OFFSET),
|
||||
s_waitcnt(0),
|
||||
# Zero out pointer regs (different addresses in emu vs hw)
|
||||
s_mov_b32(s[2], 0), s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
# s[5] has dword at offset 0 (0xAAAAAAAA), s[6] has dword at offset 4 (0xBBBBBBBB)
|
||||
self.assertEqual(st.sgpr[5], 0xAAAAAAAA)
|
||||
self.assertEqual(st.sgpr[6], 0xBBBBBBBB)
|
||||
self.assertNotEqual(st.sgpr[5], st.sgpr[6],
|
||||
f"s_load with reg offset 4 should load different value than offset 0: "
|
||||
f"s[5]=0x{st.sgpr[5]:08x}, s[6]=0x{st.sgpr[6]:08x}")
|
||||
|
||||
def test_s_load_b32_register_offset_same_as_dst(self):
|
||||
"""s_load_b32 where soffset register is same as destination.
|
||||
|
||||
This is the exact pattern that exposes the bug:
|
||||
s_load_b32(s[8], s[2:3], s[8])
|
||||
The offset should be read BEFORE the destination is overwritten.
|
||||
"""
|
||||
instructions = [
|
||||
# Load output buffer pointer from args
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
# Store test values: 0xAAAAAAAA at offset, 0xBBBBBBBB at offset+4
|
||||
s_mov_b32(s[6], 0xAAAAAAAA),
|
||||
s_mov_b32(s[7], 0xBBBBBBBB),
|
||||
v_mov_b32_e32(v[2], s[6]),
|
||||
v_mov_b32_e32(v[3], s[7]),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
global_store_b32(addr=v[0], data=v[3], saddr=s[2:3], offset=TEST_OFFSET+4),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
*CACHE_INV,
|
||||
# Set up s[4] = 4 (offset in bytes)
|
||||
s_mov_b32(s[4], 4),
|
||||
# Load using s[4] as both offset and destination
|
||||
# Should load from base + 4, then store result in s[4]
|
||||
s_load_b32(s[4], s[2:3], s[4], offset=TEST_OFFSET),
|
||||
s_waitcnt(0),
|
||||
# Also load with immediate offset 4 for comparison
|
||||
s_load_b32(s[5], s[2:3], NULL, offset=TEST_OFFSET+4),
|
||||
s_waitcnt(0),
|
||||
# Zero out pointer regs (different addresses in emu vs hw)
|
||||
s_mov_b32(s[2], 0), s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
# s[4] and s[5] should have the same value (both loaded from offset 4 = 0xBBBBBBBB)
|
||||
self.assertEqual(st.sgpr[4], 0xBBBBBBBB)
|
||||
self.assertEqual(st.sgpr[4], st.sgpr[5],
|
||||
f"s_load with reg offset s[4]=4 should match immediate offset=4: "
|
||||
f"s[4]=0x{st.sgpr[4]:08x}, s[5]=0x{st.sgpr[5]:08x}")
|
||||
|
||||
def test_s_load_b32_register_offset_zero(self):
|
||||
"""s_load_b32 with register offset = 0 should be same as immediate offset 0."""
|
||||
instructions = [
|
||||
# Load output buffer pointer from args
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
# Store test value: 0xDEADBEEF at offset
|
||||
s_mov_b32(s[7], 0xDEADBEEF),
|
||||
v_mov_b32_e32(v[2], s[7]),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
*CACHE_INV,
|
||||
# Load with register offset 0
|
||||
s_mov_b32(s[4], 0),
|
||||
s_load_b32(s[5], s[2:3], s[4], offset=TEST_OFFSET),
|
||||
s_waitcnt(0),
|
||||
# Load with immediate offset 0
|
||||
s_load_b32(s[6], s[2:3], NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt(0),
|
||||
# Zero out pointer regs (different addresses in emu vs hw)
|
||||
s_mov_b32(s[2], 0), s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.sgpr[5], 0xDEADBEEF)
|
||||
self.assertEqual(st.sgpr[5], st.sgpr[6],
|
||||
f"s_load with reg offset 0 should match immediate offset 0: "
|
||||
f"s[5]=0x{st.sgpr[5]:08x}, s[6]=0x{st.sgpr[6]:08x}")
|
||||
|
||||
def test_s_load_b32_register_plus_immediate_offset(self):
|
||||
"""s_load_b32 with both register and immediate offset should add them."""
|
||||
instructions = [
|
||||
# Load output buffer pointer from args
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
# Store test values: 0xAAAAAAAA at offset, 0xBBBBBBBB at offset+4
|
||||
s_mov_b32(s[8], 0xAAAAAAAA),
|
||||
s_mov_b32(s[9], 0xBBBBBBBB),
|
||||
v_mov_b32_e32(v[2], s[8]),
|
||||
v_mov_b32_e32(v[3], s[9]),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
global_store_b32(addr=v[0], data=v[3], saddr=s[2:3], offset=TEST_OFFSET+4),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
*CACHE_INV,
|
||||
# reg offset = 4, imm offset = 0 -> total offset = 4
|
||||
s_mov_b32(s[4], 4),
|
||||
s_load_b32(s[5], s[2:3], s[4], offset=TEST_OFFSET),
|
||||
s_waitcnt(0),
|
||||
# reg offset = 0, imm offset = 4 -> total offset = 4
|
||||
s_mov_b32(s[6], 0),
|
||||
s_load_b32(s[7], s[2:3], s[6], offset=TEST_OFFSET+4),
|
||||
s_waitcnt(0),
|
||||
# Zero out pointer regs (different addresses in emu vs hw)
|
||||
s_mov_b32(s[2], 0), s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
# Both should load from offset 4 (0xBBBBBBBB)
|
||||
self.assertEqual(st.sgpr[5], 0xBBBBBBBB)
|
||||
self.assertEqual(st.sgpr[7], 0xBBBBBBBB)
|
||||
self.assertEqual(st.sgpr[5], st.sgpr[7],
|
||||
f"reg_off=4 + imm_off=0 should equal reg_off=0 + imm_off=4: "
|
||||
f"s[5]=0x{st.sgpr[5]:08x}, s[7]=0x{st.sgpr[7]:08x}")
|
||||
|
||||
|
||||
class TestSLoadMultiDword(unittest.TestCase):
|
||||
"""Tests for multi-dword s_load with register offset."""
|
||||
|
||||
def test_s_load_b64_register_offset(self):
|
||||
"""s_load_b64 with register offset should load 2 dwords from base + reg_offset."""
|
||||
instructions = [
|
||||
# Load output buffer pointer from args
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
# Store test values: 0xAAAAAAAA, 0xBBBBBBBB at offset
|
||||
s_mov_b32(s[10], 0xAAAAAAAA),
|
||||
s_mov_b32(s[11], 0xBBBBBBBB),
|
||||
v_mov_b32_e32(v[2], s[10]),
|
||||
v_mov_b32_e32(v[3], s[11]),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
global_store_b32(addr=v[0], data=v[3], saddr=s[2:3], offset=TEST_OFFSET+4),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
*CACHE_INV,
|
||||
# Load with register offset 0
|
||||
s_mov_b32(s[4], 0),
|
||||
s_load_b64(s[6:7], s[2:3], s[4], offset=TEST_OFFSET),
|
||||
s_waitcnt(0),
|
||||
# Compare with immediate offset
|
||||
s_load_b64(s[8:9], s[2:3], NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt(0),
|
||||
# Zero out pointer regs (different addresses in emu vs hw)
|
||||
s_mov_b32(s[2], 0), s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.sgpr[6], 0xAAAAAAAA)
|
||||
self.assertEqual(st.sgpr[7], 0xBBBBBBBB)
|
||||
self.assertEqual(st.sgpr[6], st.sgpr[8])
|
||||
self.assertEqual(st.sgpr[7], st.sgpr[9])
|
||||
|
||||
def test_s_load_b128_register_offset(self):
|
||||
"""s_load_b128 with register offset should load 4 dwords from base + reg_offset."""
|
||||
instructions = [
|
||||
# Load output buffer pointer from args
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
# Store test values: 0xAAAAAAAA, 0xBBBBBBBB, 0xCCCCCCCC, 0xDDDDDDDD at offset
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
s_mov_b32(s[14], 0xAAAAAAAA),
|
||||
v_mov_b32_e32(v[2], s[14]),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
s_mov_b32(s[14], 0xBBBBBBBB),
|
||||
v_mov_b32_e32(v[2], s[14]),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+4),
|
||||
s_mov_b32(s[14], 0xCCCCCCCC),
|
||||
v_mov_b32_e32(v[2], s[14]),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+8),
|
||||
s_mov_b32(s[14], 0xDDDDDDDD),
|
||||
v_mov_b32_e32(v[2], s[14]),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+12),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
*CACHE_INV,
|
||||
# Load with register offset 0 (s_load_b128 requires 4-aligned dest: s[4], s[8], s[12], ...)
|
||||
s_mov_b32(s[15], 0),
|
||||
s_load_b128(s[4:7], s[2:3], s[15], offset=TEST_OFFSET),
|
||||
s_waitcnt(0),
|
||||
# Compare with immediate offset
|
||||
s_load_b128(s[8:11], s[2:3], NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt(0),
|
||||
# Zero out pointer regs (different addresses in emu vs hw)
|
||||
s_mov_b32(s[2], 0), s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.sgpr[4], 0xAAAAAAAA)
|
||||
self.assertEqual(st.sgpr[5], 0xBBBBBBBB)
|
||||
self.assertEqual(st.sgpr[6], 0xCCCCCCCC)
|
||||
self.assertEqual(st.sgpr[7], 0xDDDDDDDD)
|
||||
self.assertEqual(st.sgpr[4], st.sgpr[8])
|
||||
self.assertEqual(st.sgpr[5], st.sgpr[9])
|
||||
|
||||
|
||||
class TestSLoadLarge(unittest.TestCase):
|
||||
"""Tests for large s_load operations (s_load_b256, s_load_b512)."""
|
||||
|
||||
def test_s_load_b256_basic(self):
|
||||
"""s_load_b256 loads 8 consecutive dwords."""
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
# Store 8 test values
|
||||
s_mov_b32(s[20], 0x11111111),
|
||||
v_mov_b32_e32(v[2], s[20]),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
|
||||
s_mov_b32(s[20], 0x22222222),
|
||||
v_mov_b32_e32(v[2], s[20]),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+4),
|
||||
s_mov_b32(s[20], 0x33333333),
|
||||
v_mov_b32_e32(v[2], s[20]),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+8),
|
||||
s_mov_b32(s[20], 0x44444444),
|
||||
v_mov_b32_e32(v[2], s[20]),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+12),
|
||||
s_mov_b32(s[20], 0x55555555),
|
||||
v_mov_b32_e32(v[2], s[20]),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+16),
|
||||
s_mov_b32(s[20], 0x66666666),
|
||||
v_mov_b32_e32(v[2], s[20]),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+20),
|
||||
s_mov_b32(s[20], 0x77777777),
|
||||
v_mov_b32_e32(v[2], s[20]),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+24),
|
||||
s_mov_b32(s[20], 0x88888888),
|
||||
v_mov_b32_e32(v[2], s[20]),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+28),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
*CACHE_INV,
|
||||
# Load all 8 dwords with s_load_b256
|
||||
s_load_b256(s[4:11], s[2:3], NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[2], 0), s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.sgpr[4], 0x11111111)
|
||||
self.assertEqual(st.sgpr[5], 0x22222222)
|
||||
self.assertEqual(st.sgpr[6], 0x33333333)
|
||||
self.assertEqual(st.sgpr[7], 0x44444444)
|
||||
self.assertEqual(st.sgpr[8], 0x55555555)
|
||||
self.assertEqual(st.sgpr[9], 0x66666666)
|
||||
self.assertEqual(st.sgpr[10], 0x77777777)
|
||||
self.assertEqual(st.sgpr[11], 0x88888888)
|
||||
|
||||
def test_s_load_b512_basic(self):
|
||||
"""s_load_b512 loads 16 consecutive dwords."""
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
# Store 16 test values (use a pattern: 0x10, 0x20, ..., 0x100)
|
||||
*[instr for i in range(16) for instr in [
|
||||
s_mov_b32(s[20], (i + 1) * 0x11111111),
|
||||
v_mov_b32_e32(v[2], s[20]),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET + i * 4),
|
||||
]],
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
*CACHE_INV,
|
||||
# Load all 16 dwords with s_load_b512
|
||||
s_load_b512(s[64:79], s[2:3], NULL, offset=TEST_OFFSET),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
# Copy results to lower regs for verification (since st.sgpr only has 16 regs in test)
|
||||
s_mov_b32(s[4], s[64]),
|
||||
s_mov_b32(s[5], s[65]),
|
||||
s_mov_b32(s[6], s[78]),
|
||||
s_mov_b32(s[7], s[79]),
|
||||
s_mov_b32(s[2], 0), s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.sgpr[4], 0x11111111, "first dword")
|
||||
self.assertEqual(st.sgpr[5], 0x22222222, "second dword")
|
||||
self.assertEqual(st.sgpr[6], 0xFFFFFFFF & (15 * 0x11111111), "15th dword")
|
||||
self.assertEqual(st.sgpr[7], 0xFFFFFFFF & (16 * 0x11111111), "16th dword")
|
||||
|
||||
def test_s_load_b256_with_register_offset(self):
|
||||
"""s_load_b256 with register offset should add reg offset to address."""
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
# Store pattern at TEST_OFFSET+8: skip first 2 dwords
|
||||
*[instr for i in range(8) for instr in [
|
||||
s_mov_b32(s[20], (i + 1) * 0x11111111),
|
||||
v_mov_b32_e32(v[2], s[20]),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET + 8 + i * 4),
|
||||
]],
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
*CACHE_INV,
|
||||
# Load with register offset 8
|
||||
s_mov_b32(s[20], 8),
|
||||
s_load_b256(s[4:11], s[2:3], s[20], offset=TEST_OFFSET),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[2], 0), s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.sgpr[4], 0x11111111, "first dword at offset+8")
|
||||
self.assertEqual(st.sgpr[5], 0x22222222, "second dword at offset+8")
|
||||
self.assertEqual(st.sgpr[11], 0x88888888, "last dword at offset+8")
|
||||
|
||||
|
||||
class TestSLoadOffset(unittest.TestCase):
|
||||
"""Tests for s_load with different immediate offsets.
|
||||
|
||||
These tests verify that instruction deduplication correctly handles different offset values.
|
||||
If offset is made dynamic incorrectly, instructions with different offsets may load wrong data.
|
||||
"""
|
||||
|
||||
def test_s_load_different_offsets(self):
|
||||
"""Load from two different offsets and verify correct values."""
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
# Store 0xAAAAAAAA at offset 100
|
||||
s_mov_b32(s[4], 0xAAAAAAAA),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=100),
|
||||
# Store 0xBBBBBBBB at offset 200
|
||||
s_mov_b32(s[4], 0xBBBBBBBB),
|
||||
v_mov_b32_e32(v[2], s[4]),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=200),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
*CACHE_INV,
|
||||
# Load from offset 100 -> should get 0xAAAAAAAA
|
||||
s_load_b32(s[4], s[2:3], NULL, offset=100),
|
||||
# Load from offset 200 -> should get 0xBBBBBBBB
|
||||
s_load_b32(s[5], s[2:3], NULL, offset=200),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[2], 0), s_mov_b32(s[3], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.sgpr[4], 0xAAAAAAAA, f"offset 100: expected 0xAAAAAAAA, got 0x{st.sgpr[4]:08x}")
|
||||
self.assertEqual(st.sgpr[5], 0xBBBBBBBB, f"offset 200: expected 0xBBBBBBBB, got 0x{st.sgpr[5]:08x}")
|
||||
|
||||
def test_s_load_negative_offset(self):
|
||||
"""Test negative offset (21-bit signed).
|
||||
Store 0xAAAA at offset 100, 0xBBBB at offset 200.
|
||||
Load with offset -100 from base+200 -> should get 0xAAAA.
|
||||
Load with offset -100 from base+300 -> should get 0xBBBB."""
|
||||
instructions = [
|
||||
s_load_b64(s[2:3], s[80:81], 0, soffset=NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
# Store 0xAAAAAAAA at offset 100, 0xBBBBBBBB at offset 200
|
||||
s_mov_b32(s[8], 0xAAAAAAAA),
|
||||
v_mov_b32_e32(v[2], s[8]),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=100),
|
||||
s_mov_b32(s[8], 0xBBBBBBBB),
|
||||
v_mov_b32_e32(v[2], s[8]),
|
||||
global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=200),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
*CACHE_INV,
|
||||
# base+200, load with offset -100 -> should get value at 100
|
||||
s_add_u32(s[6], s[2], 200),
|
||||
s_addc_u32(s[7], s[3], 0),
|
||||
s_load_b32(s[4], s[6:7], NULL, offset=-100),
|
||||
# base+300, load with offset -100 -> should get value at 200
|
||||
s_add_u32(s[6], s[2], 300),
|
||||
s_addc_u32(s[7], s[3], 0),
|
||||
s_load_b32(s[5], s[6:7], NULL, offset=-100),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
s_mov_b32(s[2], 0),
|
||||
s_mov_b32(s[3], 0),
|
||||
s_mov_b32(s[6], 0),
|
||||
s_mov_b32(s[7], 0),
|
||||
s_mov_b32(s[8], 0),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.sgpr[4], 0xAAAAAAAA, f"offset 200-100=100: expected 0xAAAAAAAA, got 0x{st.sgpr[4]:08x}")
|
||||
self.assertEqual(st.sgpr[5], 0xBBBBBBBB, f"offset 300-100=200: expected 0xBBBBBBBB, got 0x{st.sgpr[5]:08x}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
1007
tinygrad_repo/test/amd/hw/test_sop.py
Normal file
1007
tinygrad_repo/test/amd/hw/test_sop.py
Normal file
File diff suppressed because it is too large
Load Diff
35
tinygrad_repo/test/amd/hw/test_vinterp.py
Normal file
35
tinygrad_repo/test/amd/hw/test_vinterp.py
Normal file
@@ -0,0 +1,35 @@
|
||||
"""Tests for VINTERP instructions."""
|
||||
import unittest
|
||||
from test.amd.hw.helpers import *
|
||||
|
||||
class TestVInterp(unittest.TestCase):
|
||||
def test_v_interp_p10_f32(self):
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], v[255]),
|
||||
v_cvt_f32_u32_e32(v[1], v[10]),
|
||||
s_mov_b32(s[0], f2i(100.0)),
|
||||
v_add_f32_e32(v[1], s[0], v[1]),
|
||||
v_cvt_f32_u32_e32(v[3], v[10]),
|
||||
s_mov_b32(s[1], f2i(10.0)),
|
||||
v_add_f32_e32(v[3], s[1], v[3]),
|
||||
s_mov_b32(s[2], f2i(2.0)),
|
||||
v_interp_p10_f32(v[4], v[1], s[2], v[3]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=8)
|
||||
for lane in range(4): self.assertAlmostEqual(i2f(st.vgpr[lane][4]), 212.0, places=5)
|
||||
for lane in range(4, 8): self.assertAlmostEqual(i2f(st.vgpr[lane][4]), 224.0, places=5)
|
||||
|
||||
def test_v_interp_p10_f16_f32(self):
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[10], v[255]),
|
||||
v_cvt_f32_u32_e32(v[11], v[10]),
|
||||
v_cvt_f16_f32_e32(v[1], v[11]),
|
||||
s_mov_b32(s[0], f2i(10.0)),
|
||||
v_add_f32_e32(v[12], s[0], v[11]),
|
||||
v_cvt_f16_f32_e32(v[3], v[12]),
|
||||
s_mov_b32(s[1], f2i(2.0)),
|
||||
v_interp_p10_f16_f32(v[4], v[1], s[1], v[3]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=8)
|
||||
for lane in range(4): self.assertAlmostEqual(i2f(st.vgpr[lane][4]), 12.0, places=5)
|
||||
for lane in range(4, 8): self.assertAlmostEqual(i2f(st.vgpr[lane][4]), 24.0, places=5)
|
||||
1633
tinygrad_repo/test/amd/hw/test_vop1.py
Normal file
1633
tinygrad_repo/test/amd/hw/test_vop1.py
Normal file
File diff suppressed because it is too large
Load Diff
994
tinygrad_repo/test/amd/hw/test_vop2.py
Normal file
994
tinygrad_repo/test/amd/hw/test_vop2.py
Normal file
@@ -0,0 +1,994 @@
|
||||
"""Tests for VOP2 instructions - two operand vector operations.
|
||||
|
||||
Includes: v_add_f32, v_mul_f32, v_and_b32, v_or_b32, v_xor_b32,
|
||||
v_lshrrev_b32, v_lshlrev_b32, v_fmac_f32, v_fmaak_f32, v_fmamk_f32,
|
||||
v_add_nc_u32, v_cndmask_b32, v_add_f16, v_mul_f16
|
||||
"""
|
||||
import unittest
|
||||
from test.amd.hw.helpers import *
|
||||
|
||||
class TestBasicArithmetic(unittest.TestCase):
|
||||
"""Tests for basic arithmetic VOP2 instructions."""
|
||||
|
||||
def test_v_add_f32(self):
|
||||
"""V_ADD_F32 adds two floats."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], 1.0),
|
||||
v_mov_b32_e32(v[1], 2.0),
|
||||
v_add_f32_e32(v[2], v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertAlmostEqual(i2f(st.vgpr[0][2]), 3.0, places=5)
|
||||
|
||||
def test_v_mul_f32(self):
|
||||
"""V_MUL_F32 multiplies two floats."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], 2.0),
|
||||
v_mov_b32_e32(v[1], 4.0),
|
||||
v_mul_f32_e32(v[2], v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertAlmostEqual(i2f(st.vgpr[0][2]), 8.0, places=5)
|
||||
|
||||
def test_v_add_f32_dpp_row_shl(self):
|
||||
"""V_ADD_F32 DPP row_shl swizzles src0 before the add."""
|
||||
instructions = [
|
||||
v_cvt_f32_u32_e32(v[0], v[255]),
|
||||
v_add_f32_e32(v[1], DPP, v[0], vsrc0=v[0], dpp=0x101, row_mask=0xf, bank_mask=0xf, bc=1),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=16)
|
||||
self.assertAlmostEqual(i2f(st.vgpr[0][1]), 1.0, places=5)
|
||||
self.assertAlmostEqual(i2f(st.vgpr[1][1]), 3.0, places=5)
|
||||
self.assertAlmostEqual(i2f(st.vgpr[14][1]), 29.0, places=5)
|
||||
|
||||
def test_v_fmac_f32(self):
|
||||
"""V_FMAC_F32: d = d + a*b using inline constants."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], 2.0),
|
||||
v_mov_b32_e32(v[1], 4.0),
|
||||
v_mov_b32_e32(v[2], 1.0),
|
||||
v_fmac_f32_e32(v[2], v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertAlmostEqual(i2f(st.vgpr[0][2]), 9.0, places=5)
|
||||
|
||||
def test_v_fmaak_f32(self):
|
||||
"""V_FMAAK_F32: d = a * b + K using inline constants."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], 2.0),
|
||||
v_mov_b32_e32(v[1], 4.0),
|
||||
v_fmaak_f32_e32(v[2], v[0], v[1], literal=0x3f800000),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertAlmostEqual(i2f(st.vgpr[0][2]), 9.0, places=5)
|
||||
|
||||
def test_v_fmamk_f32_basic(self):
|
||||
"""V_FMAMK_F32: d = a * K + b."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], 2.0),
|
||||
v_mov_b32_e32(v[1], 1.0),
|
||||
v_fmamk_f32_e32(v[2], v[0], v[1], literal=0x40800000),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertAlmostEqual(i2f(st.vgpr[0][2]), 9.0, places=5)
|
||||
|
||||
def test_v_fmamk_f32_small_constant(self):
|
||||
"""V_FMAMK_F32 with small constant."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], 4.0),
|
||||
v_mov_b32_e32(v[1], 1.0),
|
||||
v_fmamk_f32_e32(v[2], v[0], v[1], literal=f2i(0.5)),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertAlmostEqual(i2f(st.vgpr[0][2]), 3.0, places=5)
|
||||
|
||||
|
||||
class TestBitManipulation(unittest.TestCase):
|
||||
"""Tests for bit manipulation VOP2 instructions."""
|
||||
|
||||
def test_v_and_b32(self):
|
||||
"""V_AND_B32 bitwise and."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0xff),
|
||||
s_mov_b32(s[1], 0x0f),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_and_b32_e32(v[1], s[1], v[0]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][1], 0x0f)
|
||||
|
||||
def test_v_and_b32_quadrant(self):
|
||||
"""V_AND_B32 for quadrant extraction (n & 3)."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 15915),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_and_b32_e32(v[1], 3, v[0]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][1], 15915 & 3)
|
||||
|
||||
def test_v_lshrrev_b32(self):
|
||||
"""V_LSHRREV_B32 logical shift right."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0xff00),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_lshrrev_b32_e32(v[1], 8, v[0]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][1], 0xff)
|
||||
|
||||
def test_v_lshlrev_b32(self):
|
||||
"""V_LSHLREV_B32 logical shift left."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0xff),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_lshlrev_b32_e32(v[1], 8, v[0]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][1], 0xff00)
|
||||
|
||||
def test_v_xor_b32(self):
|
||||
"""V_XOR_B32 bitwise xor (used in sin for sign)."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x80000000),
|
||||
s_mov_b32(s[1], f2i(1.0)),
|
||||
v_mov_b32_e32(v[0], s[1]),
|
||||
v_xor_b32_e32(v[1], s[0], v[0]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertAlmostEqual(i2f(st.vgpr[0][1]), -1.0, places=5)
|
||||
|
||||
def test_v_xor_b32_sign_flip(self):
|
||||
"""V_XOR_B32 for sign flip pattern."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x80000000),
|
||||
v_mov_b32_e32(v[0], -2.0),
|
||||
v_xor_b32_e32(v[1], s[0], v[0]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertAlmostEqual(i2f(st.vgpr[0][1]), 2.0, places=5)
|
||||
|
||||
|
||||
class TestSpecialValues(unittest.TestCase):
|
||||
"""Tests for special float values - inf, nan, zero handling."""
|
||||
|
||||
def test_v_mul_f32_zero_times_inf(self):
|
||||
"""V_MUL_F32: 0 * inf = NaN."""
|
||||
import math
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], 0),
|
||||
s_mov_b32(s[0], 0x7f800000),
|
||||
v_mov_b32_e32(v[1], s[0]),
|
||||
v_mul_f32_e32(v[2], v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertTrue(math.isnan(i2f(st.vgpr[0][2])))
|
||||
|
||||
def test_v_add_f32_inf_minus_inf(self):
|
||||
"""V_ADD_F32: inf + (-inf) = NaN."""
|
||||
import math
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x7f800000),
|
||||
s_mov_b32(s[1], 0xff800000),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
v_add_f32_e32(v[2], v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertTrue(math.isnan(i2f(st.vgpr[0][2])))
|
||||
|
||||
|
||||
class TestF16Ops(unittest.TestCase):
|
||||
"""Tests for 16-bit VOP2 operations."""
|
||||
|
||||
def test_v_add_f16_basic(self):
|
||||
"""V_ADD_F16 adds two f16 values."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x3c00), # f16 1.0
|
||||
s_mov_b32(s[1], 0x4000), # f16 2.0
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
v_add_f16_e32(v[2], v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
result = st.vgpr[0][2] & 0xffff
|
||||
self.assertEqual(result, 0x4200, f"Expected 0x4200 (f16 3.0), got 0x{result:04x}")
|
||||
|
||||
def test_v_add_f16_negative(self):
|
||||
"""V_ADD_F16 with negative values."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x3c00), # f16 1.0
|
||||
s_mov_b32(s[1], 0xc000), # f16 -2.0
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
v_add_f16_e32(v[2], v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
result = st.vgpr[0][2] & 0xffff
|
||||
self.assertEqual(result, 0xbc00, f"Expected 0xbc00 (f16 -1.0), got 0x{result:04x}")
|
||||
|
||||
def test_v_mul_f16_basic(self):
|
||||
"""V_MUL_F16 multiplies two f16 values."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x4000), # f16 2.0
|
||||
s_mov_b32(s[1], 0x4200), # f16 3.0
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
v_mul_f16_e32(v[2], v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
result = st.vgpr[0][2] & 0xffff
|
||||
self.assertEqual(result, 0x4600, f"Expected 0x4600 (f16 6.0), got 0x{result:04x}")
|
||||
|
||||
def test_v_mul_f16_by_zero(self):
|
||||
"""V_MUL_F16 by zero."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x4000), # f16 2.0
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], 0),
|
||||
v_mul_f16_e32(v[2], v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
result = st.vgpr[0][2] & 0xffff
|
||||
self.assertEqual(result, 0x0000, f"Expected 0x0000 (f16 0.0), got 0x{result:04x}")
|
||||
|
||||
def test_v_fmac_f16_basic(self):
|
||||
"""V_FMAC_F16: d = d + a*b."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x4000), # f16 2.0
|
||||
s_mov_b32(s[1], 0x4200), # f16 3.0
|
||||
s_mov_b32(s[2], 0x3c00), # f16 1.0
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
v_mov_b32_e32(v[2], s[2]),
|
||||
v_fmac_f16_e32(v[2], v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
result = st.vgpr[0][2] & 0xffff
|
||||
# 2.0 * 3.0 + 1.0 = 7.0, f16 7.0 = 0x4700
|
||||
self.assertEqual(result, 0x4700, f"Expected 0x4700 (f16 7.0), got 0x{result:04x}")
|
||||
|
||||
def test_v_max_f16_basic(self):
|
||||
"""V_MAX_F16 returns the maximum of two f16 values."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x3c00), # f16 1.0
|
||||
s_mov_b32(s[1], 0x4000), # f16 2.0
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
v_max_f16_e32(v[2], v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
result = st.vgpr[0][2] & 0xffff
|
||||
self.assertEqual(result, 0x4000, f"Expected 0x4000 (f16 2.0), got 0x{result:04x}")
|
||||
|
||||
def test_v_min_f16_basic(self):
|
||||
"""V_MIN_F16 returns the minimum of two f16 values."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x3c00), # f16 1.0
|
||||
s_mov_b32(s[1], 0x4000), # f16 2.0
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
v_min_f16_e32(v[2], v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
result = st.vgpr[0][2] & 0xffff
|
||||
self.assertEqual(result, 0x3c00, f"Expected 0x3c00 (f16 1.0), got 0x{result:04x}")
|
||||
|
||||
def test_v_fmaak_f16_basic(self):
|
||||
"""V_FMAAK_F16: d = a * b + K."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x4000), # f16 2.0
|
||||
s_mov_b32(s[1], 0x4200), # f16 3.0
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
v_fmaak_f16_e32(v[2], v[0], v[1], literal=0x3c00), # + f16 1.0
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
result = st.vgpr[0][2] & 0xffff
|
||||
# 2.0 * 3.0 + 1.0 = 7.0, f16 7.0 = 0x4700
|
||||
self.assertEqual(result, 0x4700, f"Expected 0x4700 (f16 7.0), got 0x{result:04x}")
|
||||
|
||||
|
||||
class TestHiHalfOps(unittest.TestCase):
|
||||
"""Tests for VOP2 16-bit operations with hi-half operands."""
|
||||
|
||||
def test_v_add_f16_src0_hi_fold(self):
|
||||
"""V_ADD_F16 with src0 hi-half fold (same register, different halves)."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x40003c00), # lo=f16(1.0), hi=f16(2.0)
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
VOP3(VOP3Op.V_ADD_F16, vdst=v[1], src0=v[0], src1=v[0], opsel=0b0001),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
result = st.vgpr[0][1] & 0xffff
|
||||
self.assertEqual(result, 0x4200, f"Expected f16(3.0)=0x4200, got 0x{result:04x}")
|
||||
|
||||
def test_v_add_f16_src0_hi_different_reg(self):
|
||||
"""V_ADD_F16 with src0 hi-half from different register."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x40000000), # hi=f16(2.0), lo=0
|
||||
s_mov_b32(s[1], 0x00003c00), # hi=0, lo=f16(1.0)
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
VOP3(VOP3Op.V_ADD_F16, vdst=v[2], src0=v[0], src1=v[1], opsel=0b0001),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
result = st.vgpr[0][2] & 0xffff
|
||||
self.assertEqual(result, 0x4200, f"Expected f16(3.0)=0x4200, got 0x{result:04x}")
|
||||
|
||||
def test_v_mul_f16_src0_hi(self):
|
||||
"""V_MUL_F16 with src0 from high half."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x40000000), # hi=f16(2.0), lo=0
|
||||
s_mov_b32(s[1], 0x00004200), # hi=0, lo=f16(3.0)
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
VOP3(VOP3Op.V_MUL_F16, vdst=v[2], src0=v[0], src1=v[1], opsel=0b0001),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
result = st.vgpr[0][2] & 0xffff
|
||||
self.assertEqual(result, 0x4600, f"Expected f16(6.0)=0x4600, got 0x{result:04x}")
|
||||
|
||||
def test_v_mul_f16_hi_half(self):
|
||||
"""V_MUL_F16 reading from high half."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x40003c00), # lo=1.0, hi=2.0
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
VOP3(VOP3Op.V_MUL_F16, vdst=v[1], src0=v[0], src1=v[0], opsel=0b0011),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
result = st.vgpr[0][1] & 0xffff
|
||||
self.assertEqual(result, 0x4400, f"Expected f16(4.0)=0x4400, got 0x{result:04x}")
|
||||
|
||||
def test_v_fma_f16_hi_dest(self):
|
||||
"""V_FMA_F16 writing to high half with opsel.
|
||||
|
||||
Uses V_FMA_F16 (not V_FMAC_F16) because it has explicit src2 operand
|
||||
which makes opsel handling clearer.
|
||||
"""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x3c000000), # hi=f16(1.0), lo=0
|
||||
s_mov_b32(s[1], 0x4000), # f16(2.0) in lo
|
||||
s_mov_b32(s[2], 0x4200), # f16(3.0) in lo
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
v_mov_b32_e32(v[2], s[2]),
|
||||
# V_FMA_F16: dst = src0 * src1 + src2
|
||||
# opsel=0b1100: bit2=src2 hi, bit3=dst hi
|
||||
# So: v[0].hi = v[1].lo * v[2].lo + v[0].hi = 2.0 * 3.0 + 1.0 = 7.0
|
||||
VOP3(VOP3Op.V_FMA_F16, vdst=v[0], src0=v[1], src1=v[2], src2=v[0], opsel=0b1100),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
hi = (st.vgpr[0][0] >> 16) & 0xffff
|
||||
# 2.0 * 3.0 + 1.0 = 7.0, f16 7.0 = 0x4700
|
||||
self.assertEqual(hi, 0x4700, f"Expected f16(7.0)=0x4700 in hi, got 0x{hi:04x}")
|
||||
|
||||
def test_v_add_f16_multilane(self):
|
||||
"""V_ADD_F16 with multiple lanes."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x3c00), # f16 1.0
|
||||
s_mov_b32(s[1], 0x4000), # f16 2.0
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
v_add_f16_e32(v[2], v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=4)
|
||||
for lane in range(4):
|
||||
result = st.vgpr[lane][2] & 0xffff
|
||||
self.assertEqual(result, 0x4200, f"Lane {lane}: expected 0x4200, got 0x{result:04x}")
|
||||
|
||||
|
||||
class TestVop2F16HiHalf(unittest.TestCase):
|
||||
"""Regression tests for VOP2 f16 hi-half operand handling.
|
||||
|
||||
These test the bugs where:
|
||||
1. VOP2 vsrc1 >= 384 (v[128]+) wasn't extracting hi 16 bits
|
||||
2. VOP2 vdst >= 384 (v[128]+) wasn't preserving lo 16 bits
|
||||
"""
|
||||
|
||||
def test_v_add_f16_e32_vsrc1_hi_half(self):
|
||||
"""V_ADD_F16_E32 with vsrc1 from hi-half (v[128]+).
|
||||
|
||||
When vsrc1 >= 384 (representing v[128]+), the hardware reads from the hi 16 bits
|
||||
of v[vsrc1-128]. The emulator must extract bits [31:16] from the actual VGPR.
|
||||
|
||||
Regression test for: VOP2 f16 vsrc1 hi-half extraction bug.
|
||||
"""
|
||||
instructions = [
|
||||
# v[0] = 0x4000_3c00: hi=f16(2.0), lo=f16(1.0)
|
||||
s_mov_b32(s[0], 0x40003c00),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
# v_add_f16_e32 v[1], v[0], v[128] (vsrc1=v[128] reads hi of v[0])
|
||||
# In VOP2 encoding, vsrc1=384 means v[128], which maps to v[0].hi
|
||||
# v[1] = v[0].lo + v[0].hi = 1.0 + 2.0 = 3.0
|
||||
VOP2(VOP2Op.V_ADD_F16, vdst=v[1], src0=v[0], vsrc1=v[128]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
result = st.vgpr[0][1] & 0xffff
|
||||
# 1.0 + 2.0 = 3.0, f16 3.0 = 0x4200
|
||||
self.assertEqual(result, 0x4200, f"Expected f16(3.0)=0x4200, got 0x{result:04x}")
|
||||
|
||||
def test_v_mul_f16_e32_vsrc1_hi_half(self):
|
||||
"""V_MUL_F16_E32 with vsrc1 from hi-half.
|
||||
|
||||
Regression test for: VOP2 f16 vsrc1 hi-half extraction bug.
|
||||
"""
|
||||
instructions = [
|
||||
# v[0] = 0x4200_4000: hi=f16(3.0), lo=f16(2.0)
|
||||
s_mov_b32(s[0], 0x42004000),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
# v_mul_f16_e32 v[1], v[0], v[128] (vsrc1=v[128] reads hi of v[0])
|
||||
# v[1] = v[0].lo * v[0].hi = 2.0 * 3.0 = 6.0
|
||||
VOP2(VOP2Op.V_MUL_F16, vdst=v[1], src0=v[0], vsrc1=v[128]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
result = st.vgpr[0][1] & 0xffff
|
||||
# 2.0 * 3.0 = 6.0, f16 6.0 = 0x4600
|
||||
self.assertEqual(result, 0x4600, f"Expected f16(6.0)=0x4600, got 0x{result:04x}")
|
||||
|
||||
def test_v_add_f16_e32_vdst_hi_half(self):
|
||||
"""V_ADD_F16_E32 writing to hi-half destination (v[128]+).
|
||||
|
||||
When vdst >= 384 (representing v[128]+), the hardware writes to bits [31:16]
|
||||
of v[vdst-128] while preserving bits [15:0]. The emulator must merge the result.
|
||||
|
||||
Regression test for: VOP2 f16 vdst hi-half write bug.
|
||||
"""
|
||||
instructions = [
|
||||
# v[0] = 0x0000_BEEF: lo has marker value
|
||||
s_mov_b32(s[0], 0x0000BEEF),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
# v[1] = f16(1.0), v[2] = f16(2.0)
|
||||
s_mov_b32(s[1], 0x3c00),
|
||||
s_mov_b32(s[2], 0x4000),
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
v_mov_b32_e32(v[2], s[2]),
|
||||
# v_add_f16_e32 v[128], v[1], v[2] (vdst=v[128] writes hi of v[0])
|
||||
# v[0].hi = 1.0 + 2.0 = 3.0, v[0].lo preserved = 0xBEEF
|
||||
VOP2(VOP2Op.V_ADD_F16, vdst=v[128], src0=v[1], vsrc1=v[2]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
hi = (st.vgpr[0][0] >> 16) & 0xffff
|
||||
lo = st.vgpr[0][0] & 0xffff
|
||||
# hi = 3.0 = 0x4200, lo preserved = 0xBEEF
|
||||
self.assertEqual(hi, 0x4200, f"Expected hi=f16(3.0)=0x4200, got 0x{hi:04x}")
|
||||
self.assertEqual(lo, 0xBEEF, f"Expected lo preserved=0xBEEF, got 0x{lo:04x}")
|
||||
|
||||
def test_v_mul_f16_e32_vdst_hi_half(self):
|
||||
"""V_MUL_F16_E32 writing to hi-half destination.
|
||||
|
||||
Regression test for: VOP2 f16 vdst hi-half write bug.
|
||||
"""
|
||||
instructions = [
|
||||
# v[0] = 0x0000_DEAD: lo has marker value
|
||||
s_mov_b32(s[0], 0x0000DEAD),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
# v[1] = f16(2.0), v[2] = f16(4.0)
|
||||
s_mov_b32(s[1], 0x4000),
|
||||
s_mov_b32(s[2], 0x4400),
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
v_mov_b32_e32(v[2], s[2]),
|
||||
# v_mul_f16_e32 v[128], v[1], v[2] (vdst=v[128] writes hi of v[0])
|
||||
# v[0].hi = 2.0 * 4.0 = 8.0, v[0].lo preserved = 0xDEAD
|
||||
VOP2(VOP2Op.V_MUL_F16, vdst=v[128], src0=v[1], vsrc1=v[2]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
hi = (st.vgpr[0][0] >> 16) & 0xffff
|
||||
lo = st.vgpr[0][0] & 0xffff
|
||||
# hi = 8.0 = 0x4800, lo preserved = 0xDEAD
|
||||
self.assertEqual(hi, 0x4800, f"Expected hi=f16(8.0)=0x4800, got 0x{hi:04x}")
|
||||
self.assertEqual(lo, 0xDEAD, f"Expected lo preserved=0xDEAD, got 0x{lo:04x}")
|
||||
|
||||
def test_v_add_f16_e32_both_hi_half(self):
|
||||
"""V_ADD_F16_E32 with both vsrc1 and vdst as hi-half (different underlying regs).
|
||||
|
||||
Tests the combination of both fixes: reading vsrc1 from hi-half AND
|
||||
writing result to hi-half destination, using different underlying VGPRs.
|
||||
|
||||
Regression test for: VOP2 f16 hi-half bugs (combined).
|
||||
"""
|
||||
instructions = [
|
||||
# v[0] = 0x4000_xxxx: hi=f16(2.0) for vsrc1
|
||||
s_mov_b32(s[0], 0x40000000),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
# v[1] = 0x0000_3c00: lo=f16(1.0) for src0
|
||||
s_mov_b32(s[1], 0x00003c00),
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
# v[2] = 0x0000_CAFE: lo=marker for vdst preservation
|
||||
s_mov_b32(s[2], 0x0000CAFE),
|
||||
v_mov_b32_e32(v[2], s[2]),
|
||||
# v_add_f16_e32 v[130], v[1], v[128]
|
||||
# src0 = v[1].lo = 1.0
|
||||
# vsrc1 = v[128] reads v[0].hi = 2.0
|
||||
# result = 1.0 + 2.0 = 3.0
|
||||
# vdst = v[130] writes to v[2].hi, preserving v[2].lo
|
||||
VOP2(VOP2Op.V_ADD_F16, vdst=v[130], src0=v[1], vsrc1=v[128]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
hi = (st.vgpr[0][2] >> 16) & 0xffff
|
||||
lo = st.vgpr[0][2] & 0xffff
|
||||
# hi = 3.0 = 0x4200, lo preserved = 0xCAFE
|
||||
self.assertEqual(hi, 0x4200, f"Expected hi=f16(3.0)=0x4200, got 0x{hi:04x}")
|
||||
self.assertEqual(lo, 0xCAFE, f"Expected lo preserved=0xCAFE, got 0x{lo:04x}")
|
||||
|
||||
def test_v_fmac_f16_e32_vsrc1_hi_half(self):
|
||||
"""V_FMAC_F16_E32 with vsrc1 from hi-half.
|
||||
|
||||
V_FMAC_F16: vdst = vdst + src0 * vsrc1
|
||||
|
||||
Regression test for: VOP2 f16 vsrc1 hi-half extraction bug.
|
||||
"""
|
||||
instructions = [
|
||||
# v[0] = 0x4000_3c00: hi=f16(2.0), lo=f16(1.0)
|
||||
s_mov_b32(s[0], 0x40003c00),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
# v[1] = f16(3.0) = 0x4200
|
||||
s_mov_b32(s[1], 0x4200),
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
# v_fmac_f16_e32 v[1], v[0], v[128]
|
||||
# vdst = v[1] = 3.0 + v[0].lo * v[0].hi = 3.0 + 1.0 * 2.0 = 5.0
|
||||
VOP2(VOP2Op.V_FMAC_F16, vdst=v[1], src0=v[0], vsrc1=v[128]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
result = st.vgpr[0][1] & 0xffff
|
||||
# 3.0 + 1.0 * 2.0 = 5.0, f16 5.0 = 0x4500
|
||||
self.assertEqual(result, 0x4500, f"Expected f16(5.0)=0x4500, got 0x{result:04x}")
|
||||
|
||||
def test_v_fmac_f16_e32_vdst_hi_half(self):
|
||||
"""V_FMAC_F16_E32 writing to hi-half destination.
|
||||
|
||||
V_FMAC_F16: vdst.h = vdst.h + src0 * vsrc1
|
||||
|
||||
When vdst is v[128]+, the accumulator D0 must also read from the hi-half.
|
||||
This tests the bug where D0 was read from lo-half instead of hi-half.
|
||||
|
||||
Regression test for: VOP2 FMAC hi-half D0 accumulator read bug.
|
||||
"""
|
||||
instructions = [
|
||||
# v[0] = 0x3800_DEAD: hi=f16(0.5), lo=marker (0xDEAD)
|
||||
s_mov_b32(s[0], 0x3800DEAD),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
# v[1] = f16(2.0) = 0x4000
|
||||
s_mov_b32(s[1], 0x4000),
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
# v[2] = f16(3.0) = 0x4200
|
||||
s_mov_b32(s[2], 0x4200),
|
||||
v_mov_b32_e32(v[2], s[2]),
|
||||
# v_fmac_f16_e32 v[128], v[1], v[2]
|
||||
# vdst = v[128] means v[0].hi
|
||||
# D0 = v[0].hi = 0.5
|
||||
# result = D0 + src0 * vsrc1 = 0.5 + 2.0 * 3.0 = 6.5
|
||||
# v[0].hi = 6.5, v[0].lo preserved = 0xDEAD
|
||||
VOP2(VOP2Op.V_FMAC_F16, vdst=v[128], src0=v[1], vsrc1=v[2]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
hi = (st.vgpr[0][0] >> 16) & 0xffff
|
||||
lo = st.vgpr[0][0] & 0xffff
|
||||
# hi = 6.5 = 0x4680, lo preserved = 0xDEAD
|
||||
self.assertEqual(hi, 0x4680, f"Expected hi=f16(6.5)=0x4680, got 0x{hi:04x}")
|
||||
self.assertEqual(lo, 0xDEAD, f"Expected lo preserved=0xDEAD, got 0x{lo:04x}")
|
||||
|
||||
def test_v_mul_f16_e32_src0_hi_half(self):
|
||||
"""V_MUL_F16_E32 with src0 from hi-half (src0 >= v[128]).
|
||||
|
||||
When src0 >= 384 (representing v[128]+), the hardware reads from the hi 16 bits
|
||||
of v[src0-128]. The emulator must extract bits [31:16] from the actual VGPR.
|
||||
|
||||
Regression test for: VOP2 f16 src0 hi-half extraction bug.
|
||||
"""
|
||||
instructions = [
|
||||
# v[0] = 0x4000_3c00: hi=f16(2.0), lo=f16(1.0)
|
||||
s_mov_b32(s[0], 0x40003c00),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
# v[1] = f16(3.0) = 0x4200
|
||||
s_mov_b32(s[1], 0x4200),
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
# v_mul_f16_e32 v[2], v[128], v[1]
|
||||
# src0 = v[128] reads from v[0].hi = 2.0
|
||||
# result = 2.0 * 3.0 = 6.0
|
||||
VOP2(VOP2Op.V_MUL_F16, vdst=v[2], src0=v[128], vsrc1=v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
result = st.vgpr[0][2] & 0xffff
|
||||
# 2.0 * 3.0 = 6.0, f16 6.0 = 0x4600
|
||||
self.assertEqual(result, 0x4600, f"Expected f16(6.0)=0x4600, got 0x{result:04x}")
|
||||
|
||||
def test_v_add_f16_e32_src0_hi_half(self):
|
||||
"""V_ADD_F16_E32 with src0 from hi-half (src0 >= v[128]).
|
||||
|
||||
Regression test for: VOP2 f16 src0 hi-half extraction bug.
|
||||
"""
|
||||
instructions = [
|
||||
# v[0] = 0x4000_3c00: hi=f16(2.0), lo=f16(1.0)
|
||||
s_mov_b32(s[0], 0x40003c00),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
# v[1] = f16(5.0) = 0x4500
|
||||
s_mov_b32(s[1], 0x4500),
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
# v_add_f16_e32 v[2], v[128], v[1]
|
||||
# src0 = v[128] reads from v[0].hi = 2.0
|
||||
# result = 2.0 + 5.0 = 7.0
|
||||
VOP2(VOP2Op.V_ADD_F16, vdst=v[2], src0=v[128], vsrc1=v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
result = st.vgpr[0][2] & 0xffff
|
||||
# 2.0 + 5.0 = 7.0, f16 7.0 = 0x4700
|
||||
self.assertEqual(result, 0x4700, f"Expected f16(7.0)=0x4700, got 0x{result:04x}")
|
||||
|
||||
|
||||
class TestF16InlineConstants(unittest.TestCase):
|
||||
"""Regression tests for VOP2 F16 inline float constants.
|
||||
|
||||
For 16-bit VOP2 operations (v_add_f16, v_mul_f16, etc.), inline float constants
|
||||
like 1.0, 2.0 must use F16 encoding (0x3c00, 0x4000) not F32 encoding (0x3f800000).
|
||||
|
||||
The emulator's rsrc() function needs bits=16 to select F16_INLINE constants.
|
||||
|
||||
Regression test for: VOP2 16-bit inline constant using F32 instead of F16.
|
||||
"""
|
||||
|
||||
def test_v_add_f16_inline_constant_1_0(self):
|
||||
"""V_ADD_F16_E32 with inline constant 1.0 should use F16 encoding."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x3c00), # f16 1.0
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
# v_add_f16_e32 v[1], 1.0, v[0] -- 1.0 must be F16 0x3c00, not F32 0x3f800000
|
||||
v_add_f16_e32(v[1], 1.0, v[0]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
result = st.vgpr[0][1] & 0xFFFF
|
||||
# 1.0 + 1.0 = 2.0, f16 2.0 = 0x4000
|
||||
self.assertEqual(result, 0x4000, f"Expected f16(2.0)=0x4000, got 0x{result:04x}")
|
||||
|
||||
def test_v_add_f16_inline_constant_2_0(self):
|
||||
"""V_ADD_F16_E32 with inline constant 2.0."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x4200), # f16 3.0
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_add_f16_e32(v[1], 2.0, v[0]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
result = st.vgpr[0][1] & 0xFFFF
|
||||
# 2.0 + 3.0 = 5.0, f16 5.0 = 0x4500
|
||||
self.assertEqual(result, 0x4500, f"Expected f16(5.0)=0x4500, got 0x{result:04x}")
|
||||
|
||||
def test_v_mul_f16_inline_constant(self):
|
||||
"""V_MUL_F16_E32 with inline constant 2.0."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x4200), # f16 3.0
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mul_f16_e32(v[1], 2.0, v[0]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
result = st.vgpr[0][1] & 0xFFFF
|
||||
# 2.0 * 3.0 = 6.0, f16 6.0 = 0x4600
|
||||
self.assertEqual(result, 0x4600, f"Expected f16(6.0)=0x4600, got 0x{result:04x}")
|
||||
|
||||
|
||||
class TestCndmask(unittest.TestCase):
|
||||
"""Tests for V_CNDMASK_B32 and V_CNDMASK_B16."""
|
||||
|
||||
def test_v_cndmask_b16_select_src0(self):
|
||||
"""V_CNDMASK_B16 selects src0 when VCC bit is 0."""
|
||||
instructions = [
|
||||
s_mov_b32(VCC_LO, 0), # VCC = 0
|
||||
s_mov_b32(s[0], 0x3c00), # f16 1.0
|
||||
s_mov_b32(s[1], 0x4000), # f16 2.0
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
v_cndmask_b16(v[2], v[0], v[1], VCC),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
result = st.vgpr[0][2] & 0xffff
|
||||
self.assertEqual(result, 0x3c00, f"Expected src0=0x3c00, got 0x{result:04x}")
|
||||
|
||||
def test_v_cndmask_b16_select_src1(self):
|
||||
"""V_CNDMASK_B16 selects src1 when VCC bit is 1."""
|
||||
instructions = [
|
||||
s_mov_b32(VCC_LO, 1), # VCC = 1
|
||||
s_mov_b32(s[0], 0x3c00), # f16 1.0
|
||||
s_mov_b32(s[1], 0x4000), # f16 2.0
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
v_cndmask_b16(v[2], v[0], v[1], VCC),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
result = st.vgpr[0][2] & 0xffff
|
||||
self.assertEqual(result, 0x4000, f"Expected src1=0x4000, got 0x{result:04x}")
|
||||
|
||||
def test_v_cndmask_b16_write_hi(self):
|
||||
"""V_CNDMASK_B16 can write to high 16 bits with opsel."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x3c003800), # src0: hi=1.0, lo=0.5
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
s_mov_b32(s[1], 0x4000c000), # src1: hi=2.0, lo=-2.0
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
s_mov_b32(s[2], 0xDEAD0000), # v2 initial: hi=0xDEAD, lo=0
|
||||
v_mov_b32_e32(v[2], s[2]),
|
||||
s_mov_b32(VCC_LO, 0), # vcc = 0, select src0
|
||||
# opsel=0b1011: bit0=src0 hi, bit1=src1 hi, bit3=dst hi
|
||||
VOP3(VOP3Op.V_CNDMASK_B16, vdst=v[2], src0=v[0], src1=v[1], src2=SrcEnum.VCC_LO, opsel=0b1011),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
hi = (st.vgpr[0][2] >> 16) & 0xffff
|
||||
lo = st.vgpr[0][2] & 0xffff
|
||||
# vcc=0 selects src0.h = 1.0 = 0x3c00, writes to hi
|
||||
self.assertEqual(hi, 0x3c00, f"Expected hi=0x3c00 (1.0), got 0x{hi:04x}")
|
||||
self.assertEqual(lo, 0x0000, f"Expected lo preserved as 0, got 0x{lo:04x}")
|
||||
|
||||
|
||||
class TestSpecialFloatValues(unittest.TestCase):
|
||||
"""Tests for special float value handling in VOP2 instructions."""
|
||||
|
||||
def test_neg_zero_add(self):
|
||||
"""-0.0 + 0.0 = +0.0 (IEEE 754)."""
|
||||
neg_zero = 0x80000000
|
||||
instructions = [
|
||||
s_mov_b32(s[0], neg_zero),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_add_f32_e32(v[1], 0.0, v[0]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][1], 0x00000000, "Should be +0.0")
|
||||
|
||||
def test_neg_zero_mul(self):
|
||||
"""-0.0 * -1.0 = +0.0."""
|
||||
neg_zero = 0x80000000
|
||||
instructions = [
|
||||
s_mov_b32(s[0], neg_zero),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mul_f32_e32(v[1], -1.0, v[0]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][1], 0x00000000, "Should be +0.0")
|
||||
|
||||
def test_inf_minus_inf(self):
|
||||
"""+inf - inf = NaN."""
|
||||
import math
|
||||
pos_inf = 0x7f800000
|
||||
neg_inf = 0xff800000
|
||||
instructions = [
|
||||
s_mov_b32(s[0], pos_inf),
|
||||
s_mov_b32(s[1], neg_inf),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
v_sub_f32_e32(v[2], v[0], v[1]), # inf - (-inf) = inf
|
||||
v_add_f32_e32(v[3], v[0], v[1]), # inf + (-inf) = NaN
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], pos_inf, "inf - (-inf) = inf")
|
||||
self.assertTrue(math.isnan(i2f(st.vgpr[0][3])), "inf + (-inf) = NaN")
|
||||
|
||||
def test_denormal_f32_mul_ftz(self):
|
||||
"""Denormal * normal - RDNA3 flushes denormals to zero (FTZ mode)."""
|
||||
smallest_denorm = 0x00000001 # Smallest positive denormal
|
||||
instructions = [
|
||||
s_mov_b32(s[0], smallest_denorm),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mul_f32_e32(v[1], 2.0, v[0]), # Denormal input gets flushed to 0
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][1], 0x00000000)
|
||||
|
||||
|
||||
class TestCarryOps(unittest.TestCase):
|
||||
"""Tests for VOP2 carry instructions (v_add_co_ci_u32, v_sub_co_ci_u32, v_subrev_co_ci_u32)."""
|
||||
|
||||
def test_v_subrev_co_ci_u32_no_borrow(self):
|
||||
"""V_SUBREV_CO_CI_U32: D0 = S1 - S0 - VCC_IN, when VCC_IN=0."""
|
||||
instructions = [
|
||||
s_mov_b32(VCC_LO, 0), # VCC = 0 (no borrow in)
|
||||
v_mov_b32_e32(v[0], 5), # S0 = 5
|
||||
v_mov_b32_e32(v[1], 10), # S1 = 10
|
||||
v_subrev_co_ci_u32_e32(v[2], v[0], v[1]), # D0 = 10 - 5 - 0 = 5
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 5)
|
||||
self.assertEqual(st.vcc, 0) # No borrow out
|
||||
|
||||
def test_v_subrev_co_ci_u32_with_borrow(self):
|
||||
"""V_SUBREV_CO_CI_U32: D0 = S1 - S0 - VCC_IN, when VCC_IN=1."""
|
||||
instructions = [
|
||||
s_mov_b32(VCC_LO, 1), # VCC = 1 (borrow in)
|
||||
v_mov_b32_e32(v[0], 5), # S0 = 5
|
||||
v_mov_b32_e32(v[1], 10), # S1 = 10
|
||||
v_subrev_co_ci_u32_e32(v[2], v[0], v[1]), # D0 = 10 - 5 - 1 = 4
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 4)
|
||||
self.assertEqual(st.vcc, 0) # No borrow out
|
||||
|
||||
def test_v_subrev_co_ci_u32_generates_borrow(self):
|
||||
"""V_SUBREV_CO_CI_U32: generates borrow when S0 + VCC_IN > S1."""
|
||||
instructions = [
|
||||
s_mov_b32(VCC_LO, 0), # VCC = 0
|
||||
v_mov_b32_e32(v[0], 10), # S0 = 10
|
||||
v_mov_b32_e32(v[1], 5), # S1 = 5
|
||||
v_subrev_co_ci_u32_e32(v[2], v[0], v[1]), # D0 = 5 - 10 - 0 = -5 (underflow)
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 0xFFFFFFFB) # -5 as unsigned
|
||||
self.assertEqual(st.vcc, 1) # Borrow out
|
||||
|
||||
def test_v_add_co_ci_u32_no_carry(self):
|
||||
"""V_ADD_CO_CI_U32: D0 = S0 + S1 + VCC_IN, when VCC_IN=0."""
|
||||
instructions = [
|
||||
s_mov_b32(VCC_LO, 0), # VCC = 0 (no carry in)
|
||||
v_mov_b32_e32(v[0], 5), # S0 = 5
|
||||
v_mov_b32_e32(v[1], 10), # S1 = 10
|
||||
v_add_co_ci_u32_e32(v[2], v[0], v[1]), # D0 = 5 + 10 + 0 = 15
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 15)
|
||||
self.assertEqual(st.vcc, 0) # No carry out
|
||||
|
||||
def test_v_add_co_ci_u32_with_carry(self):
|
||||
"""V_ADD_CO_CI_U32: D0 = S0 + S1 + VCC_IN, when VCC_IN=1."""
|
||||
instructions = [
|
||||
s_mov_b32(VCC_LO, 1), # VCC = 1 (carry in)
|
||||
v_mov_b32_e32(v[0], 5), # S0 = 5
|
||||
v_mov_b32_e32(v[1], 10), # S1 = 10
|
||||
v_add_co_ci_u32_e32(v[2], v[0], v[1]), # D0 = 5 + 10 + 1 = 16
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 16)
|
||||
self.assertEqual(st.vcc, 0) # No carry out
|
||||
|
||||
def test_v_add_co_ci_u32_generates_carry(self):
|
||||
"""V_ADD_CO_CI_U32: generates carry when overflow occurs."""
|
||||
instructions = [
|
||||
s_mov_b32(VCC_LO, 1), # VCC = 1 (carry in)
|
||||
s_mov_b32(s[0], 0xFFFFFFFF), # max u32
|
||||
v_mov_b32_e32(v[0], s[0]), # S0 = 0xFFFFFFFF
|
||||
v_mov_b32_e32(v[1], 0), # S1 = 0
|
||||
v_add_co_ci_u32_e32(v[2], v[0], v[1]), # D0 = 0xFFFFFFFF + 0 + 1 = 0 (overflow)
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 0) # Overflowed to 0
|
||||
self.assertEqual(st.vcc, 1) # Carry out
|
||||
|
||||
def test_v_add_co_ci_u32_clears_carry(self):
|
||||
"""V_ADD_CO_CI_U32: VCC must be updated even when no carry is generated.
|
||||
|
||||
This tests the case where VCC=1 going in (carry-in consumed) but the addition
|
||||
does not overflow, so VCC must be cleared to 0.
|
||||
|
||||
Regression test for: VCC not being written by v_add_co_ci_u32_e32.
|
||||
"""
|
||||
instructions = [
|
||||
s_mov_b32(VCC_LO, 1), # VCC = 1 (carry in)
|
||||
v_mov_b32_e32(v[0], 1), # S0 = 1
|
||||
v_mov_b32_e32(v[1], 1), # S1 = 1
|
||||
v_add_co_ci_u32_e32(v[2], v[0], v[1]), # D0 = 1 + 1 + 1 = 3 (no overflow)
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 3) # 1 + 1 + 1 = 3
|
||||
self.assertEqual(st.vcc, 0) # No carry out - VCC must be cleared
|
||||
|
||||
def test_v_add_co_ci_u32_multilane_clears_vcc(self):
|
||||
"""V_ADD_CO_CI_U32 with multiple lanes: VCC bits must be updated per-lane.
|
||||
|
||||
When VCC has multiple bits set (one per active lane), and the addition doesn't
|
||||
overflow for any lane, all VCC bits must be cleared.
|
||||
|
||||
Regression test for: VCC not being written by v_add_co_ci_u32_e32 in multi-lane case.
|
||||
"""
|
||||
instructions = [
|
||||
s_mov_b32(VCC_LO, 0b11), # VCC = 0b11 (lanes 0,1 have carry-in)
|
||||
v_mov_b32_e32(v[0], 1), # S0 = 1 for all lanes
|
||||
v_mov_b32_e32(v[1], 1), # S1 = 1 for all lanes
|
||||
v_add_co_ci_u32_e32(v[2], v[0], v[1]), # D0 = 1 + 1 + 1 = 3 (no overflow)
|
||||
]
|
||||
st = run_program(instructions, n_lanes=2)
|
||||
self.assertEqual(st.vgpr[0][2], 3) # lane 0: 1 + 1 + 1 = 3
|
||||
self.assertEqual(st.vgpr[1][2], 3) # lane 1: 1 + 1 + 1 = 3
|
||||
self.assertEqual(st.vcc, 0) # No carry out for any lane - all VCC bits must be cleared
|
||||
|
||||
def test_v_add_co_ci_u32_preserves_inactive_vcc_bits(self):
|
||||
"""V_ADD_CO_CI_U32: VCC carry-out overwrites entire VCC register.
|
||||
|
||||
VOP2 carry instructions write ALL VCC bits based on carry-out, clearing
|
||||
bits for lanes that don't overflow regardless of EXEC mask.
|
||||
|
||||
Note: This differs from VOPC which only writes active lane bits.
|
||||
"""
|
||||
instructions = [
|
||||
s_mov_b32(VCC_LO, 0x00010000), # VCC bit 16 set
|
||||
v_mov_b32_e32(v[0], 1), # S0 = 1
|
||||
v_mov_b32_e32(v[1], 1), # S1 = 1
|
||||
v_add_co_ci_u32_e32(v[2], v[0], v[1]), # D0 = 1 + 1 + 0 = 2 (no carry)
|
||||
]
|
||||
st = run_program(instructions, n_lanes=4)
|
||||
self.assertEqual(st.vgpr[0][2], 2) # lane 0: 1 + 1 + 0 = 2
|
||||
# VCC should be completely cleared (all lanes have no carry-out)
|
||||
self.assertEqual(st.vcc, 0)
|
||||
|
||||
def test_v_add_co_ci_u32_all_lanes_same_result(self):
|
||||
"""V_ADD_CO_CI_U32: all active lanes should produce the same result.
|
||||
|
||||
When the same constant inputs are used across all lanes, each lane should
|
||||
compute the same result and write to its own VGPR slot.
|
||||
|
||||
Regression test for: VGPR writes not happening for all lanes.
|
||||
"""
|
||||
instructions = [
|
||||
s_mov_b32(VCC_LO, 0), # No carry-in
|
||||
v_mov_b32_e32(v[0], 3), # inline constant 3
|
||||
v_mov_b32_e32(v[1], 5), # value 5
|
||||
v_add_co_ci_u32_e32(v[1], 3, v[1]), # v[1] = 3 + v[1] + 0 = 3 + 5 = 8
|
||||
]
|
||||
st = run_program(instructions, n_lanes=4)
|
||||
# All 4 lanes should have v[1] = 8
|
||||
for lane in range(4):
|
||||
self.assertEqual(st.vgpr[lane][1], 8, f"lane {lane} should have v[1]=8")
|
||||
|
||||
def test_v_sub_co_ci_u32_no_borrow(self):
|
||||
"""V_SUB_CO_CI_U32: D0 = S0 - S1 - VCC_IN, when VCC_IN=0."""
|
||||
instructions = [
|
||||
s_mov_b32(VCC_LO, 0), # VCC = 0 (no borrow in)
|
||||
v_mov_b32_e32(v[0], 10), # S0 = 10
|
||||
v_mov_b32_e32(v[1], 5), # S1 = 5
|
||||
v_sub_co_ci_u32_e32(v[2], v[0], v[1]), # D0 = 10 - 5 - 0 = 5
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 5)
|
||||
self.assertEqual(st.vcc, 0) # No borrow out
|
||||
|
||||
def test_v_sub_co_ci_u32_vop3sd_separate_carry_regs(self):
|
||||
"""VOP3SD V_SUB_CO_CI_U32: carry-in from src2, carry-out to sdst (separate registers).
|
||||
|
||||
This tests the VOP3SD encoding where src2 specifies the carry-in register
|
||||
independently from sdst (carry-out). The bug was reading carry-in from sdst
|
||||
instead of src2.
|
||||
|
||||
Computation: D0 = S0 - S1 - carry_in = 0 - 0 - 1 = -1 = 0xFFFFFFFF
|
||||
"""
|
||||
instructions = [
|
||||
s_mov_b32(s[6], 1), # carry-in = 1 (in s[6])
|
||||
s_mov_b32(s[10], 0), # carry-out dest = 0 initially (in s[10])
|
||||
# VOP3SD: v_sub_co_ci_u32(vdst, sdst, src0, src1, src2)
|
||||
# src2 is carry-in (s[6]=1), sdst is carry-out (s[10])
|
||||
v_sub_co_ci_u32(v[0], s[10], 0, 0, s[6]), # D0 = 0 - 0 - 1 = -1
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][0], 0xFFFFFFFF) # -1 as unsigned
|
||||
self.assertEqual(st.sgpr[10], 1) # Borrow out to s[10]
|
||||
|
||||
def test_v_add_co_ci_u32_vop3sd_separate_carry_regs(self):
|
||||
"""VOP3SD V_ADD_CO_CI_U32: carry-in from src2, carry-out to sdst (separate registers).
|
||||
|
||||
This tests the VOP3SD encoding where src2 specifies the carry-in register
|
||||
independently from sdst (carry-out).
|
||||
|
||||
Computation: D0 = S0 + S1 + carry_in = 5 + 10 + 1 = 16
|
||||
"""
|
||||
instructions = [
|
||||
s_mov_b32(s[6], 1), # carry-in = 1 (in s[6])
|
||||
s_mov_b32(s[10], 0), # carry-out dest = 0 initially (in s[10])
|
||||
# VOP3SD: v_add_co_ci_u32(vdst, sdst, src0, src1, src2)
|
||||
v_add_co_ci_u32(v[0], s[10], 5, 10, s[6]), # D0 = 5 + 10 + 1 = 16
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][0], 16)
|
||||
self.assertEqual(st.sgpr[10], 0) # No carry out
|
||||
|
||||
def test_v_add_co_ci_u32_vop3sd_null_sdst(self):
|
||||
"""VOP3SD V_ADD_CO_CI_U32 with sdst=NULL: carry output is discarded.
|
||||
|
||||
When sdst=NULL (register 124), the carry-out should NOT be written anywhere.
|
||||
We verify this by checking that VCC (which we set to a sentinel value) is unchanged.
|
||||
"""
|
||||
instructions = [
|
||||
s_mov_b32(VCC_LO, 0xDEADBEEF), # Sentinel value in VCC
|
||||
s_mov_b32(s[6], 0), # carry-in = 0
|
||||
# VOP3SD with NULL sdst: carry-out should be discarded
|
||||
# Uses 0xFFFFFFFF + 1 + 0 = 0 with carry-out=1, but carry should not be written
|
||||
v_add_co_ci_u32(v[0], NULL, 0xFFFFFFFF, 1, s[6]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][0], 0) # 0xFFFFFFFF + 1 + 0 = 0 (overflow)
|
||||
self.assertEqual(st.vcc, 0xDEADBEEF) # VCC unchanged - carry was discarded
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
3656
tinygrad_repo/test/amd/hw/test_vop3.py
Normal file
3656
tinygrad_repo/test/amd/hw/test_vop3.py
Normal file
File diff suppressed because it is too large
Load Diff
1036
tinygrad_repo/test/amd/hw/test_vop3p.py
Normal file
1036
tinygrad_repo/test/amd/hw/test_vop3p.py
Normal file
File diff suppressed because it is too large
Load Diff
964
tinygrad_repo/test/amd/hw/test_vopc.py
Normal file
964
tinygrad_repo/test/amd/hw/test_vopc.py
Normal file
@@ -0,0 +1,964 @@
|
||||
"""Tests for VOPC instructions - vector compare operations.
|
||||
|
||||
Includes: v_cmp_class_f32, v_cmp_class_f16, v_cmp_eq_*, v_cmp_lt_*, v_cmp_gt_*
|
||||
"""
|
||||
import unittest
|
||||
from test.amd.hw.helpers import *
|
||||
|
||||
VCC = 106 # SGPR index for VCC_LO
|
||||
|
||||
class TestCmpClass(unittest.TestCase):
|
||||
"""Tests for V_CMP_CLASS_F32 float classification."""
|
||||
|
||||
def test_cmp_class_quiet_nan(self):
|
||||
"""V_CMP_CLASS_F32 detects quiet NaN."""
|
||||
quiet_nan = 0x7fc00000
|
||||
instructions = [
|
||||
s_mov_b32(s[0], quiet_nan),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], 0b0000000010), # bit 1 = quiet NaN
|
||||
v_cmp_class_f32_e32(v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "Should detect quiet NaN")
|
||||
|
||||
def test_cmp_class_signaling_nan(self):
|
||||
"""V_CMP_CLASS_F32 detects signaling NaN."""
|
||||
signal_nan = 0x7f800001
|
||||
instructions = [
|
||||
s_mov_b32(s[0], signal_nan),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], 0b0000000001), # bit 0 = signaling NaN
|
||||
v_cmp_class_f32_e32(v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "Should detect signaling NaN")
|
||||
|
||||
def test_cmp_class_positive_inf(self):
|
||||
"""V_CMP_CLASS_F32 detects +inf."""
|
||||
pos_inf = 0x7f800000
|
||||
instructions = [
|
||||
s_mov_b32(s[0], pos_inf),
|
||||
s_mov_b32(s[1], 0b1000000000), # bit 9 = +inf
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
v_cmp_class_f32_e32(v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "Should detect +inf")
|
||||
|
||||
def test_cmp_class_negative_inf(self):
|
||||
"""V_CMP_CLASS_F32 detects -inf."""
|
||||
neg_inf = 0xff800000
|
||||
instructions = [
|
||||
s_mov_b32(s[0], neg_inf),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], 0b0000000100), # bit 2 = -inf
|
||||
v_cmp_class_f32_e32(v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "Should detect -inf")
|
||||
|
||||
def test_cmp_class_normal_positive(self):
|
||||
"""V_CMP_CLASS_F32 detects positive normal."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], 1.0),
|
||||
s_mov_b32(s[1], 0b0100000000), # bit 8 = positive normal
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
v_cmp_class_f32_e32(v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "Should detect positive normal")
|
||||
|
||||
def test_cmp_class_normal_negative(self):
|
||||
"""V_CMP_CLASS_F32 detects negative normal."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], -1.0),
|
||||
v_mov_b32_e32(v[1], 0b0000001000), # bit 3 = negative normal
|
||||
v_cmp_class_f32_e32(v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "Should detect negative normal")
|
||||
|
||||
def test_cmp_class_quiet_nan_not_signaling(self):
|
||||
"""Quiet NaN does not match signaling NaN mask."""
|
||||
quiet_nan = 0x7fc00000
|
||||
instructions = [
|
||||
s_mov_b32(s[0], quiet_nan),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], 0b0000000001), # bit 0 = signaling NaN only
|
||||
v_cmp_class_f32_e32(v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 0, "Quiet NaN should not match signaling mask")
|
||||
|
||||
def test_cmp_class_signaling_nan_not_quiet(self):
|
||||
"""Signaling NaN does not match quiet NaN mask."""
|
||||
signal_nan = 0x7f800001
|
||||
instructions = [
|
||||
s_mov_b32(s[0], signal_nan),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], 0b0000000010), # bit 1 = quiet NaN only
|
||||
v_cmp_class_f32_e32(v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 0, "Signaling NaN should not match quiet mask")
|
||||
|
||||
def test_v_cmp_lg_f32_nan(self):
|
||||
"""v_cmp_lg_f32 is ordered not-equal (<>): NaN <> x should be False per IEEE 754."""
|
||||
quiet_nan = 0x7fc00000
|
||||
one_f32 = 0x3f800000 # 1.0f
|
||||
instructions = [
|
||||
s_mov_b32(s[0], quiet_nan),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
s_mov_b32(s[1], one_f32),
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
v_cmp_lg_f32_e32(v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 0, "v_cmp_lg_f32(NaN, 1.0) should be 0")
|
||||
|
||||
def test_v_cmp_neq_f32_nan(self):
|
||||
"""v_cmp_neq_f32 is unordered not-equal (!=): NaN != x should be True per IEEE 754."""
|
||||
quiet_nan = 0x7fc00000
|
||||
one_f32 = 0x3f800000 # 1.0f
|
||||
instructions = [
|
||||
s_mov_b32(s[0], quiet_nan),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
s_mov_b32(s[1], one_f32),
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
v_cmp_neq_f32_e32(v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "v_cmp_neq_f32(NaN, 1.0) should be 1")
|
||||
|
||||
def test_v_cmp_sets_vcc_bits(self):
|
||||
"""V_CMP_EQ sets VCC bits based on per-lane comparison."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 5),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], s[0]),
|
||||
v_cmp_eq_u32_e32(v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=4)
|
||||
self.assertEqual(st.vcc & 0xf, 0xf, "All lanes should match")
|
||||
|
||||
|
||||
class TestCmpClassF16(unittest.TestCase):
|
||||
"""Tests for V_CMP_CLASS_F16 float classification.
|
||||
|
||||
Class bit mapping:
|
||||
bit 0 = signaling NaN
|
||||
bit 1 = quiet NaN
|
||||
bit 2 = -infinity
|
||||
bit 3 = -normal
|
||||
bit 4 = -denormal
|
||||
bit 5 = -zero
|
||||
bit 6 = +zero
|
||||
bit 7 = +denormal
|
||||
bit 8 = +normal
|
||||
bit 9 = +infinity
|
||||
"""
|
||||
|
||||
def test_cmp_class_f16_positive_zero(self):
|
||||
"""V_CMP_CLASS_F16: +zero matches bit 6."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], 0x0000), # f16 +0.0
|
||||
v_mov_b32_e32(v[1], 0x40), # bit 6 = +zero
|
||||
v_cmp_class_f16_e32(v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "Should detect positive zero")
|
||||
|
||||
def test_cmp_class_f16_negative_zero(self):
|
||||
"""V_CMP_CLASS_F16: -zero matches bit 5."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x8000), # f16 -0.0
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], 0x20), # bit 5 = -zero
|
||||
v_cmp_class_f16_e32(v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "Should detect negative zero")
|
||||
|
||||
def test_cmp_class_f16_positive_normal(self):
|
||||
"""V_CMP_CLASS_F16: +1.0 (normal) matches bit 8."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x3c00), # f16 +1.0
|
||||
s_mov_b32(s[1], 0x100), # bit 8 = +normal
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
v_cmp_class_f16_e32(v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "Should detect positive normal")
|
||||
|
||||
def test_cmp_class_f16_negative_normal(self):
|
||||
"""V_CMP_CLASS_F16: -1.0 (normal) matches bit 3."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0xbc00), # f16 -1.0
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], 0x08), # bit 3 = -normal
|
||||
v_cmp_class_f16_e32(v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "Should detect negative normal")
|
||||
|
||||
def test_cmp_class_f16_positive_infinity(self):
|
||||
"""V_CMP_CLASS_F16: +inf matches bit 9."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x7c00), # f16 +inf
|
||||
s_mov_b32(s[1], 0x200), # bit 9 = +inf
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
v_cmp_class_f16_e32(v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "Should detect positive infinity")
|
||||
|
||||
def test_cmp_class_f16_negative_infinity(self):
|
||||
"""V_CMP_CLASS_F16: -inf matches bit 2."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0xfc00), # f16 -inf
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], 0x04), # bit 2 = -inf
|
||||
v_cmp_class_f16_e32(v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "Should detect negative infinity")
|
||||
|
||||
def test_cmp_class_f16_quiet_nan(self):
|
||||
"""V_CMP_CLASS_F16: quiet NaN matches bit 1."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x7e00), # f16 quiet NaN
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], 0x02), # bit 1 = quiet NaN
|
||||
v_cmp_class_f16_e32(v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "Should detect quiet NaN")
|
||||
|
||||
def test_cmp_class_f16_signaling_nan(self):
|
||||
"""V_CMP_CLASS_F16: signaling NaN matches bit 0."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x7c01), # f16 signaling NaN
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], 0x01), # bit 0 = signaling NaN
|
||||
v_cmp_class_f16_e32(v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "Should detect signaling NaN")
|
||||
|
||||
def test_cmp_class_f16_positive_denormal(self):
|
||||
"""V_CMP_CLASS_F16: positive denormal matches bit 7."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], 1), # f16 +denormal (0x0001)
|
||||
v_mov_b32_e32(v[1], 0x80), # bit 7 = +denormal
|
||||
v_cmp_class_f16_e32(v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "Should detect positive denormal")
|
||||
|
||||
def test_cmp_class_f16_negative_denormal(self):
|
||||
"""V_CMP_CLASS_F16: negative denormal matches bit 4."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x8001), # f16 -denormal
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], 0x10), # bit 4 = -denormal
|
||||
v_cmp_class_f16_e32(v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "Should detect negative denormal")
|
||||
|
||||
def test_cmp_class_f16_combined_mask_zeros(self):
|
||||
"""V_CMP_CLASS_F16: mask 0x60 covers both +zero and -zero."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], 0), # f16 +0.0
|
||||
v_mov_b32_e32(v[1], 0x60), # bits 5 and 6 (+-zero)
|
||||
v_cmp_class_f16_e32(v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "VCC should be 1 for +zero with mask 0x60")
|
||||
|
||||
def test_cmp_class_f16_combined_mask_1f8(self):
|
||||
"""V_CMP_CLASS_F16: mask 0x1f8 covers -normal,-denorm,-zero,+zero,+denorm,+normal.
|
||||
|
||||
This is the exact mask used in the f16 sin kernel at PC=46.
|
||||
"""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], 0), # f16 +0.0
|
||||
s_mov_b32(s[0], 0x1f8),
|
||||
v_mov_b32_e32(v[1], s[0]), # mask 0x1f8
|
||||
v_cmp_class_f16_e32(v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "VCC should be 1 for +zero with mask 0x1f8")
|
||||
|
||||
def test_cmp_class_f16_vop3_encoding(self):
|
||||
"""V_CMP_CLASS_F16 in VOP3 encoding (v_cmp_class_f16_e64)."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], 0), # f16 +0.0
|
||||
s_mov_b32(s[0], 0x1f8), # class mask
|
||||
v_cmp_class_f16_e64(VCC_LO, v[0], s[0]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "VCC should be 1 for +zero with VOP3 encoding")
|
||||
|
||||
def test_cmp_class_f16_vop3_normal_positive(self):
|
||||
"""V_CMP_CLASS_F16 VOP3 encoding with +1.0 (normal)."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x3c00), # f16 +1.0
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
s_mov_b32(s[1], 0x1f8), # class mask
|
||||
v_cmp_class_f16_e64(VCC_LO, v[0], s[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "VCC should be 1 for +1.0 (normal) with mask 0x1f8")
|
||||
|
||||
def test_cmp_class_f16_vop3_nan_fails_mask(self):
|
||||
"""V_CMP_CLASS_F16 VOP3: NaN should NOT match mask 0x1f8 (no NaN bits set)."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x7e00), # f16 quiet NaN
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
s_mov_b32(s[1], 0x1f8), # class mask
|
||||
v_cmp_class_f16_e64(VCC_LO, v[0], s[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 0, "VCC should be 0 for NaN with mask 0x1f8 (no NaN bits)")
|
||||
|
||||
def test_cmp_class_f16_vop3_inf_fails_mask(self):
|
||||
"""V_CMP_CLASS_F16 VOP3: +inf should NOT match mask 0x1f8 (no inf bits set)."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x7c00), # f16 +inf
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
s_mov_b32(s[1], 0x1f8), # class mask
|
||||
v_cmp_class_f16_e64(VCC_LO, v[0], s[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 0, "VCC should be 0 for +inf with mask 0x1f8 (no inf bits)")
|
||||
|
||||
|
||||
class TestCmpInt(unittest.TestCase):
|
||||
"""Tests for integer comparison operations."""
|
||||
|
||||
def test_v_cmp_eq_u32(self):
|
||||
"""V_CMP_EQ_U32 sets VCC bits based on per-lane comparison."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 5),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], s[0]),
|
||||
v_cmp_eq_u32_e32(v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=4)
|
||||
self.assertEqual(st.vcc & 0xf, 0xf, "All lanes should match")
|
||||
|
||||
def test_v_cmp_ne_u32_with_zero(self):
|
||||
"""V_CMP_NE_U32: compare with zero, used for int->bool cast."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[1], 0),
|
||||
v_cmp_eq_u32_e32(1, v[255]), # vcc = (lane == 1)
|
||||
v_cndmask_b32_e64(v[1], v[1], 1, VCC_LO), # v1[lane1] = 1
|
||||
v_cmp_ne_u32_e32(0, v[1]), # vcc = (0 != v1)
|
||||
v_cndmask_b32_e64(v[0], 0, 1, VCC_LO), # v0 = vcc ? 1 : 0
|
||||
]
|
||||
st = run_program(instructions, n_lanes=2)
|
||||
self.assertEqual(st.vgpr[0][0], 0, "lane 0: 0 != 0 should be false")
|
||||
self.assertEqual(st.vgpr[1][0], 1, "lane 1: 0 != 1 should be true")
|
||||
self.assertEqual(st.vcc & 0x3, 0x2, "VCC should be 0b10")
|
||||
|
||||
def test_v_cmp_ne_u32_all_nonzero(self):
|
||||
"""V_CMP_NE_U32: all lanes have nonzero values."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[1], 5),
|
||||
v_cmp_ne_u32_e32(0, v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=4)
|
||||
self.assertEqual(st.vcc & 0xf, 0xf, "All lanes should be != 0")
|
||||
|
||||
def test_cmp_eq_u16_opsel_lo_lo(self):
|
||||
"""V_CMP_EQ_U16 comparing lo halves."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x12340005), # lo=5, hi=0x1234
|
||||
s_mov_b32(s[1], 0xABCD0005), # lo=5, hi=0xABCD
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], s[1]),
|
||||
v_cmp_eq_u16_e32(v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "Lo halves should be equal")
|
||||
|
||||
def test_cmp_eq_u16_opsel_hi_hi(self):
|
||||
"""V_CMP_EQ_U16 comparing hi halves with VOP3 opsel."""
|
||||
instructions = [
|
||||
s_mov_b32(s[2], 0x00051234), # hi=5, lo=0x1234
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
s_mov_b32(s[2], 0x0005ABCD), # hi=5, lo=0xABCD
|
||||
v_mov_b32_e32(v[1], s[2]),
|
||||
v_cmp_eq_u16_e64(vdst=s[0], src0=v[0], src1=v[1], opsel=3),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.sgpr[0] & 1, 1, "Hi halves should be equal: 5==5")
|
||||
|
||||
def test_cmp_eq_u16_opsel_hi_hi_equal(self):
|
||||
"""V_CMP_EQ_U16 VOP3 with opsel=3 compares hi halves (equal case)."""
|
||||
instructions = [
|
||||
s_mov_b32(s[2], 0x12340005), # lo=5, hi=0x1234
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
s_mov_b32(s[2], 0x12340009), # lo=9, hi=0x1234
|
||||
v_mov_b32_e32(v[1], s[2]),
|
||||
v_cmp_eq_u16_e64(vdst=s[0], src0=v[0], src1=v[1], opsel=3),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.sgpr[0] & 1, 1, "hi==hi should be true: 0x1234==0x1234")
|
||||
|
||||
def test_cmp_gt_u16_opsel_hi(self):
|
||||
"""V_CMP_GT_U16 VOP3 with opsel=3 compares hi halves."""
|
||||
instructions = [
|
||||
s_mov_b32(s[2], 0x99990005), # lo=5, hi=0x9999
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
s_mov_b32(s[2], 0x12340005), # lo=5, hi=0x1234
|
||||
v_mov_b32_e32(v[1], s[2]),
|
||||
v_cmp_gt_u16_e64(vdst=s[0], src0=v[0], src1=v[1], opsel=3),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.sgpr[0] & 1, 1, "hi>hi should be true: 0x9999>0x1234")
|
||||
|
||||
|
||||
class TestCmpFloat(unittest.TestCase):
|
||||
"""Tests for float comparison operations."""
|
||||
|
||||
def test_v_cmp_lt_f16_vsrc1_hi(self):
|
||||
"""V_CMP_LT_F16 with both operands from high half using VOP3 opsel."""
|
||||
instructions = [
|
||||
s_mov_b32(s[2], 0x3c000000), # hi=1.0 (f16), lo=0
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
s_mov_b32(s[2], 0x40000000), # hi=2.0 (f16), lo=0
|
||||
v_mov_b32_e32(v[1], s[2]),
|
||||
v_cmp_lt_f16_e64(vdst=s[0], src0=v[0], src1=v[1], opsel=3),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.sgpr[0] & 1, 1, "1.0 < 2.0 should be true")
|
||||
|
||||
def test_v_cmp_gt_f16_vsrc1_hi(self):
|
||||
"""V_CMP_GT_F16 with both operands from high half using VOP3 opsel."""
|
||||
instructions = [
|
||||
s_mov_b32(s[2], 0x40000000), # hi=2.0 (f16), lo=0
|
||||
v_mov_b32_e32(v[0], s[2]),
|
||||
s_mov_b32(s[2], 0x3c000000), # hi=1.0 (f16), lo=0
|
||||
v_mov_b32_e32(v[1], s[2]),
|
||||
v_cmp_gt_f16_e64(vdst=s[0], src0=v[0], src1=v[1], opsel=3),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.sgpr[0] & 1, 1, "2.0 > 1.0 should be true")
|
||||
|
||||
def test_v_cmp_eq_f16_vsrc1_hi_equal(self):
|
||||
"""v_cmp_eq_f16 with equal low and high halves."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x42004200), # hi=3.0 (0x4200), lo=3.0 (0x4200)
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_cmp_eq_f16_e32(v[0], v[0].h),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "Expected vcc=1 (3.0 == 3.0)")
|
||||
|
||||
def test_v_cmp_neq_f16_vsrc1_hi(self):
|
||||
"""v_cmp_neq_f16 with different low and high halves."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0x40003c00), # hi=2.0 (0x4000), lo=1.0 (0x3c00)
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_cmp_lg_f16_e32(v[0], v[0].h),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "Expected vcc=1 (1.0 != 2.0)")
|
||||
|
||||
def test_v_cmp_nge_f16_inf_self(self):
|
||||
"""v_cmp_nge_f16 comparing -inf with itself (unordered less than).
|
||||
|
||||
Regression test: -inf < -inf should be false (IEEE 754).
|
||||
"""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0xFC00FC00), # both halves = -inf (0xFC00)
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_cmp_nge_f16_e32(v[0], v[0].h),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 0, "Expected vcc=0 (-inf >= -inf)")
|
||||
|
||||
def test_v_cmp_f16_multilane(self):
|
||||
"""v_cmp_lt_f16 with vsrc1=v128 across multiple lanes."""
|
||||
instructions = [
|
||||
# Lane 0: v0 = 0x40003c00 (hi=2.0, lo=1.0) -> 1.0 < 2.0 = true
|
||||
# Lane 1: v0 = 0x3c004000 (hi=1.0, lo=2.0) -> 2.0 < 1.0 = false
|
||||
v_mov_b32_e32(v[0], 0x40003c00), # default
|
||||
v_cmp_eq_u32_e32(1, v[255]), # vcc = (lane == 1)
|
||||
v_cndmask_b32_e64(v[0], v[0], 0x3c004000, SrcEnum.VCC_LO),
|
||||
v_cmp_lt_f16_e32(v[0], v[0].h),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=2)
|
||||
self.assertEqual(st.vcc & 1, 1, "Lane 0: expected vcc=1 (1.0 < 2.0)")
|
||||
self.assertEqual((st.vcc >> 1) & 1, 0, "Lane 1: expected vcc=0 (2.0 < 1.0)")
|
||||
|
||||
|
||||
class TestVOP3VOPCModifiers(unittest.TestCase):
|
||||
"""Tests for VOP3 VOPC with abs/neg modifiers."""
|
||||
|
||||
def test_v_cmp_ge_f32_abs_both(self):
|
||||
"""v_cmp_ge_f32 with abs on both sources: abs(0.0) >= abs(-1.0) = false.
|
||||
|
||||
Regression test: int16 mod operation uses v_cmp_ge_f32 with abs modifiers.
|
||||
"""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], 0.0),
|
||||
v_mov_b32_e32(v[1], -1.0),
|
||||
# abs=0b11 means abs(src0) and abs(src1)
|
||||
v_cmp_ge_f32_e64(VCC_LO, v[0], v[1], abs=0b11),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 0, "abs(0.0) >= abs(-1.0) should be false")
|
||||
|
||||
def test_v_cmp_ge_f32_abs_negative_divisor(self):
|
||||
"""v_cmp_ge_f32 with abs: remainder check for negative divisor.
|
||||
|
||||
Tests the exact comparison used in int16 mod: abs(rem_f) >= abs(div_f).
|
||||
For 1 % -1: rem_f = 0.0, div_f = -1.0, so abs(0.0) >= abs(-1.0) = false.
|
||||
"""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], 0.0), # remainder as float
|
||||
v_mov_b32_e32(v[1], -1.0), # divisor as float
|
||||
v_cmp_ge_f32_e64(VCC_LO, v[0], v[1], abs=0b11),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 0, "abs(0.0) >= abs(-1.0) should be false")
|
||||
|
||||
def test_v_cmp_ge_f32_abs_small_remainder(self):
|
||||
"""v_cmp_ge_f32 with abs: abs(-0.5) >= abs(-3.0) = false."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], -0.5),
|
||||
v_mov_b32_e32(v[1], -3.0),
|
||||
v_cmp_ge_f32_e64(VCC_LO, v[0], v[1], abs=0b11),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 0, "abs(-0.5) >= abs(-3.0) should be false")
|
||||
|
||||
def test_v_cmp_ge_f32_abs_equal(self):
|
||||
"""v_cmp_ge_f32 with abs: abs(-1.0) >= abs(1.0) = true."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], -1.0),
|
||||
v_mov_b32_e32(v[1], 1.0),
|
||||
v_cmp_ge_f32_e64(VCC_LO, v[0], v[1], abs=0b11),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "abs(-1.0) >= abs(1.0) should be true")
|
||||
|
||||
|
||||
class TestVOP3VOPC64Bit(unittest.TestCase):
|
||||
"""Tests for VOP3 VOPC with 64-bit operands."""
|
||||
|
||||
def test_v_cmp_lt_f64_basic(self):
|
||||
"""v_cmp_lt_f64: 0.0 < 1.0 = true."""
|
||||
zero_f64 = f2i64(0.0)
|
||||
one_f64 = f2i64(1.0)
|
||||
instructions = [
|
||||
s_mov_b32(s[0], zero_f64 & 0xffffffff),
|
||||
s_mov_b32(s[1], zero_f64 >> 32),
|
||||
s_mov_b32(s[2], one_f64 & 0xffffffff),
|
||||
s_mov_b32(s[3], one_f64 >> 32),
|
||||
v_cmp_lt_f64_e64(VCC_LO, s[0:1], s[2:3]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "0.0 < 1.0 should be true")
|
||||
|
||||
def test_v_cmp_lt_f64_negative(self):
|
||||
"""v_cmp_lt_f64: -1.0 < 0.0 = true."""
|
||||
neg_one_f64 = f2i64(-1.0)
|
||||
zero_f64 = f2i64(0.0)
|
||||
instructions = [
|
||||
s_mov_b32(s[0], neg_one_f64 & 0xffffffff),
|
||||
s_mov_b32(s[1], neg_one_f64 >> 32),
|
||||
s_mov_b32(s[2], zero_f64 & 0xffffffff),
|
||||
s_mov_b32(s[3], zero_f64 >> 32),
|
||||
v_cmp_lt_f64_e64(VCC_LO, s[0:1], s[2:3]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "-1.0 < 0.0 should be true")
|
||||
|
||||
def test_v_cmp_lt_i64_signed(self):
|
||||
"""v_cmp_lt_i64: 0 < -1 (signed) = false."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0),
|
||||
s_mov_b32(s[1], 0), # s[0:1] = 0
|
||||
s_mov_b32(s[2], 0xffffffff),
|
||||
s_mov_b32(s[3], 0xffffffff), # s[2:3] = -1
|
||||
v_cmp_lt_i64_e64(VCC_LO, s[0:1], s[2:3]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 0, "0 < -1 (signed) should be false")
|
||||
|
||||
def test_v_cmp_lt_u64_unsigned(self):
|
||||
"""v_cmp_lt_u64: 0 < 0xFFFFFFFFFFFFFFFF (unsigned) = true."""
|
||||
instructions = [
|
||||
s_mov_b32(s[0], 0),
|
||||
s_mov_b32(s[1], 0), # s[0:1] = 0
|
||||
s_mov_b32(s[2], 0xffffffff),
|
||||
s_mov_b32(s[3], 0xffffffff), # s[2:3] = max uint64
|
||||
v_cmp_lt_u64_e64(VCC_LO, s[0:1], s[2:3]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "0 < max_uint64 should be true")
|
||||
|
||||
|
||||
class TestVOPCF64(unittest.TestCase):
|
||||
"""Tests for VOPC (E32 encoding) with 64-bit float operands. Regression test for f64 compare bug."""
|
||||
|
||||
def test_v_cmp_lt_f64_e32_true(self):
|
||||
"""v_cmp_lt_f64_e32: 2.0 < 3.0 = true."""
|
||||
lo0, hi0 = f2i64(2.0) & 0xffffffff, f2i64(2.0) >> 32
|
||||
lo1, hi1 = f2i64(3.0) & 0xffffffff, f2i64(3.0) >> 32
|
||||
instructions = [
|
||||
s_mov_b32(s[0], lo0), s_mov_b32(s[1], hi0),
|
||||
s_mov_b32(s[2], lo1), s_mov_b32(s[3], hi1),
|
||||
v_mov_b32_e32(v[0], s[0]), v_mov_b32_e32(v[1], s[1]),
|
||||
v_mov_b32_e32(v[2], s[2]), v_mov_b32_e32(v[3], s[3]),
|
||||
v_cmp_lt_f64_e32(v[0:1], v[2:3]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "2.0 < 3.0 should be true")
|
||||
|
||||
def test_v_cmp_lt_f64_e32_false(self):
|
||||
"""v_cmp_lt_f64_e32: 3.0 < 2.0 = false."""
|
||||
lo0, hi0 = f2i64(3.0) & 0xffffffff, f2i64(3.0) >> 32
|
||||
lo1, hi1 = f2i64(2.0) & 0xffffffff, f2i64(2.0) >> 32
|
||||
instructions = [
|
||||
s_mov_b32(s[0], lo0), s_mov_b32(s[1], hi0),
|
||||
s_mov_b32(s[2], lo1), s_mov_b32(s[3], hi1),
|
||||
v_mov_b32_e32(v[0], s[0]), v_mov_b32_e32(v[1], s[1]),
|
||||
v_mov_b32_e32(v[2], s[2]), v_mov_b32_e32(v[3], s[3]),
|
||||
v_cmp_lt_f64_e32(v[0:1], v[2:3]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 0, "3.0 < 2.0 should be false")
|
||||
|
||||
def test_v_cmp_nlt_f64_e32_true(self):
|
||||
"""v_cmp_nlt_f64_e32: !(3.0 < 2.0) = true."""
|
||||
lo0, hi0 = f2i64(3.0) & 0xffffffff, f2i64(3.0) >> 32
|
||||
lo1, hi1 = f2i64(2.0) & 0xffffffff, f2i64(2.0) >> 32
|
||||
instructions = [
|
||||
s_mov_b32(s[0], lo0), s_mov_b32(s[1], hi0),
|
||||
s_mov_b32(s[2], lo1), s_mov_b32(s[3], hi1),
|
||||
v_mov_b32_e32(v[0], s[0]), v_mov_b32_e32(v[1], s[1]),
|
||||
v_mov_b32_e32(v[2], s[2]), v_mov_b32_e32(v[3], s[3]),
|
||||
v_cmp_nlt_f64_e32(v[0:1], v[2:3]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "!(3.0 < 2.0) should be true")
|
||||
|
||||
def test_v_cmp_nlt_f64_e32_false(self):
|
||||
"""v_cmp_nlt_f64_e32: !(2.0 < 3.0) = false."""
|
||||
lo0, hi0 = f2i64(2.0) & 0xffffffff, f2i64(2.0) >> 32
|
||||
lo1, hi1 = f2i64(3.0) & 0xffffffff, f2i64(3.0) >> 32
|
||||
instructions = [
|
||||
s_mov_b32(s[0], lo0), s_mov_b32(s[1], hi0),
|
||||
s_mov_b32(s[2], lo1), s_mov_b32(s[3], hi1),
|
||||
v_mov_b32_e32(v[0], s[0]), v_mov_b32_e32(v[1], s[1]),
|
||||
v_mov_b32_e32(v[2], s[2]), v_mov_b32_e32(v[3], s[3]),
|
||||
v_cmp_nlt_f64_e32(v[0:1], v[2:3]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 0, "!(2.0 < 3.0) should be false")
|
||||
|
||||
|
||||
class TestCmpxExec(unittest.TestCase):
|
||||
"""Tests for V_CMPX instructions that modify EXEC mask."""
|
||||
|
||||
def test_v_cmpx_ngt_f32_e64_all_true(self):
|
||||
"""V_CMPX_NGT_F32_E64: all lanes pass (literal <= all values)."""
|
||||
# 131072.0 = 0x48000000
|
||||
# All values > 131072, so !(131072 > val) = true for all
|
||||
instructions = [
|
||||
s_mov_b32(EXEC_LO, 0x7), # 3 lanes active
|
||||
v_mov_b32_e32(v[0], f2i(200000.0)), # lane 0
|
||||
v_cmp_eq_u32_e32(1, v[255]),
|
||||
v_cndmask_b32_e64(v[1], v[0], f2i(300000.0), VCC_LO), # lane 1
|
||||
v_cmp_eq_u32_e32(2, v[255]),
|
||||
v_cndmask_b32_e64(v[1], v[1], f2i(400000.0), VCC_LO), # lane 2
|
||||
# Now v[1] has: lane0=200000, lane1=300000, lane2=400000
|
||||
# Compare: !(131072.0 > v[1]) i.e., 131072.0 <= v[1]
|
||||
v_cmpx_ngt_f32_e64(EXEC_LO, f2i(131072.0), v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=3)
|
||||
# All values > 131072, so all lanes should remain active
|
||||
self.assertEqual(st.sgpr[EXEC_LO.offset] & 0x7, 0x7, "All 3 lanes should remain active")
|
||||
|
||||
def test_v_cmpx_ngt_f32_e64_some_false(self):
|
||||
"""V_CMPX_NGT_F32_E64: some lanes fail (literal > some values)."""
|
||||
instructions = [
|
||||
s_mov_b32(EXEC_LO, 0x7), # 3 lanes active
|
||||
v_mov_b32_e32(v[0], f2i(100000.0)), # lane 0: 131072 > 100000 = true, so !(true) = false
|
||||
v_cmp_eq_u32_e32(1, v[255]),
|
||||
v_cndmask_b32_e64(v[1], v[0], f2i(200000.0), VCC_LO), # lane 1: 131072 > 200000 = false, so !(false) = true
|
||||
v_cmp_eq_u32_e32(2, v[255]),
|
||||
v_cndmask_b32_e64(v[1], v[1], f2i(150000.0), VCC_LO), # lane 2: 131072 > 150000 = false, so !(false) = true
|
||||
v_cmpx_ngt_f32_e64(EXEC_LO, f2i(131072.0), v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=3)
|
||||
# lane 0: fail (100000 < 131072), lanes 1,2: pass
|
||||
self.assertEqual(st.sgpr[EXEC_LO.offset] & 0x7, 0x6, "Lanes 1,2 should be active, lane 0 inactive")
|
||||
|
||||
def test_v_cmpx_ngt_f32_e64_all_false(self):
|
||||
"""V_CMPX_NGT_F32_E64: all lanes fail (literal > all values)."""
|
||||
instructions = [
|
||||
s_mov_b32(EXEC_LO, 0x7), # 3 lanes active
|
||||
v_mov_b32_e32(v[0], f2i(100.0)), # all lanes have 100.0
|
||||
# 131072 > 100 = true, so !(true) = false for all
|
||||
v_cmpx_ngt_f32_e64(EXEC_LO, f2i(131072.0), v[0]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=3)
|
||||
self.assertEqual(st.sgpr[EXEC_LO.offset] & 0x7, 0x0, "All lanes should be inactive")
|
||||
|
||||
def test_v_cmpx_ngt_f32_e64_large_values(self):
|
||||
"""V_CMPX_NGT_F32_E64: test with values that trigger Payne-Hanek in sin().
|
||||
|
||||
This is a regression test for the sin(859240.0) bug.
|
||||
Values 859240, 1000000, 100594688 should all pass !(131072 > val).
|
||||
"""
|
||||
instructions = [
|
||||
s_mov_b32(EXEC_LO, 0x7), # 3 lanes active
|
||||
v_mov_b32_e32(v[0], f2i(859240.0)), # lane 0
|
||||
v_cmp_eq_u32_e32(1, v[255]),
|
||||
v_cndmask_b32_e64(v[1], v[0], f2i(1000000.0), VCC_LO), # lane 1
|
||||
v_cmp_eq_u32_e32(2, v[255]),
|
||||
v_cndmask_b32_e64(v[1], v[1], f2i(100594688.0), VCC_LO), # lane 2
|
||||
v_cmpx_ngt_f32_e64(EXEC_LO, f2i(131072.0), v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=3)
|
||||
# All values > 131072, so !(131072 > val) = true for all
|
||||
self.assertEqual(st.sgpr[EXEC_LO.offset] & 0x7, 0x7, "All 3 lanes should remain active")
|
||||
|
||||
|
||||
class TestVCCBehavior(unittest.TestCase):
|
||||
"""Tests for VCC condition code behavior."""
|
||||
|
||||
def test_vcc_all_lanes_true(self):
|
||||
"""VCC should have all bits set when all lanes compare true."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], 5),
|
||||
v_mov_b32_e32(v[1], 5),
|
||||
v_cmp_eq_u32_e32(v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=32)
|
||||
self.assertEqual(st.vcc, 0xFFFFFFFF, "All 32 lanes should be true")
|
||||
|
||||
def test_vcc_lane_dependent(self):
|
||||
"""VCC should differ per lane based on lane_id comparison."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], 16),
|
||||
v_cmp_lt_u32_e32(v[255], v[0]), # lanes 0-15 are < 16
|
||||
]
|
||||
st = run_program(instructions, n_lanes=32)
|
||||
self.assertEqual(st.vcc & 0xFFFF, 0xFFFF, "Lanes 0-15 should be true")
|
||||
self.assertEqual(st.vcc >> 16, 0x0000, "Lanes 16-31 should be false")
|
||||
|
||||
|
||||
class TestCmpNge(unittest.TestCase):
|
||||
"""Tests for V_CMP_NGE (not-greater-or-equal) with NaN semantics.
|
||||
|
||||
NGE = !(a >= b). With NaN inputs:
|
||||
- If either input is NaN, a >= b is false, so !(false) = true
|
||||
- This differs from a < b which returns false for NaN inputs
|
||||
"""
|
||||
|
||||
def test_v_cmp_nge_f32_normal_values(self):
|
||||
"""v_cmp_nge_f32: basic comparison with normal floats."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], f2i(1.0)),
|
||||
v_mov_b32_e32(v[1], f2i(2.0)),
|
||||
v_cmp_nge_f32_e32(v[0], v[1]), # !(1.0 >= 2.0) = !(false) = true
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "!(1.0 >= 2.0) should be true")
|
||||
|
||||
def test_v_cmp_nge_f32_equal_values(self):
|
||||
"""v_cmp_nge_f32: equal values should return false."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], f2i(1.0)),
|
||||
v_mov_b32_e32(v[1], f2i(1.0)),
|
||||
v_cmp_nge_f32_e32(v[0], v[1]), # !(1.0 >= 1.0) = !(true) = false
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 0, "!(1.0 >= 1.0) should be false")
|
||||
|
||||
def test_v_cmp_nge_f32_greater_value(self):
|
||||
"""v_cmp_nge_f32: greater value should return false."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], f2i(2.0)),
|
||||
v_mov_b32_e32(v[1], f2i(1.0)),
|
||||
v_cmp_nge_f32_e32(v[0], v[1]), # !(2.0 >= 1.0) = !(true) = false
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 0, "!(2.0 >= 1.0) should be false")
|
||||
|
||||
def test_v_cmp_nge_f32_neg_inf(self):
|
||||
"""v_cmp_nge_f32: -inf compared to normal value."""
|
||||
neg_inf = 0xff800000 # -inf
|
||||
instructions = [
|
||||
s_mov_b32(s[0], neg_inf),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], f2i(1.0)),
|
||||
v_cmp_nge_f32_e32(v[0], v[1]), # !(-inf >= 1.0) = !(false) = true
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "!(-inf >= 1.0) should be true")
|
||||
|
||||
def test_v_cmp_nge_f32_clears_inactive_vcc_bits(self):
|
||||
"""v_cmp_nge_f32 with partial EXEC clears inactive VCC bits (hardware behavior)."""
|
||||
neg_inf = 0xff800000 # -inf
|
||||
instructions = [
|
||||
# Set VCC to all 1s first
|
||||
s_mov_b32(VCC_LO, 0xFFFFFFFF),
|
||||
# Set EXEC to only lane 0
|
||||
s_mov_b32(EXEC_LO, 0x00000001),
|
||||
# v0 = 1.0 for lane 0
|
||||
v_mov_b32_e32(v[0], f2i(1.0)),
|
||||
# Compare: !(-inf >= 1.0) = true for lane 0
|
||||
v_cmp_nge_f32_e32(neg_inf, v[0]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=16)
|
||||
# Hardware clears inactive lane bits, only active lane results remain
|
||||
# Lane 0 result = 1 (true), lanes 1-15 = 0 (cleared)
|
||||
self.assertEqual(st.vcc, 0x00000001, "VCC should only have active lane results")
|
||||
|
||||
def test_v_cmp_nge_f32_nan_src0(self):
|
||||
"""v_cmp_nge_f32: NaN in src0 should return true (NaN >= x is false)."""
|
||||
quiet_nan = 0x7fc00000
|
||||
instructions = [
|
||||
s_mov_b32(s[0], quiet_nan),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], f2i(1.0)),
|
||||
v_cmp_nge_f32_e32(v[0], v[1]), # !(NaN >= 1.0) = !(false) = true
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "!(NaN >= 1.0) should be true")
|
||||
|
||||
def test_v_cmp_nge_f32_nan_src1(self):
|
||||
"""v_cmp_nge_f32: NaN in src1 should return true (x >= NaN is false)."""
|
||||
quiet_nan = 0x7fc00000
|
||||
instructions = [
|
||||
s_mov_b32(s[0], quiet_nan),
|
||||
v_mov_b32_e32(v[0], f2i(1.0)),
|
||||
v_mov_b32_e32(v[1], s[0]),
|
||||
v_cmp_nge_f32_e32(v[0], v[1]), # !(1.0 >= NaN) = !(false) = true
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "!(1.0 >= NaN) should be true")
|
||||
|
||||
def test_v_cmp_nge_f32_both_nan(self):
|
||||
"""v_cmp_nge_f32: both NaN should return true."""
|
||||
quiet_nan = 0x7fc00000
|
||||
instructions = [
|
||||
s_mov_b32(s[0], quiet_nan),
|
||||
v_mov_b32_e32(v[0], s[0]),
|
||||
v_mov_b32_e32(v[1], s[0]),
|
||||
v_cmp_nge_f32_e32(v[0], v[1]), # !(NaN >= NaN) = !(false) = true
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vcc & 1, 1, "!(NaN >= NaN) should be true")
|
||||
|
||||
|
||||
class TestCmpxPartialWavefront(unittest.TestCase):
|
||||
"""Tests for V_CMPX with partial wavefronts (fewer than 32 active lanes).
|
||||
|
||||
Regression tests for bug where v_cmpx incorrectly set EXEC bits for inactive
|
||||
lanes when the wavefront had fewer than 32 lanes. This caused garbage data
|
||||
from uninitialized lanes to corrupt memory writes.
|
||||
"""
|
||||
|
||||
def test_v_cmpx_eq_u32_partial_wave_3_lanes(self):
|
||||
"""V_CMPX_EQ_U32 with 3 active lanes should only affect those 3 lanes.
|
||||
|
||||
With n_lanes=3, initial EXEC=0x7. After v_cmpx comparing lane_id == 1,
|
||||
only lane 1 should pass, so EXEC should become 0x2 (not have bits 3-31 set).
|
||||
"""
|
||||
instructions = [
|
||||
v_cmpx_eq_u32_e32(1, v[255]), # EXEC = lanes where lane_id == 1
|
||||
]
|
||||
st = run_program(instructions, n_lanes=3)
|
||||
# Only lane 1 should be active (bit 1 set)
|
||||
self.assertEqual(st.sgpr[EXEC_LO.offset] & 0xFFFFFFFF, 0x2,
|
||||
"Only lane 1 should be active after v_cmpx_eq_u32 with 3 lanes")
|
||||
|
||||
def test_v_cmpx_eq_u32_partial_wave_5_lanes(self):
|
||||
"""V_CMPX_EQ_U32 with 5 active lanes."""
|
||||
instructions = [
|
||||
v_cmpx_eq_u32_e32(3, v[255]), # EXEC = lanes where lane_id == 3
|
||||
]
|
||||
st = run_program(instructions, n_lanes=5)
|
||||
self.assertEqual(st.sgpr[EXEC_LO.offset] & 0xFFFFFFFF, 0x8,
|
||||
"Only lane 3 should be active after v_cmpx_eq_u32 with 5 lanes")
|
||||
|
||||
def test_v_cmpx_lt_u32_partial_wave(self):
|
||||
"""V_CMPX_LT_U32 with partial wavefront."""
|
||||
# VOPC: src0 < vsrc1, so we need v_cmpx_gt_u32 to get lane_id < 2
|
||||
instructions = [
|
||||
v_cmpx_gt_u32_e32(2, v[255]), # EXEC = lanes where 2 > lane_id (i.e., lane_id < 2)
|
||||
]
|
||||
st = run_program(instructions, n_lanes=4)
|
||||
# Lanes 0,1 should be active (bits 0,1 set = 0x3)
|
||||
self.assertEqual(st.sgpr[EXEC_LO.offset] & 0xFFFFFFFF, 0x3,
|
||||
"Only lanes 0,1 should be active after v_cmpx_gt_u32(2, lane_id) with 4 lanes")
|
||||
|
||||
def test_v_cmpx_ge_u32_partial_wave(self):
|
||||
"""V_CMPX_GE_U32 with partial wavefront."""
|
||||
# VOPC: src0 >= vsrc1, so v_cmpx_le_u32(1, lane_id) gives lane_id >= 2? No.
|
||||
# v_cmpx_le_u32(src0, vsrc1) = src0 <= vsrc1 = 1 <= lane_id
|
||||
instructions = [
|
||||
v_cmpx_le_u32_e32(2, v[255]), # EXEC = lanes where 2 <= lane_id (i.e., lane_id >= 2)
|
||||
]
|
||||
st = run_program(instructions, n_lanes=4)
|
||||
# Lanes 2,3 should be active (bits 2,3 set = 0xC)
|
||||
self.assertEqual(st.sgpr[EXEC_LO.offset] & 0xFFFFFFFF, 0xC,
|
||||
"Only lanes 2,3 should be active after v_cmpx_le_u32(2, lane_id) with 4 lanes")
|
||||
|
||||
def test_v_cmpx_ne_u32_partial_wave_all_pass(self):
|
||||
"""V_CMPX_NE_U32 where all active lanes pass."""
|
||||
instructions = [
|
||||
v_cmpx_ne_u32_e32(99, v[255]), # EXEC = lanes where lane_id != 99
|
||||
]
|
||||
st = run_program(instructions, n_lanes=3)
|
||||
# All 3 lanes should remain active (bits 0,1,2 set = 0x7)
|
||||
self.assertEqual(st.sgpr[EXEC_LO.offset] & 0xFFFFFFFF, 0x7,
|
||||
"All 3 lanes should remain active when all pass")
|
||||
|
||||
def test_v_cmpx_eq_u32_partial_wave_none_pass(self):
|
||||
"""V_CMPX_EQ_U32 where no active lanes pass."""
|
||||
instructions = [
|
||||
v_cmpx_eq_u32_e32(99, v[255]), # EXEC = lanes where lane_id == 99
|
||||
]
|
||||
st = run_program(instructions, n_lanes=3)
|
||||
# No lanes should be active
|
||||
self.assertEqual(st.sgpr[EXEC_LO.offset] & 0xFFFFFFFF, 0x0,
|
||||
"No lanes should be active when none pass")
|
||||
|
||||
def test_v_cmpx_f32_partial_wave(self):
|
||||
"""V_CMPX_GT_F32 with partial wavefront - float comparison."""
|
||||
instructions = [
|
||||
v_cvt_f32_u32_e32(v[0], v[255]), # v[0] = float(lane_id)
|
||||
v_mov_b32_e32(v[1], f2i(0.5)), # v[1] = 0.5
|
||||
v_cmpx_gt_f32_e32(v[0], v[1]), # EXEC = lanes where v[0] > 0.5
|
||||
]
|
||||
st = run_program(instructions, n_lanes=4)
|
||||
# Lanes 1,2,3 have values > 0.5, lane 0 has 0.0
|
||||
self.assertEqual(st.sgpr[EXEC_LO.offset] & 0xFFFFFFFF, 0xE,
|
||||
"Lanes 1,2,3 should be active (float > 0.5)")
|
||||
|
||||
def test_v_cmpx_e64_partial_wave(self):
|
||||
"""V_CMPX_EQ_U32_E64 (VOP3 encoding) with partial wavefront."""
|
||||
instructions = [
|
||||
v_cmpx_eq_u32_e64(EXEC_LO, v[255], 2), # EXEC = lanes where lane_id == 2
|
||||
]
|
||||
st = run_program(instructions, n_lanes=4)
|
||||
self.assertEqual(st.sgpr[EXEC_LO.offset] & 0xFFFFFFFF, 0x4,
|
||||
"Only lane 2 should be active after v_cmpx_eq_u32_e64")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
201
tinygrad_repo/test/amd/hw/test_vopd.py
Normal file
201
tinygrad_repo/test/amd/hw/test_vopd.py
Normal file
@@ -0,0 +1,201 @@
|
||||
"""Tests for VOPD instructions - dual-issue vector operations.
|
||||
|
||||
VOPD executes two operations simultaneously. Key behavior:
|
||||
- Both ops read their sources BEFORE either writes (dual-issue semantics)
|
||||
- This means if X writes to a register that Y reads, Y sees the OLD value
|
||||
- Op X can use ops 0-15 (FMAC, MUL, ADD, MOV, etc.)
|
||||
- Op Y can use ops 0-18 (includes ADD_NC_U32, LSHLREV, AND)
|
||||
"""
|
||||
import unittest
|
||||
from test.amd.hw.helpers import run_program, v, v_mov_b32_e32
|
||||
from tinygrad.runtime.autogen.amd.rdna3.ins import VOPD, VOPD_LIT, VOPDOp
|
||||
|
||||
class TestVOPDBasic(unittest.TestCase):
|
||||
"""Basic VOPD functionality tests."""
|
||||
|
||||
def test_vopd_dual_mov(self):
|
||||
"""VOPD with two MOV operations to different registers."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], 0x12345678),
|
||||
v_mov_b32_e32(v[1], 0xDEADBEEF),
|
||||
# X: v[2] = v[0], Y: v[3] = v[1]
|
||||
VOPD(VOPDOp.V_DUAL_MOV_B32, VOPDOp.V_DUAL_MOV_B32, v[2], v[3], v[0], v[1], v[0], v[0]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 0x12345678)
|
||||
self.assertEqual(st.vgpr[0][3], 0xDEADBEEF)
|
||||
|
||||
def test_vopd_mov_and_add(self):
|
||||
"""VOPD with MOV (X) and ADD_NC_U32 (Y) - ADD_NC_U32 can only be Y op."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], 10),
|
||||
v_mov_b32_e32(v[1], 5),
|
||||
# X: v[2] = 100 (literal), Y: v[3] = v[0] + v[1] = 15
|
||||
VOPD(VOPDOp.V_DUAL_MOV_B32, VOPDOp.V_DUAL_ADD_NC_U32, v[2], v[3], 100, v[0], v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertEqual(st.vgpr[0][2], 100)
|
||||
self.assertEqual(st.vgpr[0][3], 15)
|
||||
|
||||
|
||||
class TestVOPDReadBeforeWrite(unittest.TestCase):
|
||||
"""Tests for VOPD dual-issue read-before-write semantics.
|
||||
|
||||
In VOPD, both X and Y operations read their sources BEFORE either writes.
|
||||
This is critical when X's destination is Y's source.
|
||||
"""
|
||||
|
||||
def test_vopd_x_writes_y_reads_same_reg(self):
|
||||
"""VOPD where X writes to a register that Y reads.
|
||||
|
||||
X: v[2] = 0 (overwrites v[2])
|
||||
Y: v[1] = v[2] + v[0] (srcy0=v[2], vsrcy1=v[0])
|
||||
|
||||
If reads happen before writes: v[1] = OLD_v[2] + v[0] = 0xFFFFFFFF + 1 = 0
|
||||
If writes happen before reads: v[1] = 0 + v[0] = 0 + 1 = 1
|
||||
|
||||
Hardware does reads-before-writes, so v[1] should be 0.
|
||||
"""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], 1), # v[0] = 1
|
||||
v_mov_b32_e32(v[1], 0x99999999), # v[1] = placeholder (will be overwritten)
|
||||
v_mov_b32_e32(v[2], 0xFFFFFFFF), # v[2] = 0xFFFFFFFF
|
||||
# X: v[2] = 0 (literal), srcx0=0, vsrcx1=v[0] (unused for MOV)
|
||||
# Y: v[1] = srcy0 + vsrcy1 = v[2] + v[0] (should read OLD v[2] = 0xFFFFFFFF)
|
||||
# vdsty encoding: (vdsty << 1) | ((vdstx & 1) ^ 1) where vdsty field = 0, vdstx = v[2]
|
||||
# So vdsty_reg = (0 << 1) | ((2 & 1) ^ 1) = 0 | 1 = 1 = v[1]
|
||||
VOPD(VOPDOp.V_DUAL_MOV_B32, VOPDOp.V_DUAL_ADD_NC_U32, v[2], v[0], 0, v[2], v[0], v[0]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
# X should have written 0 to v[2]
|
||||
self.assertEqual(st.vgpr[0][2], 0, "X should write 0 to v[2]")
|
||||
# Y should have read OLD v[2] (0xFFFFFFFF) and added v[0] (1)
|
||||
# 0xFFFFFFFF + 1 = 0 (wrap around)
|
||||
self.assertEqual(st.vgpr[0][1], 0, "Y should read OLD v[2]=0xFFFFFFFF, compute 0xFFFFFFFF+1=0")
|
||||
|
||||
def test_vopd_x_writes_y_reads_same_reg_v2(self):
|
||||
"""VOPD where X writes to a register that Y reads - cleaner test case.
|
||||
|
||||
X: v[2] = 0 (MOV)
|
||||
Y: v[1] = v[2] + v[2] (ADD_NC_U32 with both sources from v[2])
|
||||
|
||||
If reads happen before writes: v[1] = OLD_v[2] + OLD_v[2] = 100 + 100 = 200
|
||||
If writes happen before reads: v[1] = 0 + 0 = 0
|
||||
|
||||
Hardware does reads-before-writes, so v[1] should be 200.
|
||||
"""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], 0x88888888), # v[0] = unused placeholder
|
||||
v_mov_b32_e32(v[1], 0x99999999), # v[1] = placeholder (will be overwritten)
|
||||
v_mov_b32_e32(v[2], 100), # v[2] = 100
|
||||
# X: v[2] = 0 (literal)
|
||||
# Y: v[1] = srcy0 + vsrcy1 = v[2] + v[2] (should read OLD v[2] = 100)
|
||||
VOPD(VOPDOp.V_DUAL_MOV_B32, VOPDOp.V_DUAL_ADD_NC_U32, v[2], v[0], 0, v[2], v[0], v[2]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
# X should have written 0 to v[2]
|
||||
self.assertEqual(st.vgpr[0][2], 0, "X should write 0 to v[2]")
|
||||
# Y should have read OLD v[2] (100) twice and added them
|
||||
self.assertEqual(st.vgpr[0][1], 200, "Y should read OLD v[2]=100 twice, compute 100+100=200")
|
||||
|
||||
|
||||
class TestVOPDLiterals(unittest.TestCase):
|
||||
"""Tests for VOPD instructions that use SIMM32 literals (FMAAK, FMAMK)."""
|
||||
|
||||
def test_vopd_fmaak_f32(self):
|
||||
"""VOPD V_DUAL_FMAAK_F32: D = S0 * S1 + SIMM32 (literal addend).
|
||||
|
||||
Tests that the 32-bit literal (SIMM32) is correctly passed to the instruction.
|
||||
fma(2.0, 3.0, 10.0) = 2*3 + 10 = 16.0
|
||||
"""
|
||||
from test.amd.hw.helpers import f2i, i2f
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], f2i(2.0)), # v[0] = 2.0
|
||||
v_mov_b32_e32(v[1], f2i(3.0)), # v[1] = 3.0
|
||||
# VOPD args: opx, opy, vdstx, vdsty, srcx0, srcy0, vsrcx1, vsrcy1
|
||||
# X: v[2] = fma(srcx0, vsrcx1, SIMM32) = v[0]*v[1]+10.0 = 2*3+10 = 16
|
||||
# Y: v[3] = srcy0 (MOV) = v[0] = 2.0
|
||||
VOPD_LIT(VOPDOp.V_DUAL_FMAAK_F32, VOPDOp.V_DUAL_MOV_B32, v[2], v[3], v[0], v[0], v[1], v[0], literal=f2i(10.0)),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertAlmostEqual(i2f(st.vgpr[0][2]), 16.0, places=5, msg="fma(2.0, 3.0, 10.0) should be 16.0")
|
||||
|
||||
def test_vopd_fmamk_f32(self):
|
||||
"""VOPD V_DUAL_FMAMK_F32: D = S0 * SIMM32 + S1 (literal multiplier).
|
||||
|
||||
Tests that the 32-bit literal (SIMM32) is correctly used as the multiplier.
|
||||
fma(2.0, 5.0, 3.0) = 2*5 + 3 = 13.0
|
||||
"""
|
||||
from test.amd.hw.helpers import f2i, i2f
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], f2i(2.0)), # v[0] = 2.0
|
||||
v_mov_b32_e32(v[1], f2i(3.0)), # v[1] = 3.0
|
||||
# X: v[2] = fma(srcx0, SIMM32, vsrcx1) = v[0]*5.0+v[1] = 2*5+3 = 13
|
||||
# Y: v[3] = srcy0 (MOV) = v[0] = 2.0
|
||||
VOPD_LIT(VOPDOp.V_DUAL_FMAMK_F32, VOPDOp.V_DUAL_MOV_B32, v[2], v[3], v[0], v[0], v[1], v[0], literal=f2i(5.0)),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertAlmostEqual(i2f(st.vgpr[0][2]), 13.0, places=5, msg="fma(2.0, 5.0, 3.0) should be 13.0")
|
||||
|
||||
|
||||
class TestVOPDDot2Acc(unittest.TestCase):
|
||||
"""Tests for V_DUAL_DOT2ACC_F32_F16 - packed f16 dot product accumulate."""
|
||||
|
||||
def test_vopd_dot2acc_f32_f16_basic(self):
|
||||
"""V_DUAL_DOT2ACC_F32_F16: D += lo(S0)*lo(S1) + hi(S0)*hi(S1).
|
||||
|
||||
S0 = pack(1.0h, 2.0h), S1 = pack(3.0h, 4.0h), D = 10.0f
|
||||
result = 10.0 + 1.0*3.0 + 2.0*4.0 = 10.0 + 3.0 + 8.0 = 21.0
|
||||
"""
|
||||
from test.amd.hw.helpers import f2i, i2f, f32_to_f16
|
||||
pk_s0 = f32_to_f16(1.0) | (f32_to_f16(2.0) << 16) # lo=1.0h, hi=2.0h
|
||||
pk_s1 = f32_to_f16(3.0) | (f32_to_f16(4.0) << 16) # lo=3.0h, hi=4.0h
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], pk_s0),
|
||||
v_mov_b32_e32(v[1], pk_s1),
|
||||
v_mov_b32_e32(v[3], f2i(10.0)), # accumulator in v[3] (vdsty with vdstx=v[4])
|
||||
# X: v[4] = MOV v[0] (don't care), Y: v[3] += dot2(v[0], v[1])
|
||||
VOPD(VOPDOp.V_DUAL_MOV_B32, VOPDOp.V_DUAL_DOT2ACC_F32_F16, v[4], v[3], v[0], v[0], v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertAlmostEqual(i2f(st.vgpr[0][3]), 21.0, places=2, msg="10.0 + 1.0*3.0 + 2.0*4.0 = 21.0")
|
||||
|
||||
def test_vopd_dot2acc_f32_f16_zero_accum(self):
|
||||
"""V_DUAL_DOT2ACC_F32_F16 with zero accumulator — pure dot product.
|
||||
|
||||
S0 = pack(0.5h, -1.0h), S1 = pack(2.0h, 3.0h), D = 0.0f
|
||||
result = 0.0 + 0.5*2.0 + (-1.0)*3.0 = 1.0 - 3.0 = -2.0
|
||||
"""
|
||||
from test.amd.hw.helpers import f2i, i2f, f32_to_f16
|
||||
pk_s0 = f32_to_f16(0.5) | (f32_to_f16(-1.0) << 16)
|
||||
pk_s1 = f32_to_f16(2.0) | (f32_to_f16(3.0) << 16)
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], pk_s0),
|
||||
v_mov_b32_e32(v[1], pk_s1),
|
||||
v_mov_b32_e32(v[3], f2i(0.0)), # zero accumulator in v[3]
|
||||
VOPD(VOPDOp.V_DUAL_MOV_B32, VOPDOp.V_DUAL_DOT2ACC_F32_F16, v[4], v[3], v[0], v[0], v[0], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=1)
|
||||
self.assertAlmostEqual(i2f(st.vgpr[0][3]), -2.0, places=2, msg="0.5*2.0 + (-1.0)*3.0 = -2.0")
|
||||
|
||||
|
||||
class TestVOPDMultilane(unittest.TestCase):
|
||||
"""Tests for VOPD with multiple lanes."""
|
||||
|
||||
def test_vopd_multilane_mov_add(self):
|
||||
"""VOPD MOV and ADD with multiple active lanes - no register conflict."""
|
||||
instructions = [
|
||||
v_mov_b32_e32(v[0], 5),
|
||||
v_mov_b32_e32(v[1], 10),
|
||||
# X: v[2] = 100 (constant), Y: v[1] = v[0] + v[1] = 5 + 10 = 15
|
||||
# vdsty_reg = (vdsty << 1) | ((vdstx.offset & 1) ^ 1) = (0 << 1) | ((258 & 1) ^ 1) = 0 | 1 = 1
|
||||
VOPD(VOPDOp.V_DUAL_MOV_B32, VOPDOp.V_DUAL_ADD_NC_U32, v[2], v[0], 100, v[0], v[2], v[1]),
|
||||
]
|
||||
st = run_program(instructions, n_lanes=4)
|
||||
for lane in range(4):
|
||||
self.assertEqual(st.vgpr[lane][2], 100, f"Lane {lane}: v[2] should be 100")
|
||||
self.assertEqual(st.vgpr[lane][1], 15, f"Lane {lane}: v[1] should be 15 (5+10)")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
212
tinygrad_repo/test/amd/test_custom_kernel.py
Normal file
212
tinygrad_repo/test/amd/test_custom_kernel.py
Normal file
@@ -0,0 +1,212 @@
|
||||
import unittest
|
||||
import functools
|
||||
import numpy as np
|
||||
from tinygrad import Tensor, Device, dtypes
|
||||
from tinygrad.uop.ops import UOp, Ops, KernelInfo
|
||||
from tinygrad.engine.realize import run_linear, estimate_uop, compile_linear
|
||||
from tinygrad.renderer import Estimates
|
||||
from tinygrad.dtype import AddrSpace
|
||||
from tinygrad.helpers import getenv
|
||||
from tinygrad.runtime.autogen.amd.rdna3.ins import *
|
||||
import tinygrad.runtime.autogen.amd.rdna3.ins as r3
|
||||
import tinygrad.runtime.autogen.amd.rdna4.ins as r4
|
||||
from tinygrad.renderer.amd.dsl import s, v, NULL
|
||||
from test.amd.helpers import TARGET_TO_ARCH
|
||||
from extra.gemm.amd_asm_matmul import Kernel
|
||||
|
||||
def custom_add_one(A:UOp) -> UOp:
|
||||
A = A.flatten()
|
||||
assert dtypes.is_float(A.dtype.base), f"buffer dtype must be float32, got {A.dtype}"
|
||||
threads = UOp.special(A.numel(), "lidx0")
|
||||
insts = [
|
||||
s_load_b64(s[0:1], s[0:1], soffset=NULL),
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
v_lshlrev_b32_e32(v[0], 2, v[0]), # element offset
|
||||
global_load_b32(v[1], v[0], saddr=s[0:1]),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_mov_b32_e32(v[2], 1.0),
|
||||
v_add_f32_e32(v[1], v[1], v[2]),
|
||||
global_store_b32(addr=v[0], data=v[1], saddr=s[0:1]),
|
||||
s_endpgm(),
|
||||
]
|
||||
sink = UOp.sink(A.base, threads, arg=KernelInfo(f"custom_add_one_{A.numel()}", estimates=Estimates(ops=A.numel(), mem=A.numel()*4*2)))
|
||||
return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.DEVICE, arg="AMD"), UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in insts]))))
|
||||
|
||||
def custom_add_var(A:UOp, B:UOp) -> UOp:
|
||||
A,B = A.flatten(), B.flatten()
|
||||
assert A.dtype.base == dtypes.uint32, f"buffer dtype must be uint32, got {A.dtype}"
|
||||
threads = UOp.special(A.numel(), "lidx0")
|
||||
var = UOp.variable("var", 0, 10)
|
||||
insts = [
|
||||
s_load_b128(s[4:7], s[0:1]),
|
||||
s_load_b32(s[8], s[0:1], offset=0x10), # all threads load the same variable
|
||||
s_waitcnt_lgkmcnt(sdst=NULL, simm16=0),
|
||||
v_lshlrev_b32_e32(v[0], 2, v[0]), # element offset, different per thread
|
||||
global_load_b32(v[1], v[0], saddr=s[6:7]),
|
||||
s_waitcnt_vmcnt(sdst=NULL, simm16=0),
|
||||
v_add_nc_u32_e32(v[1], s[8], v[1]),
|
||||
global_store_b32(addr=v[0], data=v[1], saddr=s[4:5]),
|
||||
s_endpgm(),
|
||||
]
|
||||
sink = UOp.sink(A.base, B.base, var, threads, arg=KernelInfo(f"custom_add_var_{A.numel()}"))
|
||||
return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.DEVICE, arg="AMD"), UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in insts]))))
|
||||
|
||||
def custom_wave_sync(A:UOp, arch:str) -> UOp:
|
||||
# 4 waves across 1024 WG — enough to saturate a SIMD with many concurrent WGs
|
||||
# s_sleep yields the SIMD so waves from different WGs interleave, causing barrier packet reordering
|
||||
threads = UOp.special(128, "lidx0")
|
||||
wg = UOp.special(1024, "gidx0")
|
||||
insts = []
|
||||
for _ in range(4):
|
||||
insts.append(s_sleep(4))
|
||||
insts += [s_barrier()] if arch == "rdna3" else [r4.s_barrier_signal(), r4.s_barrier_wait()]
|
||||
insts += [s_nop(0)]*4
|
||||
insts.append(s_endpgm())
|
||||
sink = UOp.sink(A.base, threads, wg, arg=KernelInfo("custom_wave_sync"))
|
||||
return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.DEVICE, arg="AMD"), UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in insts]))))
|
||||
|
||||
def custom_lds_sync(A:UOp, arch:str) -> UOp:
|
||||
A = A.flatten()
|
||||
num_threads = A.shape[0]
|
||||
threads = UOp.special(num_threads, "lidx0")
|
||||
wg = UOp.special(1, "gidx0")
|
||||
lds = UOp(Ops.DEFINE_LOCAL, dtypes.uint8.ptr(size=512, addrspace=AddrSpace.LOCAL), (), 'lds') # 128 * 4 bytes
|
||||
isa = r4 if arch == "rdna4" else r3
|
||||
wait_kmcnt = [isa.s_wait_kmcnt(simm16=0)] if arch == "rdna4" else [isa.s_waitcnt_lgkmcnt(sdst=NULL, simm16=0)]
|
||||
wait_dscnt = [isa.s_wait_dscnt(simm16=0)] if arch == "rdna4" else [isa.s_waitcnt_lgkmcnt(sdst=NULL, simm16=0)]
|
||||
barrier = [isa.s_barrier_signal(ssrc0=-1), isa.s_barrier_wait(simm16=-1)] if arch == "rdna4" else [isa.s_barrier()]
|
||||
global_store = [isa.global_store_b32(vaddr=v[6:7], saddr=s[0:1], vsrc=v[5])] if arch == "rdna4" \
|
||||
else [isa.global_store_b32(addr=v[6], data=v[5], saddr=s[0:1])]
|
||||
insts = [
|
||||
isa.s_load_b64(s[0:1], s[0:1], soffset=NULL),
|
||||
*wait_kmcnt,
|
||||
isa.v_lshlrev_b32_e32(v[1], 2, v[0]),
|
||||
# lds[thread_idx] = thread_idx
|
||||
isa.ds_store_b32(addr=v[1], data0=v[0]),
|
||||
*wait_dscnt,
|
||||
*barrier,
|
||||
# out[threaed_idx] = thread_idx == num_threads ? -1 : lds[thread_idx + 1]
|
||||
isa.v_add_nc_u32_e32(v[2], 4, v[1]),
|
||||
isa.v_cmp_gt_u32_e32(num_threads-1, v[0]),
|
||||
isa.ds_load_b32(vdst=v[3], addr=v[2]),
|
||||
*wait_dscnt,
|
||||
isa.v_mov_b32_e32(v[4], -1),
|
||||
isa.v_cndmask_b32_e32(v[5], v[4], v[3]),
|
||||
isa.v_lshlrev_b32_e32(v[6], 2, v[0]),
|
||||
*global_store,
|
||||
isa.s_endpgm(),
|
||||
]
|
||||
sink = UOp.sink(A.base, lds, threads, wg, arg=KernelInfo("custom_lds_sync"))
|
||||
return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.DEVICE, arg="AMD"), UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in insts]))))
|
||||
|
||||
def custom_handwritten(A:UOp) -> UOp:
|
||||
A = A.flatten()
|
||||
threads = UOp.special(128, "lidx0")
|
||||
wg = UOp.special(1, "gidx0")
|
||||
lds = UOp(Ops.DEFINE_LOCAL, dtypes.uint8.ptr(size=512, addrspace=AddrSpace.LOCAL), (), 'lds') # 128 * 4 bytes
|
||||
pipes = {getenv("PIPE", "")} if getenv("PIPE", "") else {"SALU", "VALU", "TRANSCENDENTAL", "WMMA"}
|
||||
k = Kernel()
|
||||
# wrap in loop to filter out icache misses
|
||||
LOOP_N, UNROLL_N = 8, 5
|
||||
k.emit(r4.s_mov_b32(s[1], LOOP_N))
|
||||
k.label("loop")
|
||||
if "SALU" in pipes:
|
||||
for i in range(UNROLL_N):
|
||||
k.emit(r4.s_mov_b32(s[20+i], i))
|
||||
k.emit(r4.s_min_i32(s[30+i], i))
|
||||
k.emit(r4.s_mov_b32(s[40+i], i))
|
||||
k.emit(r4.s_mul_i32(s[14+i], s[12+i], 32))
|
||||
if "VALU" in pipes:
|
||||
for i in range(UNROLL_N):
|
||||
k.emit(r4.v_mov_b32_e32(v[20+i], i))
|
||||
k.emit(r4.v_lshlrev_b64_e32(v[30+2*i:31+2*i], 2, v[12+i:13+i]))
|
||||
k.emit(r4.v_mad_co_u64_u32(v[40+2*i:41+2*i], NULL, v[12+i], v[13+i], v[14+i:15+i]))
|
||||
if "TRANSCENDENTAL" in pipes:
|
||||
# transcendental VALU runs on the TFU, it can run regular VALU at the same time
|
||||
for i in range(UNROLL_N):
|
||||
k.emit(r4.v_mov_b32_e32(v[20+i], i))
|
||||
k.emit(r4.v_s_rcp_f32(s[20+i], s[12+i]))
|
||||
k.emit(r4.v_rcp_f32_e32(v[30+i], v[12+i]))
|
||||
k.emit(r4.v_s_exp_f32(s[30+i], s[12+i]))
|
||||
if "WMMA" in pipes:
|
||||
base = 30
|
||||
for i in range(UNROLL_N):
|
||||
a = base + i*40
|
||||
b, cd = a + 4, a + 8
|
||||
k.emit(r4.v_wmma_f32_16x16x16_f16(v[cd:cd+7], v[a:a+3], v[b:b+3], v[cd:cd+7]))
|
||||
a = base + i*40 + 16
|
||||
b, cd = a + 2, a + 4
|
||||
k.emit(r4.v_wmma_i32_16x16x16_iu8(v[cd:cd+7], v[a:a+1], v[b:b+1], v[cd:cd+7]))
|
||||
k.emit(r4.s_add_co_i32(s[1], s[1], -1))
|
||||
k.emit(r4.s_cmp_eq_i32(s[1], 0))
|
||||
k.emit(r4.s_cbranch_scc0(), target="loop")
|
||||
k.emit(r4.s_endpgm())
|
||||
insts = k.finalize()
|
||||
sink = UOp.sink(A.base, threads, wg, lds, arg=KernelInfo("custom_handwritten"))
|
||||
return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.DEVICE, arg="AMD"), UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in insts]))))
|
||||
|
||||
def custom_data_deps(A:UOp) -> UOp:
|
||||
A = A.flatten()
|
||||
threads = UOp.special(A.numel(), "lidx0")
|
||||
k = Kernel()
|
||||
k.emit(s_load_b64(s[0:1], s[0:1], soffset=NULL))
|
||||
k.emit(s_waitcnt_lgkmcnt(sdst=NULL, simm16=0))
|
||||
k.emit(v_lshlrev_b32_e32(v[0], 2, v[0]))
|
||||
k.emit(global_load_b32(v[1], v[0], saddr=s[0:1]))
|
||||
k.emit(s_waitcnt_vmcnt(sdst=NULL, simm16=0))
|
||||
k.emit(v_add_f32_e32(v[1], 1.0, v[1]))
|
||||
k.emit(global_store_b32(addr=v[0], data=v[1], saddr=s[0:1]))
|
||||
k.emit(s_endpgm())
|
||||
insts = k.finalize()
|
||||
sink = UOp.sink(A.base, threads, arg=KernelInfo("custom_data_deps"))
|
||||
return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.DEVICE, arg="AMD"), UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in insts]))))
|
||||
|
||||
@unittest.skipUnless(Device.DEFAULT == "AMD", "requires AMD device")
|
||||
class TestCustomKernel(unittest.TestCase):
|
||||
def setUp(self): self.arch = TARGET_TO_ARCH[Device["AMD"].arch]
|
||||
|
||||
def test_simple(self):
|
||||
if self.arch != "rdna3": self.skipTest("only rdna3")
|
||||
a = Tensor.full((16, 16), 1.).contiguous().realize()
|
||||
a = Tensor.custom_kernel(a, fxn=custom_add_one)[0]
|
||||
linear = compile_linear(a.schedule_linear())
|
||||
est = estimate_uop(linear.src[-1])
|
||||
self.assertEqual(est.ops, a.numel())
|
||||
self.assertEqual(est.mem, a.nbytes()*2)
|
||||
run_linear(linear)
|
||||
self.assertTrue((a.numpy() == 2.).all())
|
||||
|
||||
def test_variable(self):
|
||||
if self.arch != "rdna3": self.skipTest("only rdna3")
|
||||
b = Tensor.full((16, 16), 1, dtype=dtypes.uint32).contiguous().realize()
|
||||
a = Tensor.zeros_like(b).contiguous().realize()
|
||||
a = Tensor.custom_kernel(a, b, fxn=custom_add_var)[0]
|
||||
linear = a.schedule_linear()
|
||||
for i in range(4):
|
||||
run_linear(linear, var_vals={"var":i})
|
||||
self.assertTrue((a.numpy() == 1+i).all())
|
||||
|
||||
def test_lds_sync(self):
|
||||
if self.arch not in ("rdna3", "rdna4"): self.skipTest("only rdna3/rdna4")
|
||||
a = Tensor.empty(128, dtype=dtypes.int32).contiguous().realize()
|
||||
a = Tensor.custom_kernel(a, fxn=functools.partial(custom_lds_sync, arch=self.arch))[0]
|
||||
a.realize()
|
||||
ref = Tensor.arange(1, 129, dtype=dtypes.int32)
|
||||
ref[127] = -1
|
||||
self.assertListEqual(a.tolist(), ref.tolist())
|
||||
|
||||
def test_handwritten(self):
|
||||
if self.arch != "rdna4": self.skipTest("only tested on rdna4")
|
||||
a = Tensor.empty(1024, dtype=dtypes.int32).contiguous().realize()
|
||||
a = Tensor.custom_kernel(a, fxn=custom_handwritten)[0]
|
||||
a.realize()
|
||||
|
||||
def test_data_deps(self):
|
||||
if self.arch != "rdna3": self.skipTest("only tested on rdna3")
|
||||
a = Tensor(np.full(32, 5.0, dtype=np.float32)).realize()
|
||||
a = Tensor.custom_kernel(a, fxn=custom_data_deps)[0]
|
||||
a.realize()
|
||||
self.assertTrue((a.numpy() == 6.0).all())
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
170
tinygrad_repo/test/amd/test_dsl2.py
Normal file
170
tinygrad_repo/test/amd/test_dsl2.py
Normal file
@@ -0,0 +1,170 @@
|
||||
import unittest
|
||||
from tinygrad.renderer.amd.dsl import *
|
||||
from tinygrad.renderer.amd.dsl import VDSTYField
|
||||
from tinygrad.runtime.autogen.amd.rdna3.enum import VOP1Op, VOP2Op
|
||||
from tinygrad.runtime.autogen.amd.rdna3.ins import VOP1
|
||||
|
||||
class TestRegisters(unittest.TestCase):
|
||||
def test_vgpr_single(self):
|
||||
self.assertEqual(repr(v[5]), "v[5]")
|
||||
self.assertEqual(v[5].offset, 261) # 256 + 5
|
||||
self.assertEqual(v[5].sz, 1)
|
||||
|
||||
def test_sgpr_single(self):
|
||||
self.assertEqual(repr(s[10]), "s[10]")
|
||||
self.assertEqual(s[10].offset, 10)
|
||||
|
||||
def test_vgpr_range(self):
|
||||
self.assertEqual(repr(v[0:3]), "v[0:3]")
|
||||
self.assertEqual(v[0:3].offset, 256)
|
||||
self.assertEqual(v[0:3].sz, 4)
|
||||
|
||||
def test_sgpr_range(self):
|
||||
self.assertEqual(repr(s[4:5]), "s[4:5]")
|
||||
self.assertEqual(s[4:5].sz, 2)
|
||||
|
||||
def test_ttmp_reslice(self):
|
||||
# ttmp is src[108:123], so ttmp[0] should be src[108]
|
||||
self.assertEqual(ttmp[0].offset, 108)
|
||||
self.assertEqual(ttmp[1].offset, 109)
|
||||
# ttmp[0:1] is 2 elements (inclusive slicing)
|
||||
self.assertEqual(ttmp[0:1].offset, 108)
|
||||
self.assertEqual(ttmp[0:1].sz, 2)
|
||||
# ttmp[0:1][0] should be src[108]
|
||||
self.assertEqual(ttmp[0:1][0].offset, 108)
|
||||
|
||||
def test_special_regs(self):
|
||||
self.assertEqual(NULL.offset, 124)
|
||||
self.assertEqual(M0.offset, 125)
|
||||
self.assertEqual(EXEC_LO.offset, 126)
|
||||
self.assertEqual(EXEC_HI.offset, 127)
|
||||
# Check repr round-trips
|
||||
self.assertEqual(repr(NULL), "NULL")
|
||||
self.assertEqual(repr(M0), "M0")
|
||||
self.assertEqual(repr(EXEC_LO), "EXEC_LO")
|
||||
self.assertEqual(repr(EXEC), "EXEC")
|
||||
|
||||
def test_vcc(self):
|
||||
self.assertEqual(VCC.offset, 106)
|
||||
self.assertEqual(VCC.sz, 2)
|
||||
self.assertEqual(VCC_LO.offset, 106)
|
||||
self.assertEqual(VCC_HI.offset, 107)
|
||||
# Check repr round-trips
|
||||
self.assertEqual(repr(VCC_LO), "VCC_LO")
|
||||
self.assertEqual(repr(VCC_HI), "VCC_HI")
|
||||
self.assertEqual(repr(VCC), "VCC")
|
||||
|
||||
def test_float_constants(self):
|
||||
self.assertEqual(src[240].offset, 240)
|
||||
self.assertEqual(repr(src[240]), "0.5")
|
||||
self.assertEqual(repr(src[242]), "1.0")
|
||||
self.assertEqual(repr(src[243]), "-1.0")
|
||||
|
||||
def test_int_constants(self):
|
||||
self.assertEqual(repr(src[128]), "0")
|
||||
self.assertEqual(repr(src[129]), "1")
|
||||
self.assertEqual(repr(src[192]), "64")
|
||||
self.assertEqual(repr(src[193]), "-1")
|
||||
self.assertEqual(repr(src[208]), "-16")
|
||||
|
||||
class TestEnumBitField(unittest.TestCase):
|
||||
def test_enum_name(self):
|
||||
self.assertEqual(VOP1Op.V_MOV_B32_E32.name, "V_MOV_B32_E32")
|
||||
|
||||
def test_enum_value(self):
|
||||
self.assertEqual(VOP1Op.V_MOV_B32_E32.value, 1)
|
||||
|
||||
def test_enum_comparison(self):
|
||||
self.assertEqual(VOP1Op.V_MOV_B32_E32, VOP1Op.V_MOV_B32_E32)
|
||||
self.assertNotEqual(VOP1Op.V_NOP_E32, VOP1Op.V_MOV_B32_E32)
|
||||
|
||||
def test_enum_different_types(self):
|
||||
# VOP1Op and VOP2Op are different enums, even if same value
|
||||
self.assertNotEqual(VOP1Op.V_MOV_B32_E32, VOP2Op.V_CNDMASK_B32_E32)
|
||||
|
||||
def test_wrong_enum_type_raises(self):
|
||||
# Passing VOP2Op to VOP1 should raise
|
||||
with self.assertRaises(RuntimeError):
|
||||
VOP1(VOP2Op.V_CNDMASK_B32_E32, v[5], v[6])
|
||||
|
||||
class TestVOP1(unittest.TestCase):
|
||||
def test_class_setup(self):
|
||||
self.assertEqual(VOP1._size(), 4)
|
||||
field_names = [n for n, _ in VOP1._fields]
|
||||
self.assertIn('encoding', field_names)
|
||||
self.assertIn('op', field_names)
|
||||
self.assertIn('vdst', field_names)
|
||||
self.assertIn('src0', field_names)
|
||||
|
||||
def test_encoding_vgpr_vgpr(self):
|
||||
i = VOP1(VOP1Op.V_MOV_B32_E32, v[5], v[6])
|
||||
raw = i._raw
|
||||
# Check each field
|
||||
self.assertEqual((raw >> 25) & 0x7f, 0b0111111) # encoding
|
||||
self.assertEqual((raw >> 17) & 0xff, 5) # vdst (just VGPR index)
|
||||
self.assertEqual((raw >> 9) & 0xff, 1) # op
|
||||
self.assertEqual(raw & 0x1ff, 262) # src0 (256 + 6)
|
||||
|
||||
def test_encoding_vgpr_sgpr(self):
|
||||
i = VOP1(VOP1Op.V_MOV_B32_E32, v[5], s[10])
|
||||
raw = i._raw
|
||||
self.assertEqual((raw >> 17) & 0xff, 5) # vdst (just VGPR index)
|
||||
self.assertEqual(raw & 0x1ff, 10) # src0 (SGPR encoded)
|
||||
|
||||
def test_to_bytes(self):
|
||||
i = VOP1(VOP1Op.V_MOV_B32_E32, v[5], v[6])
|
||||
b = i.to_bytes()
|
||||
self.assertEqual(len(b), 4)
|
||||
self.assertEqual(int.from_bytes(b, 'little'), i._raw)
|
||||
|
||||
def test_from_bytes(self):
|
||||
i1 = VOP1(VOP1Op.V_MOV_B32_E32, v[5], v[6])
|
||||
i2 = VOP1.from_bytes(i1.to_bytes())
|
||||
self.assertEqual(i1._raw, i2._raw)
|
||||
|
||||
def test_repr(self):
|
||||
i = VOP1(VOP1Op.V_MOV_B32_E32, v[5], v[6])
|
||||
self.assertEqual(repr(i), "v_mov_b32_e32(v[5], v[6])")
|
||||
|
||||
def test_repr_sgpr_src(self):
|
||||
i = VOP1(VOP1Op.V_MOV_B32_E32, v[5], s[10])
|
||||
self.assertEqual(repr(i), "v_mov_b32_e32(v[5], s[10])")
|
||||
|
||||
def test_kwargs(self):
|
||||
i1 = VOP1(VOP1Op.V_MOV_B32_E32, v[5], v[6])
|
||||
i2 = VOP1(op=VOP1Op.V_MOV_B32_E32, vdst=v[5], src0=v[6])
|
||||
self.assertEqual(i1._raw, i2._raw)
|
||||
|
||||
def test_kwargs_partial(self):
|
||||
i1 = VOP1(VOP1Op.V_MOV_B32_E32, v[5], v[6])
|
||||
i2 = VOP1(VOP1Op.V_MOV_B32_E32, src0=v[6], vdst=v[5])
|
||||
self.assertEqual(i1._raw, i2._raw)
|
||||
|
||||
class TestVDSTYField(unittest.TestCase):
|
||||
def test_encode_even_vgpr(self):
|
||||
f = VDSTYField(6, 0) # 7-bit field
|
||||
self.assertEqual(f.encode(v[0]), 0)
|
||||
self.assertEqual(f.encode(v[2]), 1)
|
||||
self.assertEqual(f.encode(v[4]), 2)
|
||||
self.assertEqual(f.encode(v[254]), 127)
|
||||
|
||||
def test_encode_non_vgpr_raises(self):
|
||||
f = VDSTYField(6, 0)
|
||||
with self.assertRaises(ValueError) as ctx:
|
||||
f.encode(s[0])
|
||||
self.assertIn("VGPR", str(ctx.exception))
|
||||
|
||||
def test_encode_non_reg_raises(self):
|
||||
f = VDSTYField(6, 0)
|
||||
with self.assertRaises(TypeError) as ctx:
|
||||
f.encode(42)
|
||||
self.assertIn("Reg", str(ctx.exception))
|
||||
|
||||
def test_decode_returns_raw(self):
|
||||
f = VDSTYField(6, 0)
|
||||
# decode returns raw value, actual vdsty computed with vdstx context
|
||||
self.assertEqual(f.decode(0), 0)
|
||||
self.assertEqual(f.decode(127), 127)
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
396
tinygrad_repo/test/amd/test_emu2_pcode.py
Normal file
396
tinygrad_repo/test/amd/test_emu2_pcode.py
Normal file
@@ -0,0 +1,396 @@
|
||||
"""Tests for the pcode parser."""
|
||||
import unittest
|
||||
from collections import defaultdict
|
||||
from tinygrad.helpers import DEBUG
|
||||
from tinygrad.dtype import dtypes
|
||||
from tinygrad.uop.ops import UOp, Ops
|
||||
from test.mockgpu.amd.emu import parse_pcode
|
||||
from test.mockgpu.amd.pcode import parse_expr
|
||||
from tinygrad.runtime.autogen.amd.rdna3.str_pcode import PCODE
|
||||
from tinygrad.runtime.autogen.amd.rdna3.enum import VOP1Op, VOP2Op, SOP2Op, DSOp, GLOBALOp
|
||||
|
||||
def _srcs():
|
||||
"""Create minimal source variables for pcode parsing."""
|
||||
def u32(v=0): return UOp.const(dtypes.uint32, v)
|
||||
return {'S0': u32(), 'S1': u32(), 'S2': u32(), 'SCC': u32(), 'VCC': UOp.const(dtypes.uint64, 0), 'laneId': u32()}
|
||||
|
||||
class TestBasicParsing(unittest.TestCase):
|
||||
"""Test basic pcode parsing for common instruction patterns."""
|
||||
|
||||
def test_v_add_f32(self):
|
||||
"""Test parsing V_ADD_F32 pcode."""
|
||||
_, assigns = parse_pcode(PCODE[VOP2Op.V_ADD_F32_E32], _srcs())
|
||||
self.assertEqual(len(assigns), 1)
|
||||
dest, _ = assigns[0]
|
||||
self.assertTrue(dest.startswith('D0'))
|
||||
|
||||
def test_v_lshlrev_b32(self):
|
||||
"""Test parsing V_LSHLREV_B32 pcode."""
|
||||
_, assigns = parse_pcode(PCODE[VOP2Op.V_LSHLREV_B32_E32], _srcs())
|
||||
self.assertEqual(len(assigns), 1)
|
||||
|
||||
def test_s_cselect_b32(self):
|
||||
"""Test parsing S_CSELECT_B32 pcode with ternary."""
|
||||
_, assigns = parse_pcode(PCODE[SOP2Op.S_CSELECT_B32], _srcs())
|
||||
self.assertEqual(len(assigns), 1)
|
||||
|
||||
def test_v_add_co_ci_u32(self):
|
||||
"""Test parsing V_ADD_CO_CI_U32 with carry."""
|
||||
_, assigns = parse_pcode(PCODE[VOP2Op.V_ADD_CO_CI_U32_E32], _srcs())
|
||||
self.assertGreaterEqual(len(assigns), 1)
|
||||
|
||||
class TestWithSources(unittest.TestCase):
|
||||
"""Test pcode parsing with actual source operand values."""
|
||||
|
||||
def test_v_add_f32_with_sources(self):
|
||||
"""Test V_ADD_F32 with actual float constants."""
|
||||
s0 = UOp.const(dtypes.uint32, 0x3f800000) # 1.0f
|
||||
s1 = UOp.const(dtypes.uint32, 0x40000000) # 2.0f
|
||||
_, assigns = parse_pcode(PCODE[VOP2Op.V_ADD_F32_E32], {'S0': s0, 'S1': s1})
|
||||
self.assertEqual(len(assigns), 1)
|
||||
dest, val = assigns[0]
|
||||
self.assertTrue(dest.startswith('D0'))
|
||||
# Result should be an ADD operation
|
||||
self.assertEqual(val.op, Ops.ADD)
|
||||
|
||||
def test_v_mul_f32_with_sources(self):
|
||||
"""Test V_MUL_F32 with actual float constants."""
|
||||
s0 = UOp.const(dtypes.uint32, 0x40000000) # 2.0f
|
||||
s1 = UOp.const(dtypes.uint32, 0x40400000) # 3.0f
|
||||
_, assigns = parse_pcode(PCODE[VOP2Op.V_MUL_F32_E32], {'S0': s0, 'S1': s1})
|
||||
self.assertEqual(len(assigns), 1)
|
||||
dest, val = assigns[0]
|
||||
self.assertEqual(val.op, Ops.MUL)
|
||||
|
||||
class TestParseExpr(unittest.TestCase):
|
||||
"""Test the parse_expr function directly."""
|
||||
|
||||
def test_integer_literals(self):
|
||||
"""Test parsing integer literals."""
|
||||
self.assertEqual(parse_expr('0', {}).arg, 0)
|
||||
self.assertEqual(parse_expr('42', {}).arg, 42)
|
||||
self.assertEqual(parse_expr('42U', {}).arg, 42)
|
||||
|
||||
def test_negative_integers(self):
|
||||
"""Test parsing negative integer literals."""
|
||||
result = parse_expr('-1', {})
|
||||
self.assertEqual(result.arg, -1)
|
||||
self.assertEqual(result.dtype, dtypes.int)
|
||||
|
||||
def test_float_literals(self):
|
||||
"""Test parsing float literals."""
|
||||
result = parse_expr('1.0F', {})
|
||||
self.assertEqual(result.arg, 1.0)
|
||||
self.assertEqual(result.dtype, dtypes.float32)
|
||||
|
||||
def test_hex_literals(self):
|
||||
"""Test parsing hex literals."""
|
||||
result = parse_expr('0xFF', {})
|
||||
self.assertEqual(result.arg, 255)
|
||||
|
||||
def test_variable_lookup(self):
|
||||
"""Test variable lookup in parse_expr."""
|
||||
vrs = {'x': UOp.const(dtypes.uint32, 42)}
|
||||
result = parse_expr('x', vrs)
|
||||
self.assertEqual(result.arg, 42)
|
||||
|
||||
def test_binary_ops(self):
|
||||
"""Test parsing binary operations."""
|
||||
vrs = {'a': UOp.const(dtypes.uint32, 10), 'b': UOp.const(dtypes.uint32, 5)}
|
||||
|
||||
# Addition
|
||||
result = parse_expr('a + b', vrs)
|
||||
self.assertEqual(result.op, Ops.ADD)
|
||||
|
||||
# Subtraction with constant folding
|
||||
result = parse_expr('10 - 5', {})
|
||||
self.assertEqual(result.op, Ops.CONST)
|
||||
self.assertEqual(result.arg, 5)
|
||||
|
||||
def test_ternary(self):
|
||||
"""Test parsing ternary expressions."""
|
||||
vrs = {'cond': UOp.const(dtypes.bool, True), 'a': UOp.const(dtypes.uint32, 1), 'b': UOp.const(dtypes.uint32, 0)}
|
||||
result = parse_expr('cond ? a : b', vrs)
|
||||
self.assertEqual(result.op, Ops.WHERE)
|
||||
|
||||
|
||||
class TestForLoopParsing(unittest.TestCase):
|
||||
"""Test for loop parsing (CLZ/CTZ patterns)."""
|
||||
|
||||
def test_clz_pcode_exists(self):
|
||||
"""Verify CLZ pcode is available."""
|
||||
pcode = PCODE.get(VOP1Op.V_CLZ_I32_U32_E32)
|
||||
self.assertIsNotNone(pcode)
|
||||
assert pcode is not None
|
||||
self.assertIn('for', pcode.lower())
|
||||
|
||||
def test_clz_parsing(self):
|
||||
"""Test CLZ pcode parsing produces correct structure."""
|
||||
pcode = PCODE[VOP1Op.V_CLZ_I32_U32_E32]
|
||||
S0 = UOp.const(dtypes.uint32, 0xFFFFFFFF) # All ones - CLZ should be 0
|
||||
_vrs, assigns = parse_pcode(pcode, {'S0': S0})
|
||||
|
||||
self.assertEqual(len(assigns), 1)
|
||||
dest, val = assigns[0]
|
||||
self.assertTrue(dest.startswith('D0'))
|
||||
# Result should be a nested WHERE structure
|
||||
self.assertEqual(val.op, Ops.WHERE)
|
||||
|
||||
def test_clz_with_zero(self):
|
||||
"""Test CLZ with input 0 - should return -1."""
|
||||
pcode = PCODE[VOP1Op.V_CLZ_I32_U32_E32]
|
||||
S0 = UOp.const(dtypes.uint32, 0)
|
||||
_vrs, assigns = parse_pcode(pcode, {'S0': S0})
|
||||
|
||||
# Check that the innermost value (default) is -1 (may be wrapped in CAST)
|
||||
val = assigns[0][1]
|
||||
# Traverse to innermost WHERE
|
||||
while val.op == Ops.WHERE:
|
||||
val = val.src[2] # false branch
|
||||
# Unwrap CAST if present
|
||||
while val.op == Ops.CAST:
|
||||
val = val.src[0]
|
||||
self.assertEqual(val.arg, -1)
|
||||
|
||||
def test_ctz_parsing(self):
|
||||
"""Test CTZ pcode parsing."""
|
||||
pcode = PCODE.get(VOP1Op.V_CTZ_I32_B32_E32)
|
||||
if pcode is None:
|
||||
self.skipTest("V_CTZ_I32_B32_E32 pcode not available")
|
||||
|
||||
S0 = UOp.const(dtypes.uint32, 1) # LSB set - CTZ should be 0
|
||||
_vrs, assigns = parse_pcode(pcode, {'S0': S0})
|
||||
self.assertEqual(len(assigns), 1)
|
||||
|
||||
class TestDSPcodePatterns(unittest.TestCase):
|
||||
"""Test DS instruction pcode patterns."""
|
||||
|
||||
def test_global_atomic_add_f32_parsing(self):
|
||||
"""Test GLOBAL_ATOMIC_ADD_F32 keeps memory values in float dtype."""
|
||||
vmem = UOp.param(2, dtypes.uint32.ptr(1024))
|
||||
srcs = {
|
||||
'ADDR': UOp.const(dtypes.uint64, 0),
|
||||
'DATA': UOp.const(dtypes.uint32, 0x3f800000),
|
||||
'_vmem': vmem,
|
||||
}
|
||||
|
||||
_, assigns = parse_pcode(PCODE[GLOBALOp.GLOBAL_ATOMIC_ADD_F32], srcs)
|
||||
mem_write = next(val for dest, val in assigns if dest == 'MEM[ADDR].f32')
|
||||
self.assertEqual(mem_write[1].op, Ops.ADD) # type: ignore[index]
|
||||
self.assertEqual(mem_write[1].dtype, dtypes.float32) # type: ignore[index]
|
||||
|
||||
def test_ds_load_b32_pcode(self):
|
||||
"""Test DS_LOAD_B32 pcode is parseable."""
|
||||
pcode = PCODE.get(DSOp.DS_LOAD_B32)
|
||||
self.assertIsNotNone(pcode)
|
||||
assert pcode is not None
|
||||
self.assertIn('RETURN_DATA', pcode)
|
||||
self.assertIn('MEM[', pcode)
|
||||
|
||||
def test_ds_store_b32_pcode(self):
|
||||
"""Test DS_STORE_B32 pcode is parseable."""
|
||||
pcode = PCODE.get(DSOp.DS_STORE_B32)
|
||||
self.assertIsNotNone(pcode)
|
||||
assert pcode is not None
|
||||
self.assertIn('MEM[', pcode)
|
||||
self.assertIn('DATA', pcode)
|
||||
|
||||
def test_mem_read_parsing(self):
|
||||
"""Test MEM[addr].type read expression parsing."""
|
||||
# Create a mock LDS buffer
|
||||
lds = UOp.param(3, dtypes.uint32.ptr(16384))
|
||||
addr = UOp.const(dtypes.uint32, 0)
|
||||
vrs = {'_lds': lds, 'ADDR': addr, 'OFFSET': UOp.const(dtypes.uint32, 0)}
|
||||
|
||||
result = parse_expr('MEM[ADDR + OFFSET].b32', vrs)
|
||||
# Should be an INDEX operation into LDS
|
||||
self.assertIsNotNone(result)
|
||||
|
||||
def test_ds_store_2addr_b32_parsing(self):
|
||||
"""Test DS_STORE_2ADDR_B32 pcode parsing produces MEM writes."""
|
||||
pcode = PCODE.get(DSOp.DS_STORE_2ADDR_B32)
|
||||
self.assertIsNotNone(pcode)
|
||||
assert pcode is not None
|
||||
srcs = {
|
||||
'ADDR': UOp.const(dtypes.uint32, 0),
|
||||
'OFFSET0': UOp.const(dtypes.uint32, 0),
|
||||
'OFFSET1': UOp.const(dtypes.uint32, 1),
|
||||
'DATA': UOp.const(dtypes.uint32, 0xAAAAAAAA),
|
||||
'DATA2': UOp.const(dtypes.uint32, 0xBBBBBBBB),
|
||||
}
|
||||
srcs['laneId'] = UOp.const(dtypes.uint32, 0)
|
||||
_, assigns = parse_pcode(pcode, srcs)
|
||||
# Should have 2 MEM write assignments
|
||||
self.assertEqual(len(assigns), 2)
|
||||
for dest, val in assigns:
|
||||
self.assertTrue(dest.startswith('MEM['))
|
||||
# val should be (addr, write_val) tuple
|
||||
self.assertIsInstance(val, tuple)
|
||||
self.assertEqual(len(val), 2) # type: ignore[arg-type]
|
||||
|
||||
def test_ds_load_2addr_b32_parsing(self):
|
||||
"""Test DS_LOAD_2ADDR_B32 pcode parsing produces RETURN_DATA assignments."""
|
||||
pcode = PCODE.get(DSOp.DS_LOAD_2ADDR_B32)
|
||||
self.assertIsNotNone(pcode)
|
||||
assert pcode is not None
|
||||
lds = UOp.param(3, dtypes.uint32.ptr(16384))
|
||||
srcs = {
|
||||
'ADDR': UOp.const(dtypes.uint32, 0),
|
||||
'OFFSET0': UOp.const(dtypes.uint32, 0),
|
||||
'OFFSET1': UOp.const(dtypes.uint32, 1),
|
||||
'_lds': lds,
|
||||
}
|
||||
srcs['laneId'] = UOp.const(dtypes.uint32, 0)
|
||||
_, assigns = parse_pcode(pcode, srcs)
|
||||
# Should have 2 RETURN_DATA assignments
|
||||
self.assertEqual(len(assigns), 2)
|
||||
self.assertEqual(assigns[0][0], 'RETURN_DATA[31:0]')
|
||||
self.assertEqual(assigns[1][0], 'RETURN_DATA[63:32]')
|
||||
|
||||
def test_ds_store_address_calculation(self):
|
||||
"""Test DS_STORE_2ADDR_B32 calculates correct addresses (offset * 4)."""
|
||||
pcode = PCODE.get(DSOp.DS_STORE_2ADDR_B32)
|
||||
assert pcode is not None
|
||||
srcs = {
|
||||
'ADDR': UOp.const(dtypes.uint32, 100),
|
||||
'OFFSET0': UOp.const(dtypes.uint32, 2),
|
||||
'OFFSET1': UOp.const(dtypes.uint32, 5),
|
||||
'DATA': UOp.const(dtypes.uint32, 0xAAAAAAAA),
|
||||
'DATA2': UOp.const(dtypes.uint32, 0xBBBBBBBB),
|
||||
}
|
||||
srcs['laneId'] = UOp.const(dtypes.uint32, 0)
|
||||
_, assigns = parse_pcode(pcode, srcs)
|
||||
# Check addresses: 100 + 2*4 = 108, 100 + 5*4 = 120
|
||||
# assigns[i][1] is (addr, val) tuple for MEM writes; mypy sees UOp
|
||||
self.assertEqual(assigns[0][1][0].simplify().arg, 108) # type: ignore[index]
|
||||
self.assertEqual(assigns[1][1][0].simplify().arg, 120) # type: ignore[index]
|
||||
|
||||
def test_ds_store_data_values(self):
|
||||
"""Test DS_STORE_2ADDR_B32 uses correct data values."""
|
||||
pcode = PCODE.get(DSOp.DS_STORE_2ADDR_B32)
|
||||
assert pcode is not None
|
||||
srcs = {
|
||||
'ADDR': UOp.const(dtypes.uint32, 0),
|
||||
'OFFSET0': UOp.const(dtypes.uint32, 0),
|
||||
'OFFSET1': UOp.const(dtypes.uint32, 1),
|
||||
'DATA': UOp.const(dtypes.uint32, 0xAAAAAAAA),
|
||||
'DATA2': UOp.const(dtypes.uint32, 0xBBBBBBBB),
|
||||
}
|
||||
srcs['laneId'] = UOp.const(dtypes.uint32, 0)
|
||||
_, assigns = parse_pcode(pcode, srcs)
|
||||
# assigns[i][1] is (addr, val) tuple for MEM writes; mypy sees UOp
|
||||
# DATA[31:0] should preserve the value
|
||||
self.assertEqual(assigns[0][1][1].simplify().arg, 0xAAAAAAAA) # type: ignore[index]
|
||||
self.assertEqual(assigns[1][1][1].simplify().arg, 0xBBBBBBBB) # type: ignore[index]
|
||||
|
||||
class TestConditionalParsing(unittest.TestCase):
|
||||
"""Test conditional (if/elsif/else) pcode parsing."""
|
||||
|
||||
def test_ternary_in_assignment(self):
|
||||
"""Test parsing ternary expression (which becomes WHERE)."""
|
||||
# S_CSELECT_B32: D0.u32 = SCC ? S0.u32 : S1.u32
|
||||
pcode = PCODE[SOP2Op.S_CSELECT_B32]
|
||||
s0 = UOp.const(dtypes.uint32, 10)
|
||||
s1 = UOp.const(dtypes.uint32, 20)
|
||||
scc = UOp.const(dtypes.uint32, 1)
|
||||
_vrs, assigns = parse_pcode(pcode, {'S0': s0, 'S1': s1, 'SCC': scc})
|
||||
self.assertEqual(len(assigns), 1)
|
||||
dest, val = assigns[0]
|
||||
self.assertTrue(dest.startswith('D0'))
|
||||
# Result should be a WHERE (ternary becomes WHERE)
|
||||
self.assertEqual(val.op, Ops.WHERE)
|
||||
|
||||
class TestConcatWidthParsing(unittest.TestCase):
|
||||
"""Test that bit extracts keep the right width for concat/unary ops."""
|
||||
|
||||
def test_permlanex16_altrow_concat(self):
|
||||
for row, expected in [(0, 1), (1, 0), (2, 3), (3, 2)]:
|
||||
parsed = parse_expr('{ row[1], ~row[0] }', {'row': UOp.const(dtypes.uint32, row)})
|
||||
self.assertEqual(parsed.simplify().arg, expected)
|
||||
|
||||
def test_permlane64_altlane_concat(self):
|
||||
for lane, expected in [(0, 32), (1, 33), (31, 63), (32, 0), (63, 31)]:
|
||||
parsed = parse_expr('{ ~lane[5], lane[4:0] }', {'lane': UOp.const(dtypes.uint32, lane)})
|
||||
self.assertEqual(parsed.simplify().arg, expected)
|
||||
|
||||
def test_permlane64_wave64_pcode_indices(self):
|
||||
vgpr = UOp.param(0, dtypes.uint32.ptr(256))
|
||||
srcs = {
|
||||
'SRC0': UOp.const(dtypes.uint32, 0),
|
||||
'VDST': UOp.const(dtypes.uint32, 1),
|
||||
'EXEC_LO': UOp.const(dtypes.uint32, 0xFFFFFFFF),
|
||||
'EXEC': UOp.const(dtypes.uint64, 0xFFFFFFFFFFFFFFFF),
|
||||
'_vgpr': vgpr,
|
||||
'_wave_size': 64,
|
||||
'S0': UOp.const(dtypes.uint32, 0),
|
||||
'S1': UOp.const(dtypes.uint32, 0),
|
||||
'S2': UOp.const(dtypes.uint32, 0),
|
||||
}
|
||||
|
||||
def load_idx(v: UOp) -> int:
|
||||
simp = v.simplify()
|
||||
self.assertEqual(simp.op, Ops.LOAD)
|
||||
self.assertEqual(simp.src[0].op, Ops.INDEX)
|
||||
idx = simp.src[0].src[1].simplify()
|
||||
self.assertEqual(idx.op, Ops.CONST)
|
||||
return idx.arg
|
||||
|
||||
_, assigns = parse_pcode(PCODE[VOP1Op.V_PERMLANE64_B32_E32], srcs)
|
||||
self.assertEqual(len(assigns), 64)
|
||||
for lane, (dst_idx, src_idx) in {0: (64, 32), 31: (95, 63), 32: (96, 0), 63: (127, 31)}.items():
|
||||
self.assertEqual(assigns[lane][1][0].simplify().arg, dst_idx) # type: ignore[index]
|
||||
self.assertEqual(load_idx(assigns[lane][1][1]), src_idx) # type: ignore[index]
|
||||
|
||||
class TestAllPcode(unittest.TestCase):
|
||||
"""Test that all pcode from all architectures can be parsed."""
|
||||
|
||||
def _make_srcs(self):
|
||||
"""Create dummy source variables for pcode parsing."""
|
||||
u32, u64 = lambda v=0: UOp.const(dtypes.uint32, v), lambda v=0: UOp.const(dtypes.uint64, v)
|
||||
lds = UOp.param(3, dtypes.uint32.ptr(16384))
|
||||
return {'laneId': u32(), 'laneID': u32(), 'S0': u32(), 'S1': u32(), 'S2': u32(), 'S3': u32(), 'SRC0': u32(),
|
||||
'D0': u32(), 'D1': u32(), 'DST': u32(), 'VDST': u32(), 'SDST': u32(),
|
||||
'VCC': u64(), 'VCCZ': u32(), 'EXEC': u64(), 'EXEC_LO': u32(), 'EXECZ': u32(), 'SCC': u32(),
|
||||
'SIMM16': u32(), 'SIMM32': u32(), 'OFFSET': u32(), 'OFFSET0': u32(), 'OFFSET1': u32(), 'offset1': u32(),
|
||||
'ADDR': u32(), 'ADDR_BASE': u32(), 'TADDR': u32(), 'DATA': u32(), 'DATA0': u32(), 'DATA1': u32(), 'DATA2': u32(),
|
||||
'VDATA': u32(), 'VDATA0': u32(), 'VDATA1': u32(), 'VDATA2': u32(), 'VDATA3': u32(),
|
||||
'OPSEL': u32(), 'OPSEL_HI': u32(), 'NEG': u32(), 'NEG_HI': u32(), 'CLAMP': u32(),
|
||||
'M0': u32(), 'PC': u64(), 'DENORM': u32(1), 'ROUND_MODE': u32(), 'ROUND_TOWARD_ZERO': u32(),
|
||||
'ROUND_NEAREST_EVEN': u32(), 'WAVE_STATUS': u32(),
|
||||
'MAX_FLOAT_F32': u32(0x7f7fffff), 'Unsigned': u32(1), 'clampedLOD': u32(),
|
||||
'_lds': lds, '_vmem': lds, '_active': UOp.const(dtypes.bool, True)}
|
||||
|
||||
def _parse_all_pcode(self, pcode_dict, arch: str, min_pct: float):
|
||||
"""Parse all pcode. RuntimeError = parser limitation (ok), other exceptions = real bugs."""
|
||||
srcs = self._make_srcs()
|
||||
passed, skipped, errors = 0, 0, defaultdict(list)
|
||||
for op, pcode in pcode_dict.items():
|
||||
try:
|
||||
parse_pcode(pcode, srcs)
|
||||
passed += 1
|
||||
except RuntimeError as e:
|
||||
skipped += 1
|
||||
errors[str(e)].append(op.name)
|
||||
except Exception as e: self.fail(f"[{arch}] {op.name}: {e}\nPcode: {pcode[:200]}")
|
||||
total = len(pcode_dict)
|
||||
pct = 100 * passed / total
|
||||
print(f"{arch}: {passed}/{total} ({pct:.1f}%) parsed, {skipped} skipped")
|
||||
if DEBUG >= 2:
|
||||
for err, ops in sorted(errors.items(), key=lambda x: -len(x[1])):
|
||||
print(f" {err}: {', '.join(ops[:5])}{'...' if len(ops) > 5 else ''} ({len(ops)})")
|
||||
self.assertGreaterEqual(pct, min_pct, f"[{arch}] {pct:.1f}% < {min_pct}% threshold")
|
||||
|
||||
def test_parse_all_cdna_pcode(self):
|
||||
from tinygrad.runtime.autogen.amd.cdna.str_pcode import PCODE as CDNA_PCODE
|
||||
self._parse_all_pcode(CDNA_PCODE, "CDNA", min_pct=60)
|
||||
|
||||
def test_parse_all_rdna3_pcode(self):
|
||||
from tinygrad.runtime.autogen.amd.rdna3.str_pcode import PCODE as RDNA3_PCODE
|
||||
self._parse_all_pcode(RDNA3_PCODE, "RDNA3", min_pct=90)
|
||||
|
||||
def test_parse_all_rdna4_pcode(self):
|
||||
from tinygrad.runtime.autogen.amd.rdna4.str_pcode import PCODE as RDNA4_PCODE
|
||||
self._parse_all_pcode(RDNA4_PCODE, "RDNA4", min_pct=65)
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
229
tinygrad_repo/test/amd/test_formats.py
Normal file
229
tinygrad_repo/test/amd/test_formats.py
Normal file
@@ -0,0 +1,229 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Test DS and other compute-relevant instruction formats.
|
||||
|
||||
Note: Graphics-only formats (EXP, MUBUF, MTBUF, MIMG) are not supported - use GLOBAL/FLAT for memory access in compute.
|
||||
"""
|
||||
import unittest
|
||||
from tinygrad.runtime.autogen.amd.rdna3.ins import *
|
||||
from tinygrad.renderer.amd.dsl import VCC_HI, EXEC_LO, NULL
|
||||
OFF = NULL # OFF is alias for NULL
|
||||
from tinygrad.renderer.amd import detect_format
|
||||
|
||||
|
||||
class TestDS(unittest.TestCase):
|
||||
"""Test DS (data share / LDS) instructions."""
|
||||
|
||||
def test_ds_store_b32(self):
|
||||
# ds_store_b32 v0, v1
|
||||
# GFX11: encoding: [0x00,0x00,0x34,0xd8,0x00,0x01,0x00,0x00]
|
||||
inst = ds_store_b32(addr=v[0], data0=v[1])
|
||||
self.assertEqual(inst.to_bytes(), bytes([0x00,0x00,0x34,0xd8,0x00,0x01,0x00,0x00]))
|
||||
|
||||
def test_ds_load_b32(self):
|
||||
# ds_load_b32 v0, v1
|
||||
# GFX11: encoding: [0x00,0x00,0xd8,0xd8,0x01,0x00,0x00,0x00]
|
||||
inst = ds_load_b32(vdst=v[0], addr=v[1])
|
||||
self.assertEqual(inst.to_bytes(), bytes([0x00,0x00,0xd8,0xd8,0x01,0x00,0x00,0x00]))
|
||||
|
||||
def test_ds_store_b32_offset(self):
|
||||
# ds_store_b32 v0, v1 offset:64
|
||||
# GFX11: encoding: [0x40,0x00,0x34,0xd8,0x00,0x01,0x00,0x00]
|
||||
inst = ds_store_b32(addr=v[0], data0=v[1], offset0=64)
|
||||
self.assertEqual(inst.to_bytes(), bytes([0x40,0x00,0x34,0xd8,0x00,0x01,0x00,0x00]))
|
||||
|
||||
def test_ds_load_b64(self):
|
||||
# ds_load_b64 v[0:1], v2
|
||||
# GFX11: encoding: [0x00,0x00,0xd8,0xd9,0x02,0x00,0x00,0x00]
|
||||
inst = ds_load_b64(vdst=v[0:1], addr=v[2])
|
||||
self.assertEqual(inst.to_bytes(), bytes([0x00,0x00,0xd8,0xd9,0x02,0x00,0x00,0x00]))
|
||||
|
||||
def test_ds_add_u32(self):
|
||||
# ds_add_u32 v0, v1
|
||||
# GFX11: encoding: [0x00,0x00,0x00,0xd8,0x00,0x01,0x00,0x00]
|
||||
inst = ds_add_u32(addr=v[0], data0=v[1])
|
||||
self.assertEqual(inst.to_bytes(), bytes([0x00,0x00,0x00,0xd8,0x00,0x01,0x00,0x00]))
|
||||
|
||||
def test_ds_store_b32_gds(self):
|
||||
# ds_store_b32 v0, v1 gds
|
||||
# GFX11: encoding: [0x00,0x00,0x36,0xd8,0x00,0x01,0x00,0x00]
|
||||
inst = ds_store_b32(addr=v[0], data0=v[1], gds=1)
|
||||
self.assertEqual(inst.to_bytes(), bytes([0x00,0x00,0x36,0xd8,0x00,0x01,0x00,0x00]))
|
||||
|
||||
|
||||
class TestVOP3(unittest.TestCase):
|
||||
"""Test VOP3 (3-operand vector) instructions."""
|
||||
|
||||
def test_v_fma_f32(self):
|
||||
# v_fma_f32 v0, v1, v2, v3
|
||||
# GFX11: encoding: [0x00,0x00,0x13,0xd6,0x01,0x05,0x0e,0x04]
|
||||
inst = v_fma_f32(vdst=v[0], src0=v[1], src1=v[2], src2=v[3])
|
||||
self.assertEqual(inst.to_bytes(), bytes([0x00,0x00,0x13,0xd6,0x01,0x05,0x0e,0x04]))
|
||||
|
||||
def test_v_mad_f32(self):
|
||||
# v_fmac_f32_e64 v0, v1, v2 (fmac is fma with implicit dst as src2)
|
||||
# Use v_fma_f32 with vdst == src2
|
||||
inst = v_fma_f32(vdst=v[0], src0=v[1], src1=v[2], src2=v[0])
|
||||
self.assertEqual(inst.to_bytes()[:4], bytes([0x00,0x00,0x13,0xd6]))
|
||||
|
||||
def test_v_add3_u32(self):
|
||||
# v_add3_u32 v0, v1, v2, v3
|
||||
# GFX11: encoding: [0x00,0x00,0x55,0xd6,0x01,0x05,0x0e,0x04]
|
||||
inst = v_add3_u32(vdst=v[0], src0=v[1], src1=v[2], src2=v[3])
|
||||
self.assertEqual(inst.to_bytes(), bytes([0x00,0x00,0x55,0xd6,0x01,0x05,0x0e,0x04]))
|
||||
|
||||
|
||||
class TestFLAT(unittest.TestCase):
|
||||
"""Test FLAT/GLOBAL/SCRATCH memory instructions."""
|
||||
|
||||
def test_global_load_b32(self):
|
||||
# global_load_b32 v0, v[1:2], off (seg=2 for global)
|
||||
# GFX11: encoding: [0x00,0x00,0x52,0xdc,0x01,0x00,0x7c,0x00]
|
||||
inst = global_load_b32(vdst=v[0], addr=v[1:2], saddr=OFF)
|
||||
self.assertEqual(inst.to_bytes(), bytes([0x00,0x00,0x52,0xdc,0x01,0x00,0x7c,0x00]))
|
||||
|
||||
def test_global_store_b32(self):
|
||||
# global_store_b32 v[0:1], v2, off (seg=2 for global)
|
||||
# GFX11: encoding: [0x00,0x00,0x6a,0xdc,0x00,0x02,0x7c,0x00]
|
||||
inst = global_store_b32(addr=v[0:1], data=v[2], saddr=OFF)
|
||||
self.assertEqual(inst.to_bytes(), bytes([0x00,0x00,0x6a,0xdc,0x00,0x02,0x7c,0x00]))
|
||||
|
||||
def test_global_load_b32_saddr(self):
|
||||
# global_load_b32 v0, v1, s[0:1] (seg=2 for global)
|
||||
# GFX11: encoding: [0x00,0x00,0x52,0xdc,0x01,0x00,0x00,0x00]
|
||||
inst = global_load_b32(vdst=v[0], addr=v[1], saddr=s[0:1])
|
||||
self.assertEqual(inst.to_bytes(), bytes([0x00,0x00,0x52,0xdc,0x01,0x00,0x00,0x00]))
|
||||
|
||||
def test_global_load_b32_offset(self):
|
||||
# global_load_b32 v0, v[1:2], off offset:256 (seg=2 for global)
|
||||
# GFX11: encoding: [0x00,0x01,0x52,0xdc,0x01,0x00,0x7c,0x00]
|
||||
inst = global_load_b32(vdst=v[0], addr=v[1:2], saddr=OFF, offset=256)
|
||||
self.assertEqual(inst.to_bytes(), bytes([0x00,0x01,0x52,0xdc,0x01,0x00,0x7c,0x00]))
|
||||
|
||||
def test_global_load_b64(self):
|
||||
# global_load_b64 v[0:1], v[2:3], off (seg=2 for global)
|
||||
# GFX11: encoding: [0x00,0x00,0x56,0xdc,0x02,0x00,0x7c,0x00]
|
||||
inst = global_load_b64(vdst=v[0:1], addr=v[2:3], saddr=OFF)
|
||||
self.assertEqual(inst.to_bytes(), bytes([0x00,0x00,0x56,0xdc,0x02,0x00,0x7c,0x00]))
|
||||
|
||||
|
||||
class TestSMEM(unittest.TestCase):
|
||||
"""Test SMEM (scalar memory) instructions - regression tests for glc/dlc bit positions."""
|
||||
|
||||
def test_smem_dlc_bit_position(self):
|
||||
# s_load_b32 s5, s[2:3], s0 dlc - tests that DLC is at bit 13 (not bit 14)
|
||||
# GFX11: encoding: [0x41,0x21,0x00,0xf4,0x00,0x00,0x00,0x00]
|
||||
inst = s_load_b32(sdata=s[5], sbase=s[2:3], soffset=s[0], dlc=1)
|
||||
self.assertEqual(inst.to_bytes(), bytes([0x41,0x21,0x00,0xf4,0x00,0x00,0x00,0x00]))
|
||||
|
||||
def test_smem_glc_bit_position(self):
|
||||
# s_load_b32 s5, s[2:3], s0 glc - tests that GLC is at bit 14 (not bit 16)
|
||||
# GFX11: encoding: [0x41,0x41,0x00,0xf4,0x00,0x00,0x00,0x00]
|
||||
inst = s_load_b32(sdata=s[5], sbase=s[2:3], soffset=s[0], glc=1)
|
||||
self.assertEqual(inst.to_bytes(), bytes([0x41,0x41,0x00,0xf4,0x00,0x00,0x00,0x00]))
|
||||
|
||||
def test_smem_glc_dlc_combined(self):
|
||||
# s_load_b32 s5, s[2:3], s0 glc dlc - tests both flags together
|
||||
# GFX11: encoding: [0x41,0x61,0x00,0xf4,0x00,0x00,0x00,0x00]
|
||||
inst = s_load_b32(sdata=s[5], sbase=s[2:3], soffset=s[0], glc=1, dlc=1)
|
||||
self.assertEqual(inst.to_bytes(), bytes([0x41,0x61,0x00,0xf4,0x00,0x00,0x00,0x00]))
|
||||
|
||||
def test_smem_disasm_roundtrip_dlc(self):
|
||||
# Test that disassembly/reassembly preserves DLC bit correctly
|
||||
data = bytes([0x41,0x21,0x00,0xf4,0x00,0x00,0x00,0x00])
|
||||
decoded = SMEM.from_bytes(data)
|
||||
self.assertEqual(decoded.to_bytes(), data)
|
||||
|
||||
def test_smem_disasm_roundtrip_glc_dlc(self):
|
||||
# Test that disassembly/reassembly preserves GLC+DLC bits correctly
|
||||
data = bytes([0x41,0x61,0x00,0xf4,0x00,0x00,0x00,0x00])
|
||||
decoded = SMEM.from_bytes(data)
|
||||
self.assertEqual(decoded.to_bytes(), data)
|
||||
|
||||
|
||||
class TestVOP3Literal(unittest.TestCase):
|
||||
"""Test VOP3 literal handling - regression tests for Inst64 literal encoding."""
|
||||
|
||||
def test_vop3_with_literal(self):
|
||||
# v_add3_u32 v5, vcc_hi, 0xaf123456, v255
|
||||
# GFX11: encoding: [0x05,0x00,0x55,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
|
||||
inst = VOP3(VOP3Op.V_ADD3_U32, vdst=v[5], src0=VCC_HI, src1=0xaf123456, src2=v[255])
|
||||
expected = bytes([0x05,0x00,0x55,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf])
|
||||
self.assertEqual(inst.to_bytes(), expected)
|
||||
|
||||
def test_vop3_literal_null_operand(self):
|
||||
# v_add3_u32 v5, null, exec_lo, 0xaf123456
|
||||
# GFX11: encoding: [0x05,0x00,0x55,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
|
||||
inst = VOP3(VOP3Op.V_ADD3_U32, vdst=v[5], src0=NULL, src1=EXEC_LO, src2=0xaf123456)
|
||||
expected = bytes([0x05,0x00,0x55,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf])
|
||||
self.assertEqual(inst.to_bytes(), expected)
|
||||
|
||||
def test_vop3p_with_literal(self):
|
||||
# Test VOP3P literal encoding (also uses Inst64)
|
||||
inst = VOP3P(VOP3POp.V_PK_ADD_F16, vdst=v[5], src0=0.5, src1=0x12345678, src2=v[0])
|
||||
self.assertEqual(len(inst.to_bytes()), 12) # 8 bytes + 4 byte literal
|
||||
|
||||
|
||||
class TestDetectFormat(unittest.TestCase):
|
||||
"""Test detect_format uses encoding from autogen classes."""
|
||||
|
||||
def test_detect_sopp(self):
|
||||
self.assertEqual(detect_format(s_endpgm().to_bytes()), SOPP)
|
||||
self.assertEqual(detect_format(s_nop(0).to_bytes()), SOPP)
|
||||
self.assertEqual(detect_format(s_barrier().to_bytes()), SOPP)
|
||||
|
||||
def test_detect_sop1(self):
|
||||
self.assertEqual(detect_format(s_mov_b32(s[0], 0).to_bytes()), SOP1)
|
||||
self.assertEqual(detect_format(s_mov_b64(s[0:1], 0).to_bytes()), SOP1)
|
||||
|
||||
def test_detect_sop2(self):
|
||||
self.assertEqual(detect_format(s_add_u32(s[0], s[1], s[2]).to_bytes()), SOP2)
|
||||
self.assertEqual(detect_format(s_mul_i32(s[0], s[1], s[2]).to_bytes()), SOP2)
|
||||
|
||||
def test_detect_sopc(self):
|
||||
self.assertEqual(detect_format(s_cmp_eq_i32(s[0], s[1]).to_bytes()), SOPC)
|
||||
|
||||
def test_detect_sopk(self):
|
||||
self.assertEqual(detect_format(s_movk_i32(s[0], 0x1234).to_bytes()), SOPK)
|
||||
|
||||
def test_detect_vop1(self):
|
||||
self.assertEqual(detect_format(v_mov_b32_e32(v[0], 0).to_bytes()), VOP1)
|
||||
self.assertEqual(detect_format(v_rcp_f32_e32(v[0], v[1]).to_bytes()), VOP1)
|
||||
|
||||
def test_detect_vop2(self):
|
||||
self.assertEqual(detect_format(v_add_f32_e32(v[0], v[1], v[2]).to_bytes()), VOP2)
|
||||
self.assertEqual(detect_format(v_mul_f32_e32(v[0], v[1], v[2]).to_bytes()), VOP2)
|
||||
|
||||
def test_detect_vopc(self):
|
||||
self.assertEqual(detect_format(v_cmp_eq_f32_e32(v[0], v[1]).to_bytes()), VOPC)
|
||||
self.assertEqual(detect_format(v_cmp_lt_i32_e32(v[0], v[1]).to_bytes()), VOPC)
|
||||
|
||||
def test_detect_vop3(self):
|
||||
self.assertEqual(detect_format(v_add_f32_e64(v[0], v[1], v[2]).to_bytes()), VOP3)
|
||||
self.assertEqual(detect_format(v_fma_f32(v[0], v[1], v[2], v[3]).to_bytes()), VOP3)
|
||||
|
||||
def test_detect_vop3p(self):
|
||||
self.assertEqual(detect_format(VOP3P(VOP3POp.V_PK_ADD_F16, v[0], v[1], v[2], v[3]).to_bytes()), VOP3P)
|
||||
|
||||
def test_detect_smem(self):
|
||||
self.assertEqual(detect_format(s_load_b32(sdata=s[0], sbase=s[2:3], offset=0).to_bytes()), SMEM)
|
||||
self.assertEqual(detect_format(s_load_b64(sdata=s[0:1], sbase=s[2:3], soffset=s[5]).to_bytes()), SMEM)
|
||||
|
||||
def test_detect_ds(self):
|
||||
self.assertEqual(detect_format(ds_load_b32(v[0], v[1]).to_bytes()), DS)
|
||||
self.assertEqual(detect_format(ds_store_b32(v[0], v[1]).to_bytes()), DS)
|
||||
|
||||
def test_detect_flat(self):
|
||||
self.assertEqual(detect_format(global_load_b32(vdst=v[0], addr=v[1:2], saddr=NULL).to_bytes()), GLOBAL)
|
||||
self.assertEqual(detect_format(global_store_b32(addr=v[0:1], data=v[2], saddr=NULL).to_bytes()), GLOBAL)
|
||||
|
||||
def test_detect_vopd(self):
|
||||
inst = VOPD(VOPDOp.V_DUAL_MOV_B32, VOPDOp.V_DUAL_MOV_B32, vdstx=v[0], vdsty=v[1], srcx0=0, srcy0=0)
|
||||
self.assertEqual(detect_format(inst.to_bytes()), VOPD)
|
||||
|
||||
def test_detect_vinterp(self):
|
||||
inst = VINTERP(VINTERPOp.V_INTERP_P10_F32, vdst=v[0], src0=v[1], src1=v[2], src2=v[3])
|
||||
self.assertEqual(detect_format(inst.to_bytes()), VINTERP)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
197
tinygrad_repo/test/amd/test_handwritten.py
Normal file
197
tinygrad_repo/test/amd/test_handwritten.py
Normal file
@@ -0,0 +1,197 @@
|
||||
# do not change these tests. we need to fix bugs to make them pass
|
||||
# the Inst constructor should be looking at the types of the fields to correctly set the value
|
||||
|
||||
import unittest, struct
|
||||
from tinygrad.runtime.autogen.amd.rdna3.ins import *
|
||||
from tinygrad.renderer.amd.dsl import Inst
|
||||
from test.amd.test_roundtrip import compile_asm
|
||||
from test.amd.disasm import disasm
|
||||
|
||||
class IntegrationTestBase(unittest.TestCase):
|
||||
inst: Inst
|
||||
arch: str
|
||||
def tearDown(self):
|
||||
if not hasattr(self, 'inst'): return
|
||||
b = self.inst.to_bytes()
|
||||
st = disasm(self.inst)
|
||||
# Test that the instruction can be compiled by LLVM and produces the same bytes
|
||||
desc = f"{st:25s} {self.inst} {b!r}"
|
||||
self.assertEqual(b, compile_asm(st, arch=self.arch), desc)
|
||||
print(desc)
|
||||
|
||||
class TestIntegration(IntegrationTestBase):
|
||||
arch: str = "rdna3"
|
||||
|
||||
def test_wmma(self):
|
||||
self.inst = v_wmma_f32_16x16x16_f16(v[0:7], v[184:191], v[136:143], v[0:7])
|
||||
|
||||
def test_load_b128(self):
|
||||
self.inst = s_load_b128(s[4:7], s[0:1], NULL, 0)
|
||||
|
||||
def test_load_b128_wrong_size(self):
|
||||
# this should have to be 4 regs on the loaded to
|
||||
with self.assertRaises(TypeError):
|
||||
self.inst = s_load_b128(s[4:6], s[0:1], NULL, 0)
|
||||
|
||||
def test_mov_b32(self):
|
||||
self.inst = s_mov_b32(s[80], s[0])
|
||||
|
||||
def test_mov_b64(self):
|
||||
self.inst = s_mov_b64(s[80:81], s[0:1])
|
||||
|
||||
def test_mov_b32_wrong(self):
|
||||
with self.assertRaises(Exception):
|
||||
self.inst = s_mov_b32(s[80:81], s[0:1])
|
||||
with self.assertRaises(Exception):
|
||||
self.inst = s_mov_b32(s[80:81], s[0])
|
||||
with self.assertRaises(Exception):
|
||||
self.inst = s_mov_b32(s[80], s[0:1])
|
||||
|
||||
def test_mov_b64_wrong(self):
|
||||
with self.assertRaises(Exception):
|
||||
self.inst = s_mov_b64(s[80], s[0])
|
||||
with self.assertRaises(Exception):
|
||||
self.inst = s_mov_b64(s[80], s[0:1])
|
||||
with self.assertRaises(Exception):
|
||||
self.inst = s_mov_b64(s[80:81], s[0])
|
||||
|
||||
def test_load_b128_no_0(self):
|
||||
self.inst = s_load_b128(s[4:7], s[0:1], NULL)
|
||||
|
||||
def test_load_b128_s(self):
|
||||
self.inst = s_load_b128(s[4:7], s[0:1], s[8], 0)
|
||||
|
||||
def test_load_b128_v(self):
|
||||
with self.assertRaises(TypeError):
|
||||
self.inst = s_load_b128(s[4:7], s[0:1], v[8], 0)
|
||||
|
||||
def test_load_b128_off(self):
|
||||
self.inst = s_load_b128(s[4:7], s[0:1], NULL, 3)
|
||||
|
||||
def test_simple_stos(self):
|
||||
self.inst = s_mov_b32(s[0], s[1])
|
||||
|
||||
def test_simple_wrong(self):
|
||||
with self.assertRaises(TypeError):
|
||||
self.inst = s_mov_b32(v[0], s[1])
|
||||
|
||||
def test_simple_vtov(self):
|
||||
self.inst = v_mov_b32_e32(v[0], v[1])
|
||||
|
||||
def test_simple_stov(self):
|
||||
self.inst = v_mov_b32_e32(v[0], s[2])
|
||||
|
||||
def test_simple_float_to_v(self):
|
||||
self.inst = v_mov_b32_e32(v[0], 1.0)
|
||||
|
||||
def test_simple_v_to_float(self):
|
||||
with self.assertRaises(TypeError):
|
||||
self.inst = v_mov_b32_e32(1, v[0])
|
||||
|
||||
def test_invalid_field(self):
|
||||
with self.assertRaises(TypeError):
|
||||
self.inst = s_load_b128(s[4:7], s[0:1], NULL, ioffset=0x8)
|
||||
|
||||
def test_simple_int_to_v(self):
|
||||
self.inst = v_mov_b32_e32(v[0], 1)
|
||||
|
||||
def test_three_add(self):
|
||||
self.inst = v_add_co_ci_u32_e32(v[3], s[7], v[3])
|
||||
|
||||
def test_three_add_v(self):
|
||||
self.inst = v_add_co_ci_u32_e32(v[3], v[7], v[3])
|
||||
|
||||
def test_three_add_const(self):
|
||||
self.inst = v_add_co_ci_u32_e32(v[3], 2.0, v[3])
|
||||
|
||||
def test_swaitcnt_lgkm(self): self.inst = s_waitcnt(0xfc07)
|
||||
def test_swaitcnt_vm(self): self.inst = s_waitcnt(0x03f7)
|
||||
|
||||
def test_vmad(self):
|
||||
self.inst = v_mad_u64_u32(v[1:2], NULL, s[2], 3, v[1:2])
|
||||
|
||||
def test_large_imm(self):
|
||||
self.inst = v_mov_b32_e32(v[0], 0x1234)
|
||||
|
||||
def test_dual_mov(self):
|
||||
self.inst = VOPD(VOPDOp.V_DUAL_MOV_B32, VOPDOp.V_DUAL_MOV_B32, vdstx=v[0], vdsty=v[1], srcx0=v[2], srcy0=v[4])
|
||||
|
||||
def test_dual_mul(self):
|
||||
self.inst = v_dual_mul_f32(VOPDOp.V_DUAL_MUL_F32, vdstx=v[0], vdsty=v[1], srcx0=v[2], vsrcx1=v[3], srcy0=v[4], vsrcy1=v[5])
|
||||
|
||||
def test_simple_int_to_s(self):
|
||||
self.inst = s_mov_b32(s[0], 3)
|
||||
|
||||
def test_complex_int_to_s(self):
|
||||
self.inst = s_mov_b32(s[0], 0x235646)
|
||||
|
||||
def test_simple_float_to_s(self):
|
||||
self.inst = s_mov_b32(s[0], 1.0)
|
||||
|
||||
def test_complex_float_to_s(self):
|
||||
self.inst = s_mov_b32(s[0], 1337.0)
|
||||
int_inst = s_mov_b32(s[0], struct.unpack("I", struct.pack("f", 1337.0))[0])
|
||||
self.assertEqual(self.inst, int_inst)
|
||||
|
||||
class TestIntegrationCDNA(IntegrationTestBase):
|
||||
arch = "cdna"
|
||||
|
||||
def test_mfma(self):
|
||||
from tinygrad.runtime.autogen.amd.cdna.ins import v_mfma_f32_16x16x16_f16
|
||||
self.inst = v_mfma_f32_16x16x16_f16(v[0:3], v[0:1], v[0:1], 0)
|
||||
|
||||
def test_mfma_fp8(self):
|
||||
from tinygrad.runtime.autogen.amd.cdna.ins import v_mfma_f32_16x16x128_f8f6f4
|
||||
self.inst = v_mfma_f32_16x16x128_f8f6f4(v[0:3], v[0:5], v[0:5], 1, cbsz=2, blgp=2)
|
||||
|
||||
class TestRegisterSliceSyntax(unittest.TestCase):
|
||||
"""
|
||||
Issue: Register slice syntax should use AMD assembly convention (inclusive end).
|
||||
|
||||
In AMD assembly, s[4:7] means registers s4, s5, s6, s7 (4 registers, inclusive).
|
||||
The DSL should match this convention so that:
|
||||
- s[4:7] gives 4 registers
|
||||
- Disassembler output can be copied directly back into DSL code
|
||||
|
||||
Fix: Change _RegFactory.__getitem__ to use inclusive end:
|
||||
key.stop - key.start + 1 (instead of key.stop - key.start)
|
||||
"""
|
||||
def test_register_slice_count(self):
|
||||
# s[4:7] should give 4 registers: s4, s5, s6, s7 (AMD convention, inclusive)
|
||||
reg = s[4:7]
|
||||
self.assertEqual(reg.sz, 4, "s[4:7] should give 4 registers (s4, s5, s6, s7)")
|
||||
|
||||
def test_register_slice_roundtrip(self):
|
||||
# Round-trip: DSL -> disasm -> DSL should preserve register count
|
||||
reg = s[4:7] # 4 registers in AMD convention
|
||||
inst = s_load_b128(reg, s[0:1], NULL, 0)
|
||||
d = disasm(inst)
|
||||
# Disasm shows s[4:7] - user should be able to copy this back
|
||||
self.assertIn("s[4:7]", d)
|
||||
# And s[4:7] in DSL should give the same 4 registers
|
||||
reg_from_disasm = s[4:7]
|
||||
self.assertEqual(reg_from_disasm.sz, 4, "s[4:7] from disasm should give 4 registers")
|
||||
|
||||
class TestInstructionEquality(unittest.TestCase):
|
||||
"""
|
||||
Issue: No __eq__ method - instruction comparison requires repr() workaround.
|
||||
|
||||
Two identical instructions should compare equal with ==, but currently:
|
||||
inst1 == inst2 returns False
|
||||
|
||||
The test_handwritten.py works around this with:
|
||||
self.assertEqual(repr(self.inst), repr(reasm))
|
||||
"""
|
||||
def test_identical_instructions_equal(self):
|
||||
inst1 = v_mov_b32_e32(v[0], v[1])
|
||||
inst2 = v_mov_b32_e32(v[0], v[1])
|
||||
self.assertEqual(inst1, inst2, "identical instructions should be equal")
|
||||
|
||||
def test_different_instructions_not_equal(self):
|
||||
inst1 = v_mov_b32_e32(v[0], v[1])
|
||||
inst2 = v_mov_b32_e32(v[0], v[2])
|
||||
self.assertNotEqual(inst1, inst2, "different instructions should not be equal")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
117
tinygrad_repo/test/amd/test_integration.py
Normal file
117
tinygrad_repo/test/amd/test_integration.py
Normal file
@@ -0,0 +1,117 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Integration test: round-trip RDNA3 assembly through LLVM toolchain."""
|
||||
import unittest
|
||||
from tinygrad.runtime.autogen.amd.rdna3.ins import *
|
||||
from test.amd.helpers import llvm_assemble, llvm_disasm
|
||||
|
||||
def waitcnt(vmcnt: int = 0x3f, expcnt: int = 0x7, lgkmcnt: int = 0x3f) -> int:
|
||||
return (expcnt & 0x7) | ((lgkmcnt & 0x3f) << 4) | ((vmcnt & 0x3f) << 10)
|
||||
|
||||
def assemble_and_disassemble(instructions: list, mcpu: str = "gfx1100", mattr: str = "+real-true16,+wavefrontsize32") -> list[str]:
|
||||
"""Assemble instructions with our DSL, then disassemble with LLVM."""
|
||||
code_bytes = b''.join(inst.to_bytes() for inst in instructions)
|
||||
return llvm_disasm(code_bytes, mcpu, mattr)
|
||||
|
||||
class TestIntegration(unittest.TestCase):
|
||||
"""Test our DSL output matches LLVM disassembly."""
|
||||
|
||||
def test_simple_sop1(self):
|
||||
"""Test SOP1 instructions round-trip."""
|
||||
instructions = [s_mov_b32(s[0], s[1]), s_mov_b32(s[2], 0), s_not_b32(s[3], s[4])]
|
||||
disasm = assemble_and_disassemble(instructions)
|
||||
self.assertIn('s_mov_b32', disasm[0])
|
||||
self.assertIn('s_mov_b32', disasm[1])
|
||||
self.assertIn('s_not_b32', disasm[2])
|
||||
|
||||
def test_simple_sop2(self):
|
||||
"""Test SOP2 instructions round-trip."""
|
||||
instructions = [s_add_u32(s[0], s[1], s[2]), s_sub_u32(s[3], s[4], 10), s_and_b32(s[5], s[6], s[7])]
|
||||
disasm = assemble_and_disassemble(instructions)
|
||||
self.assertIn('s_add_u32', disasm[0])
|
||||
self.assertIn('s_sub_u32', disasm[1])
|
||||
self.assertIn('s_and_b32', disasm[2])
|
||||
|
||||
def test_simple_vop2(self):
|
||||
"""Test VOP2 instructions round-trip."""
|
||||
instructions = [v_add_f32_e32(v[0], v[1], v[2]), v_mul_f32_e32(v[3], 1.0, v[4]), v_and_b32_e32(v[5], 10, v[6])]
|
||||
disasm = assemble_and_disassemble(instructions)
|
||||
self.assertIn('v_add_f32', disasm[0])
|
||||
self.assertIn('v_mul_f32', disasm[1])
|
||||
|
||||
def test_control_flow(self):
|
||||
"""Test control flow instructions."""
|
||||
instructions = [s_waitcnt(simm16=waitcnt(lgkmcnt=0)), s_endpgm()]
|
||||
disasm = assemble_and_disassemble(instructions)
|
||||
self.assertIn('s_waitcnt', disasm[0])
|
||||
self.assertIn('s_endpgm', disasm[1])
|
||||
|
||||
def test_memory_ops(self):
|
||||
"""Test memory instructions."""
|
||||
instructions = [s_load_b32(s[0], s[0:1], NULL), s_waitcnt(simm16=waitcnt(lgkmcnt=0)), global_store_b32(addr=v[0:1], data=v[2], saddr=OFF),
|
||||
s_endpgm()]
|
||||
disasm = assemble_and_disassemble(instructions)
|
||||
self.assertIn('s_load_b32', disasm[0])
|
||||
self.assertIn('s_waitcnt', disasm[1])
|
||||
self.assertIn('global_store_b32', disasm[2])
|
||||
|
||||
def test_full_kernel(self):
|
||||
"""Test a complete kernel similar to tinygrad output."""
|
||||
instructions = [v_mov_b32_e32(v[0], s[0]), v_mov_b32_e32(v[1], s[1]), global_load_b32(vdst=v[2], addr=v[0:1], saddr=OFF),
|
||||
s_waitcnt(simm16=waitcnt(vmcnt=0)), v_add_f32_e32(v[2], 1.0, v[2]), global_store_b32(addr=v[0:1], data=v[2], saddr=OFF),
|
||||
s_endpgm()]
|
||||
disasm = assemble_and_disassemble(instructions)
|
||||
self.assertTrue(any('global_load' in d for d in disasm))
|
||||
self.assertTrue(any('v_add_f32' in d for d in disasm))
|
||||
self.assertTrue(any('global_store' in d for d in disasm))
|
||||
self.assertTrue(any('s_endpgm' in d for d in disasm))
|
||||
|
||||
def test_bytes_roundtrip(self):
|
||||
"""Test that our bytes match what LLVM assembler produces."""
|
||||
inst = s_mov_b32(s[0], s[1])
|
||||
our_bytes = inst.to_bytes()
|
||||
llvm_bytes = llvm_assemble(["s_mov_b32 s0, s1"], "gfx1100", "+real-true16,+wavefrontsize32")[0]
|
||||
self.assertEqual(our_bytes, llvm_bytes, f"Bytes mismatch: ours={our_bytes.hex()} LLVM={llvm_bytes.hex()}")
|
||||
|
||||
class TestTinygradIntegration(unittest.TestCase):
|
||||
"""Test that we can parse tinygrad kernel disassembly."""
|
||||
|
||||
def _get_kernel_code(self, op_fn) -> bytes:
|
||||
from tinygrad import Tensor
|
||||
from tinygrad.helpers import Target
|
||||
from tinygrad.codegen import to_program
|
||||
from tinygrad.renderer.llvmir import AMDLLVMRenderer
|
||||
from tinygrad.runtime.support.elf import elf_loader
|
||||
from tinygrad.uop.ops import Ops
|
||||
|
||||
result = op_fn(Tensor)
|
||||
linear = result.schedule_linear()
|
||||
sink_items = [call for call in linear.src if call.src[0].op == Ops.SINK]
|
||||
assert len(sink_items) > 0, "No SINK in schedule"
|
||||
renderer = AMDLLVMRenderer(Target("AMD", arch='gfx1100'))
|
||||
prg = to_program(sink_items[0].src[0], renderer)
|
||||
lib = renderer.compiler.compile(prg.src[3].arg)
|
||||
return next(s.content for s in elf_loader(lib)[1] if s.name == ".text")
|
||||
|
||||
def test_simple_add_kernel(self):
|
||||
"""Generate a simple add kernel from tinygrad and verify disassembly."""
|
||||
code = self._get_kernel_code(lambda T: T([1.0, 2.0, 3.0, 4.0]).realize() + T([5.0, 6.0, 7.0, 8.0]).realize())
|
||||
instrs = llvm_disasm(code, "gfx1100", "+real-true16,+wavefrontsize32")
|
||||
self.assertTrue(len(instrs) > 0, "No instructions in disassembly")
|
||||
self.assertTrue(any('s_endpgm' in i for i in instrs), "Missing s_endpgm")
|
||||
|
||||
def test_matmul_kernel(self):
|
||||
"""Generate a matmul kernel and verify disassembly has expected patterns."""
|
||||
code = self._get_kernel_code(lambda T: T.rand(4, 4).realize() @ T.rand(4, 4).realize())
|
||||
instrs = llvm_disasm(code, "gfx1100", "+real-true16,+wavefrontsize32")
|
||||
has_mul = any('mul' in i.lower() for i in instrs)
|
||||
has_add = any('add' in i.lower() for i in instrs)
|
||||
self.assertTrue(has_mul or has_add, "Matmul should have mul/add ops")
|
||||
|
||||
def test_disasm_to_bytes_roundtrip(self):
|
||||
"""Verify s_endpgm encoding matches between our DSL and LLVM."""
|
||||
our_bytes = s_endpgm().to_bytes()
|
||||
llvm_bytes = llvm_assemble(["s_endpgm"], "gfx1100", "+real-true16,+wavefrontsize32")[0]
|
||||
self.assertEqual(our_bytes, llvm_bytes, f"s_endpgm mismatch: ours={our_bytes.hex()} LLVM={llvm_bytes.hex()}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
156
tinygrad_repo/test/amd/test_llvm.py
Normal file
156
tinygrad_repo/test/amd/test_llvm.py
Normal file
@@ -0,0 +1,156 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Test AMD assembler/disassembler against LLVM test vectors.
|
||||
|
||||
Only compute-relevant instruction formats are tested. Graphics-only formats not supported:
|
||||
- MUBUF/MTBUF: buffer instructions with resource descriptors (use GLOBAL/FLAT instead)
|
||||
- MIMG: image/texture instructions
|
||||
- EXP/VEXPORT: export instructions for pixel/vertex output
|
||||
- VIMAGE/VSAMPLE: image sampling instructions (RDNA4)
|
||||
- VBUFFER: buffer instructions (RDNA4)
|
||||
"""
|
||||
import unittest, re, functools
|
||||
from tinygrad.helpers import fetch
|
||||
from test.amd.disasm import disasm
|
||||
from tinygrad.renderer.amd import decode_inst, detect_format
|
||||
from test.amd.helpers import llvm_assemble, llvm_filter_valid_asm, get_target, get_mattr
|
||||
|
||||
LLVM_BASE = "https://raw.githubusercontent.com/llvm/llvm-project/llvmorg-21.1.0/llvm/test/MC/AMDGPU"
|
||||
|
||||
# RDNA3 (gfx11) test files for compute instructions
|
||||
# Excluded: gfx11_asm_mubuf.s, gfx11_asm_mtbuf.s, gfx11_asm_mimg.s, gfx11_asm_mubuf_alias.s, gfx11_asm_mtbuf_alias.s (graphics-only)
|
||||
RDNA_FILES = ['gfx11_asm_sop1.s', 'gfx11_asm_sop2.s', 'gfx11_asm_sopp.s', 'gfx11_asm_sopk.s', 'gfx11_asm_sopc.s',
|
||||
'gfx11_asm_vop1.s', 'gfx11_asm_vop2.s', 'gfx11_asm_vopc.s', 'gfx11_asm_vop3.s', 'gfx11_asm_vop3p.s', 'gfx11_asm_vinterp.s',
|
||||
'gfx11_asm_vopd.s', 'gfx11_asm_vopcx.s', 'gfx11_asm_vop3_from_vop1.s', 'gfx11_asm_vop3_from_vop2.s', 'gfx11_asm_vop3_from_vopc.s',
|
||||
'gfx11_asm_vop3_from_vopcx.s', 'gfx11_asm_ds.s', 'gfx11_asm_smem.s', 'gfx11_asm_flat.s',
|
||||
'gfx11_asm_wmma.s', 'gfx11_asm_vop3_features.s', 'gfx11_asm_vop3p_features.s', 'gfx11_asm_vopd_features.s',
|
||||
'gfx11_asm_vop3_alias.s', 'gfx11_asm_vop3p_alias.s', 'gfx11_asm_vopc_alias.s', 'gfx11_asm_vopcx_alias.s', 'gfx11_asm_vinterp_alias.s',
|
||||
'gfx11_asm_smem_alias.s']
|
||||
# CDNA (gfx9/gfx90a/gfx942/gfx950) test files for compute instructions
|
||||
# Excluded: gfx9_asm_mubuf.s, gfx9_asm_mtbuf.s, gfx90a_ldst_acc.s (has MIMG mixed in)
|
||||
# Exclude gfx90a: 'gfx90a_asm_features.s', 'mai-gfx90a.s',
|
||||
# Exclude gfx950: 'gfx950_asm_features.s' (disasm error)
|
||||
CDNA_FILES = ['gfx9_asm_sop1.s', 'gfx9_asm_sop2.s', 'gfx9_asm_sopp.s', 'gfx9_asm_sopk.s', 'gfx9_asm_sopc.s',
|
||||
'gfx9_asm_vop1.s', 'gfx9_asm_vop2.s', 'gfx9_asm_vopc.s', 'gfx9_asm_vop3.s', 'gfx9_asm_vop3p.s',
|
||||
'gfx9_asm_ds.s', 'gfx9_asm_flat.s', 'gfx9_asm_smem.s',
|
||||
'flat-scratch-gfx942.s', 'gfx942_asm_features.s', 'mai-gfx942.s',
|
||||
'gfx950_asm_vop1.s', 'gfx950_asm_read_tr.s', 'mai-gfx950.s']
|
||||
# RDNA4 (gfx12) test files for compute instructions
|
||||
# Excluded: gfx12_asm_vbuffer_mubuf.s, gfx12_asm_vbuffer_mtbuf.s, gfx12_asm_exp.s (graphics-only)
|
||||
RDNA4_FILES = ['gfx12_asm_sop1.s', 'gfx12_asm_sop2.s', 'gfx12_asm_sopp.s', 'gfx12_asm_sopk.s', 'gfx12_asm_sopc.s',
|
||||
'gfx12_asm_vop1.s', 'gfx12_asm_vop2.s', 'gfx12_asm_vopc.s', 'gfx12_asm_vopcx.s', 'gfx12_asm_vop3.s', 'gfx12_asm_vop3c.s',
|
||||
'gfx12_asm_vop3cx.s', 'gfx12_asm_vop3p.s', 'gfx12_asm_vop3_from_vop1.s', 'gfx12_asm_vop3_from_vop2.s',
|
||||
'gfx12_asm_vop3p_features.s', 'gfx12_asm_vopd.s', 'gfx12_asm_vopd_features.s',
|
||||
'gfx12_asm_ds.s', 'gfx12_asm_smem.s', 'gfx12_asm_vflat.s',
|
||||
'gfx12_asm_wmma_w32.s']
|
||||
|
||||
def _parse_llvm_tests(text: str, pattern: str) -> list[tuple[str, bytes]]:
|
||||
tests = []
|
||||
for block in text.split('\n\n'):
|
||||
asm_text, encoding = None, None
|
||||
for line in block.split('\n'):
|
||||
line = line.strip()
|
||||
if not line or line.startswith(('.', ';')): continue
|
||||
if not line.startswith('//'):
|
||||
asm_text = line.split('//')[0].strip() or asm_text
|
||||
if m := re.search(pattern + r'[^:]*:.*?(?:encoding:\s*)?\[(0x[0-9a-f,x\s]+)\]', line, re.I):
|
||||
encoding = m.group(1).replace('0x', '').replace(',', '').replace(' ', '')
|
||||
if asm_text and encoding:
|
||||
try: tests.append((asm_text, bytes.fromhex(encoding)))
|
||||
except ValueError: pass
|
||||
return tests
|
||||
|
||||
def _get_tests_uncached(f: str, arch: str) -> list[tuple[str, bytes]]:
|
||||
text = fetch(f"{LLVM_BASE}/{f}").read_bytes().decode('utf-8', errors='ignore')
|
||||
if arch == "rdna3":
|
||||
# Match GFX11 and W32 only (wavefront32 mode)
|
||||
tests = _parse_llvm_tests(text, r'(?:GFX11|W32)')
|
||||
elif arch == "rdna4":
|
||||
# Match GFX12 (but not GFX1250) and W32 only (wavefront32 mode)
|
||||
tests = _parse_llvm_tests(text, r'(?:GFX12(?!50)|W32)')
|
||||
elif 'gfx90a' in f or 'gfx942' in f or 'gfx950' in f:
|
||||
tests = _parse_llvm_tests(text, r'(?:GFX90A|GFX942|GFX950)')
|
||||
else:
|
||||
tests = _parse_llvm_tests(text, r'(?:VI9|GFX9|CHECK)')
|
||||
# Exclude v_interp_* (graphics-only, not on CDNA)
|
||||
if arch == "cdna": tests = [(asm, data) for asm, data in tests if not asm.startswith('v_interp_')]
|
||||
# Filter out tests where original ASM isn't valid on target (e.g., gfx9 tests with gfx942/gfx950 constraints)
|
||||
if arch == "cdna" and not ('gfx942' in f or 'gfx950' in f or 'gfx90a' in f):
|
||||
tests = llvm_filter_valid_asm(tests, get_target(arch), get_mattr(arch))
|
||||
return tests
|
||||
|
||||
@functools.cache
|
||||
def _get_tests(f: str, arch: str) -> list[tuple[str, bytes]]: return _get_tests_uncached(f, arch)
|
||||
|
||||
def _make_test(f: str, arch: str, test_type: str):
|
||||
def test(self):
|
||||
tests = _get_tests(f, arch)
|
||||
name = f"{arch}_{test_type}_{f}"
|
||||
mcpu = "gfx942" if arch == "cdna" and "gfx942" in f else get_target(arch)
|
||||
if test_type == "roundtrip":
|
||||
passed, skipped = 0, 0
|
||||
for _, data in tests:
|
||||
try:
|
||||
decoded = detect_format(data, arch).from_bytes(data)
|
||||
self.assertEqual(decoded.to_bytes()[:len(data)], data)
|
||||
passed += 1
|
||||
except ValueError: skipped += 1 # skip invalid opcodes not in enum
|
||||
print(f"{name}: {passed} passed, {skipped} skipped")
|
||||
self.assertEqual(skipped, 0, f"{name}: {skipped} tests skipped, expected 0")
|
||||
elif test_type == "repr":
|
||||
# Test that eval(repr(inst)) reproduces the instruction
|
||||
if arch == "rdna3": import tinygrad.runtime.autogen.amd.rdna3.ins as ins # type: ignore[no-redef]
|
||||
elif arch == "rdna4": import tinygrad.runtime.autogen.amd.rdna4.ins as ins # type: ignore[no-redef]
|
||||
elif arch == "cdna": import tinygrad.runtime.autogen.amd.cdna.ins as ins # type: ignore[no-redef]
|
||||
ns = {k: getattr(ins, k) for k in dir(ins) if not k.startswith('_')}
|
||||
passed, skipped = 0, 0
|
||||
for _, data in tests:
|
||||
try:
|
||||
decoded = detect_format(data, arch).from_bytes(data)
|
||||
if decoded.to_bytes()[:len(data)] != data:
|
||||
skipped += 1
|
||||
continue # skip if binary roundtrip fails
|
||||
r = repr(decoded)
|
||||
try:
|
||||
decoded2 = eval(r, ns) # noqa: S307
|
||||
if decoded == decoded2: passed += 1
|
||||
else: skipped += 1
|
||||
except Exception: skipped += 1
|
||||
except ValueError: skipped += 1
|
||||
print(f"{name}: {passed} passed, {skipped} skipped")
|
||||
self.assertEqual(skipped, 0, f"{name}: {skipped} tests skipped, expected 0")
|
||||
elif test_type == "disasm":
|
||||
to_test = []
|
||||
for _, data in tests:
|
||||
try:
|
||||
decoded = decode_inst(data, arch)
|
||||
enc = decoded.to_bytes()[:len(data)]
|
||||
# Skip if roundtrip fails, disasm fails, or op_name is missing (disasm starts with space)
|
||||
if enc == data and (d := disasm(decoded)) and not d.startswith(' '): to_test.append((enc, d))
|
||||
except Exception: pass
|
||||
skipped = len(tests) - len(to_test)
|
||||
print(f"{name}: {len(to_test)} passed, {skipped} skipped")
|
||||
self.assertEqual(skipped, 0, f"{name}: {skipped} tests skipped, expected 0")
|
||||
# Compare disasm->reassemble with original encoding (filter reserved bit cases where LLVM can't reproduce)
|
||||
llvm_bytes = llvm_assemble([t[1] for t in to_test], mcpu, get_mattr(arch))
|
||||
valid = [(enc, d, llvm) for (enc, d), llvm in zip(to_test, llvm_bytes) if llvm == enc]
|
||||
print(f"{name}: {len(valid)}/{len(to_test)} matched LLVM encoding")
|
||||
for enc, _, llvm in valid: self.assertEqual(llvm, enc)
|
||||
return test
|
||||
|
||||
class TestLLVM(unittest.TestCase): pass
|
||||
|
||||
for f in RDNA_FILES:
|
||||
setattr(TestLLVM, f"test_rdna3_roundtrip_{f.replace('.s', '').replace('-', '_')}", _make_test(f, "rdna3", "roundtrip"))
|
||||
setattr(TestLLVM, f"test_rdna3_disasm_{f.replace('.s', '').replace('-', '_')}", _make_test(f, "rdna3", "disasm"))
|
||||
setattr(TestLLVM, f"test_rdna3_repr_{f.replace('.s', '').replace('-', '_')}", _make_test(f, "rdna3", "repr"))
|
||||
for f in CDNA_FILES:
|
||||
setattr(TestLLVM, f"test_cdna_roundtrip_{f.replace('.s', '').replace('-', '_')}", _make_test(f, "cdna", "roundtrip"))
|
||||
setattr(TestLLVM, f"test_cdna_disasm_{f.replace('.s', '').replace('-', '_')}", _make_test(f, "cdna", "disasm"))
|
||||
setattr(TestLLVM, f"test_cdna_repr_{f.replace('.s', '').replace('-', '_')}", _make_test(f, "cdna", "repr"))
|
||||
for f in RDNA4_FILES:
|
||||
setattr(TestLLVM, f"test_rdna4_roundtrip_{f.replace('.s', '').replace('-', '_')}", _make_test(f, "rdna4", "roundtrip"))
|
||||
setattr(TestLLVM, f"test_rdna4_disasm_{f.replace('.s', '').replace('-', '_')}", _make_test(f, "rdna4", "disasm"))
|
||||
setattr(TestLLVM, f"test_rdna4_repr_{f.replace('.s', '').replace('-', '_')}", _make_test(f, "rdna4", "repr"))
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
50
tinygrad_repo/test/amd/test_mockgpu_invalid.py
Normal file
50
tinygrad_repo/test/amd/test_mockgpu_invalid.py
Normal file
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Test that invalid instructions raise exceptions through the mock GPU stack."""
|
||||
import unittest, subprocess, os, sys, time
|
||||
|
||||
class TestMockGPUInvalidInstruction(unittest.TestCase):
|
||||
def test_unsupported_instruction_raises(self):
|
||||
"""Test that unsupported instructions raise immediately through the full MOCKGPU stack."""
|
||||
test_code = '''
|
||||
import struct
|
||||
from tinygrad import Device, Tensor
|
||||
from tinygrad.engine.realize import compile_linear
|
||||
from tinygrad.runtime.ops_amd import AMDProgram
|
||||
|
||||
dev = Device["AMD"]
|
||||
a = Tensor([1.0]).realize()
|
||||
b = a + 1
|
||||
linear = compile_linear(b.schedule_linear())
|
||||
lib = bytearray(linear.src[-1].src[0].src[4].arg)
|
||||
|
||||
# Find s_endpgm (0xBFB00000) and replace with V_MOVRELD_B32 (op=66) which has no pcode
|
||||
# VOP1 encoding: bits[31:25]=0x7E, op=bits[16:9], so op=66 -> 66<<9 = 0x8400
|
||||
found = False
|
||||
for i in range(0, len(lib) - 4, 4):
|
||||
if struct.unpack("<I", lib[i:i+4])[0] == 0xBFB00000:
|
||||
lib[i:i+4] = struct.pack("<I", 0x7E008400)
|
||||
found = True
|
||||
break
|
||||
assert found, "s_endpgm not found"
|
||||
|
||||
patched_prg = AMDProgram(dev, "patched", bytes(lib))
|
||||
b.uop.buffer.allocate()
|
||||
patched_prg(b.uop.buffer._buf, a.uop.buffer._buf, global_size=(1,1,1), local_size=(1,1,1))
|
||||
dev.synchronize()
|
||||
'''
|
||||
|
||||
env = os.environ.copy()
|
||||
env["DEV"] = "MOCKKFD+AMD"
|
||||
env["HCQDEV_WAIT_TIMEOUT_MS"] = "10000"
|
||||
|
||||
st = time.perf_counter()
|
||||
result = subprocess.run([sys.executable, "-c", test_code], env=env, capture_output=True, text=True, timeout=60)
|
||||
elapsed = time.perf_counter() - st
|
||||
|
||||
self.assertNotEqual(result.returncode, 0, "should have raised")
|
||||
self.assertTrue("Error" in result.stderr, f"expected an error in stderr, got: {result.stderr[:500]}")
|
||||
# Should exit immediately, not wait for the full timeout
|
||||
self.assertLess(elapsed, 9.0, f"should exit immediately on emulator exception, took {elapsed:.1f}s")
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
53
tinygrad_repo/test/amd/test_pdf.py
Normal file
53
tinygrad_repo/test/amd/test_pdf.py
Normal file
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Test PDF pseudocode extraction from generate.py."""
|
||||
import unittest
|
||||
from tinygrad.renderer.amd.generate import extract_pdf_text, extract_pcode, parse_xml, ARCHS, FIXES
|
||||
|
||||
EXPECTED_PAGES = {"rdna3": 655, "rdna4": 711, "cdna": 610}
|
||||
|
||||
class TestPcodePDF(unittest.TestCase):
|
||||
pages: dict
|
||||
enums: dict
|
||||
pcode: dict
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.pages = {arch: extract_pdf_text(cfg["pdf"]) for arch, cfg in ARCHS.items()}
|
||||
cls.enums = {}
|
||||
for arch, cfg in ARCHS.items():
|
||||
_, enums, _, _, _, _ = parse_xml(cfg["xml"])
|
||||
for fmt, ops in FIXES.get(arch, {}).items(): enums.setdefault(fmt, {}).update(ops)
|
||||
cls.enums[arch] = enums
|
||||
cls.pcode = {arch: extract_pcode(cls.pages[arch], {n: op for ops in cls.enums[arch].values() for op, n in ops.items()}) for arch in ARCHS}
|
||||
|
||||
def test_page_counts(self):
|
||||
for name, exp in EXPECTED_PAGES.items():
|
||||
self.assertEqual(len(self.pages[name]), exp, f"{name} page count")
|
||||
|
||||
def test_pcode_extracted(self):
|
||||
"""Check we extracted a reasonable number of pcode entries."""
|
||||
for name in ARCHS:
|
||||
self.assertGreater(len(self.pcode[name]), 500, f"{name} pcode count too low")
|
||||
|
||||
def test_pcode_rdna3_tricky(self):
|
||||
"""Test specific pseudocode patterns that are tricky to extract correctly."""
|
||||
pcode = self.pcode['rdna3']
|
||||
# BUFFER_ATOMIC_MAX_U64: should have 4 statements (not truncated)
|
||||
self.assertEqual(pcode[('BUFFER_ATOMIC_MAX_U64', 72)],
|
||||
'tmp = MEM[ADDR].u64;\nsrc = DATA.u64;\nMEM[ADDR].u64 = src >= tmp ? src : tmp;\nRETURN_DATA.u64 = tmp')
|
||||
# GLOBAL_STORE_B128: should have 4 MEM stores (not truncated)
|
||||
self.assertEqual(pcode[('GLOBAL_STORE_B128', 29)],
|
||||
'MEM[ADDR].b32 = VDATA[31 : 0];\nMEM[ADDR + 4U].b32 = VDATA[63 : 32];\n'
|
||||
'MEM[ADDR + 8U].b32 = VDATA[95 : 64];\nMEM[ADDR + 12U].b32 = VDATA[127 : 96]')
|
||||
# S_CMOVK_I32: should have full if/endif block
|
||||
self.assertEqual(pcode[('S_CMOVK_I32', 2)],
|
||||
"if SCC then\nD0.i32 = 32'I(signext(SIMM16.i16))\nendif")
|
||||
|
||||
def test_pcode_no_examples(self):
|
||||
"""Pseudocode should not contain example lines with '=>'."""
|
||||
for name in ARCHS:
|
||||
for (op_name, opcode), code in self.pcode[name].items():
|
||||
self.assertNotIn('=>', code, f"{name} {op_name} contains example line with '=>'")
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
66
tinygrad_repo/test/amd/test_rdna3_asm.py
Normal file
66
tinygrad_repo/test/amd/test_rdna3_asm.py
Normal file
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env python3
|
||||
import unittest
|
||||
from tinygrad.runtime.autogen.amd.rdna3.ins import *
|
||||
from test.amd.helpers import llvm_assemble
|
||||
from test.amd.disasm import disasm
|
||||
|
||||
def _asm(asm: str) -> bytes: return llvm_assemble([asm], 'gfx1100', '+real-true16,+wavefrontsize32')[0]
|
||||
|
||||
class TestRDNA3Asm(unittest.TestCase):
|
||||
def test_full_program(self):
|
||||
"""Test the full program from rdna3fun.py matches LLVM output."""
|
||||
program = [
|
||||
v_bfe_u32(v[1], v[0], 10, 10),
|
||||
s_load_b128(s[4:7], s[0:1], NULL),
|
||||
v_and_b32_e32(v[0], 0x3FF, v[0]),
|
||||
s_mulk_i32(s[3], 0x87),
|
||||
v_mad_u64_u32(v[1:2], NULL, s[2], 3, v[1:2]),
|
||||
v_mul_u32_u24_e32(v[0], 45, v[0]),
|
||||
v_ashrrev_i32_e32(v[2], 31, v[1]),
|
||||
v_add3_u32(v[0], v[0], s[3], v[1]),
|
||||
v_lshlrev_b64(v[2:3], 2, v[1:2]),
|
||||
v_ashrrev_i32_e32(v[1], 31, v[0]),
|
||||
v_lshlrev_b64(v[0:1], 2, v[0:1]),
|
||||
s_waitcnt(0xfc07), # lgkmcnt(0)
|
||||
v_add_co_u32(v[2], VCC_LO, s[6], v[2]),
|
||||
v_add_co_ci_u32_e32(v[3], s[7], v[3]),
|
||||
v_add_co_u32(v[0], VCC_LO, s[4], v[0]),
|
||||
global_load_b32(vdst=v[2], addr=v[2:3], saddr=OFF),
|
||||
v_add_co_ci_u32_e32(v[1], s[5], v[1]),
|
||||
s_waitcnt(0x03f7), # vmcnt(0)
|
||||
global_store_b32(addr=v[0:1], data=v[2], saddr=OFF),
|
||||
s_endpgm(),
|
||||
]
|
||||
|
||||
asm_lines = [
|
||||
"v_bfe_u32 v1, v0, 10, 10", "s_load_b128 s[4:7], s[0:1], null", "v_and_b32_e32 v0, 0x3FF, v0",
|
||||
"s_mulk_i32 s3, 0x87", "v_mad_u64_u32 v[1:2], null, s2, 3, v[1:2]", "v_mul_u32_u24_e32 v0, 45, v0",
|
||||
"v_ashrrev_i32_e32 v2, 31, v1", "v_add3_u32 v0, v0, s3, v1", "v_lshlrev_b64 v[2:3], 2, v[1:2]",
|
||||
"v_ashrrev_i32_e32 v1, 31, v0", "v_lshlrev_b64 v[0:1], 2, v[0:1]", "s_waitcnt lgkmcnt(0)",
|
||||
"v_add_co_u32 v2, vcc_lo, s6, v2", "v_add_co_ci_u32_e32 v3, vcc_lo, s7, v3, vcc_lo",
|
||||
"v_add_co_u32 v0, vcc_lo, s4, v0", "global_load_b32 v2, v[2:3], off",
|
||||
"v_add_co_ci_u32_e32 v1, vcc_lo, s5, v1, vcc_lo", "s_waitcnt vmcnt(0)",
|
||||
"global_store_b32 v[0:1], v2, off", "s_endpgm",
|
||||
]
|
||||
expected = llvm_assemble(asm_lines, 'gfx1100', '+real-true16,+wavefrontsize32')
|
||||
for inst, rt in zip(program, asm_lines): print(f"{disasm(inst):50s} {rt}")
|
||||
for inst, exp in zip(program, expected): self.assertEqual(inst.to_bytes(), exp)
|
||||
|
||||
def test_sop2_s_add_u32(self):
|
||||
inst = SOP2(SOP2Op.S_ADD_U32, s[3], s[0], s[1])
|
||||
self.assertEqual(inst.to_bytes(), _asm("s_add_u32 s3, s0, s1"))
|
||||
|
||||
def test_vop2_v_and_b32_inline_const(self):
|
||||
inst = v_and_b32_e32(v[0], 10, v[0])
|
||||
self.assertEqual(inst.to_bytes(), _asm("v_and_b32_e32 v0, 10, v0"))
|
||||
|
||||
def test_sopp_s_endpgm(self):
|
||||
inst = s_endpgm()
|
||||
self.assertEqual(inst.to_bytes(), _asm("s_endpgm"))
|
||||
|
||||
def test_sop1_s_mov_b32(self):
|
||||
inst = s_mov_b32(s[0], s[1])
|
||||
self.assertEqual(inst.to_bytes(), _asm("s_mov_b32 s0, s1"))
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
279
tinygrad_repo/test/amd/test_roundtrip.py
Normal file
279
tinygrad_repo/test/amd/test_roundtrip.py
Normal file
@@ -0,0 +1,279 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Roundtrip tests: generate tinygrad kernels, decode instructions, re-encode, verify match."""
|
||||
import unittest, io, sys, re
|
||||
from dataclasses import dataclass
|
||||
from tinygrad import Device
|
||||
from tinygrad.renderer.amd import detect_format
|
||||
from test.amd.helpers import llvm_assemble, llvm_disasm, get_target, get_mattr
|
||||
from test.amd.disasm import disasm
|
||||
|
||||
def disassemble_lib(lib: bytes, compiler) -> list[tuple[str, bytes]]:
|
||||
"""Disassemble ELF binary and return list of (instruction_text, machine_code_bytes)."""
|
||||
old_stdout = sys.stdout
|
||||
sys.stdout = io.StringIO()
|
||||
compiler.disassemble(lib)
|
||||
output = sys.stdout.getvalue()
|
||||
sys.stdout = old_stdout
|
||||
|
||||
results = []
|
||||
for line in output.splitlines():
|
||||
if '//' not in line: continue
|
||||
instr = line.split('//')[0].strip()
|
||||
if not instr: continue
|
||||
comment = line.split('//')[1].strip()
|
||||
if ':' not in comment: continue
|
||||
hex_str = comment.split(':')[1].strip().split()[0]
|
||||
try:
|
||||
machine_bytes = bytes.fromhex(hex_str)[::-1] # big-endian to little-endian
|
||||
results.append((instr, machine_bytes))
|
||||
except ValueError:
|
||||
continue
|
||||
return results
|
||||
|
||||
def compile_asm(instr: str, arch: str = 'rdna3') -> bytes:
|
||||
"""Compile a single instruction using LLVM."""
|
||||
return llvm_assemble([instr], get_target(arch), get_mattr(arch))[0]
|
||||
|
||||
def compile_asm_batch(instrs: list[str], arch: str = 'rdna3') -> list[bytes]:
|
||||
"""Compile multiple instructions with a single LLVM emission."""
|
||||
return llvm_assemble(instrs, get_target(arch), get_mattr(arch))
|
||||
|
||||
def compile_and_disasm_batch(instrs: list[str], arch: str = 'rdna3') -> list[str]:
|
||||
"""Compile instructions with LLVM and get LLVM's disassembly."""
|
||||
if not instrs: return []
|
||||
mcpu, mattr = get_target(arch), get_mattr(arch)
|
||||
code = b''.join(llvm_assemble(instrs, mcpu, mattr))
|
||||
return llvm_disasm(code, mcpu, mattr)[:len(instrs)]
|
||||
|
||||
@dataclass
|
||||
class KernelSnapshot:
|
||||
code: bytes
|
||||
src: str
|
||||
global_size: tuple[int, int, int]
|
||||
local_size: tuple[int, int, int]
|
||||
buf_idxs: list[int] # indices into shared buffer pool
|
||||
buf_sizes: list[int] # sizes for each buffer index
|
||||
|
||||
def get_kernels_from_tinygrad(op_fn) -> tuple[list[KernelSnapshot], dict[int, int], dict[int, bytes]]:
|
||||
"""Compile a tinygrad operation and extract all kernels with their buffer mappings."""
|
||||
from tinygrad import Tensor
|
||||
from tinygrad.uop.ops import Ops
|
||||
from tinygrad.engine.realize import compile_linear, resolve_params, unwrap_multi
|
||||
from tinygrad.runtime.support.elf import elf_loader
|
||||
|
||||
out = op_fn(Tensor)
|
||||
linear = compile_linear(out.schedule_linear())
|
||||
kernels = []
|
||||
buf_pool: dict[int, int] = {} # buffer id -> size
|
||||
buf_data: dict[int, bytes] = {} # buffer id -> initial data from COPY
|
||||
|
||||
for call in linear.src:
|
||||
ast = call.src[0]
|
||||
for bufs, _ in unwrap_multi(call, resolve_params(call, ())):
|
||||
if ast.op is Ops.COPY:
|
||||
# Handle COPY: extract source data to initialize destination buffer
|
||||
if len(bufs) >= 2:
|
||||
dst_buf, src_buf = bufs[0], bufs[1]
|
||||
dst_id = id(dst_buf)
|
||||
if dst_id not in buf_pool:
|
||||
buf_pool[dst_id] = dst_buf.nbytes
|
||||
# Get source data if it's from numpy/CPU
|
||||
if hasattr(src_buf, 'base') and src_buf.base is not None and src_buf.base.is_allocated():
|
||||
src_data = bytes(src_buf.base._buf)
|
||||
buf_data[dst_id] = src_data
|
||||
elif ast.op is Ops.PROGRAM:
|
||||
info = ast.arg
|
||||
if len(ast.src) > 4 and ast.src[4].op is Ops.BINARY:
|
||||
lib = bytes(ast.src[4].arg)
|
||||
_, sections, _ = elf_loader(lib)
|
||||
for sec in sections:
|
||||
if sec.name == '.text':
|
||||
buf_idxs = []
|
||||
buf_sizes = []
|
||||
for b in bufs:
|
||||
buf_id = id(b)
|
||||
if buf_id not in buf_pool:
|
||||
buf_pool[buf_id] = b.nbytes
|
||||
buf_idxs.append(buf_id)
|
||||
buf_sizes.append(b.nbytes)
|
||||
kernels.append(KernelSnapshot(
|
||||
code=bytes(sec.content),
|
||||
src=ast.src[3].arg,
|
||||
global_size=tuple(info.global_size),
|
||||
local_size=tuple(info.local_size),
|
||||
buf_idxs=buf_idxs,
|
||||
buf_sizes=buf_sizes
|
||||
))
|
||||
if not kernels: raise RuntimeError("No kernel found")
|
||||
return kernels, buf_pool, buf_data
|
||||
|
||||
@unittest.skipUnless(Device.DEFAULT == "AMD", "requires AMD device")
|
||||
class TestTinygradKernelRoundtrip(unittest.TestCase):
|
||||
"""Test roundtrip on real tinygrad-generated kernels using get_kernels_from_tinygrad pattern."""
|
||||
arch = 'rdna3'
|
||||
|
||||
def _test_kernel_roundtrip(self, op_fn):
|
||||
"""Generate kernel from op_fn, test:
|
||||
1. decode -> reencode matches original bytes
|
||||
2. disasm() -> LLVM asm -> bytes matches original (validates disasm correctness)
|
||||
3. our disasm() matches LLVM's disassembly string (informational)
|
||||
"""
|
||||
arch = self.arch
|
||||
|
||||
from tinygrad.runtime.support.elf import elf_loader
|
||||
from tinygrad.runtime.support.compiler_amd import HIPCompiler, AMDLLVMCompiler
|
||||
from tinygrad.helpers import DEV
|
||||
|
||||
kernels, _, _ = get_kernels_from_tinygrad(op_fn)
|
||||
# rendered source can be C or llvmir
|
||||
compiler = (AMDLLVMCompiler if DEV.renderer == "LLVM" else HIPCompiler)(get_target(arch))
|
||||
|
||||
# First pass: decode all instructions and collect info
|
||||
decoded_instrs: list[tuple] = [] # list of (ki, offset, orig_bytes, decoded, our_disasm, decode_ok, decode_err)
|
||||
for ki, kernel in enumerate(kernels):
|
||||
offset = 0
|
||||
code = next((s.content for s in elf_loader(compiler.compile(kernel.src))[1] if s.name == ".text"))
|
||||
while offset < len(code):
|
||||
remaining = code[offset:]
|
||||
fmt = detect_format(remaining, arch)
|
||||
base_size = fmt._size()
|
||||
if len(remaining) < base_size:
|
||||
break
|
||||
|
||||
try:
|
||||
decoded = fmt.from_bytes(remaining) # pass all remaining bytes so from_bytes can read literal
|
||||
size = decoded.size() # actual size including literal
|
||||
orig_bytes = remaining[:size]
|
||||
reencoded = decoded.to_bytes()
|
||||
our_disasm = disasm(decoded)
|
||||
decode_ok = reencoded == orig_bytes
|
||||
decode_err: str | None = None if decode_ok else f"orig={orig_bytes.hex()} reenc={reencoded.hex()}"
|
||||
decoded_instrs.append((ki, offset, orig_bytes, decoded, our_disasm, decode_ok, decode_err))
|
||||
except Exception as e:
|
||||
decoded_instrs.append((ki, offset, remaining[:base_size], None, None, False, str(e)))
|
||||
size = base_size
|
||||
|
||||
offset += size
|
||||
|
||||
# Collect disasm strings for batched LLVM calls - skip unknown opcodes (op_X) that LLVM can't compile
|
||||
asm_test_instrs: list[tuple[int, str, bytes]] = [] # (idx, our_disasm, orig_bytes) for asm test
|
||||
disasm_test_instrs: list[tuple[int, str]] = [] # (idx, our_disasm) for disasm comparison test
|
||||
|
||||
for idx, (ki, offset, orig_bytes, decoded, our_disasm, decode_ok, decode_err) in enumerate(decoded_instrs):
|
||||
if our_disasm is None: continue
|
||||
# Skip unknown opcodes and malformed instructions
|
||||
if our_disasm.startswith('op_') or re.search(r', \d+, \d+, \d+,', our_disasm): continue
|
||||
asm_test_instrs.append((idx, our_disasm, orig_bytes))
|
||||
disasm_test_instrs.append((idx, our_disasm))
|
||||
|
||||
# Batch compile for asm test (our disasm -> LLVM asm -> bytes)
|
||||
asm_llvm_results = compile_asm_batch([d for _, d, _ in asm_test_instrs], arch)
|
||||
asm_llvm_map = {idx: (result, orig) for (idx, _, orig), result in zip(asm_test_instrs, asm_llvm_results)}
|
||||
|
||||
# Batch compile+disasm for disasm comparison test
|
||||
disasm_llvm_results = compile_and_disasm_batch([d for _, d in disasm_test_instrs], arch)
|
||||
disasm_llvm_map = {idx: result for (idx, _), result in zip(disasm_test_instrs, disasm_llvm_results)}
|
||||
|
||||
# Now evaluate results
|
||||
decode_passed, decode_failed, decode_skipped = 0, 0, 0
|
||||
asm_passed, asm_failed, asm_skipped = 0, 0, 0
|
||||
disasm_passed, disasm_failed, disasm_skipped = 0, 0, 0
|
||||
decode_failures: list[str] = []
|
||||
asm_failures: list[str] = []
|
||||
disasm_failures: list[str] = []
|
||||
|
||||
for idx, (ki, offset, orig_bytes, decoded, our_disasm, decode_ok, decode_err) in enumerate(decoded_instrs):
|
||||
# Decode test
|
||||
if decode_ok:
|
||||
decode_passed += 1
|
||||
elif decode_err == "no format":
|
||||
decode_skipped += 1
|
||||
else:
|
||||
decode_failed += 1
|
||||
decode_failures.append(f"K{ki}@{offset}: {our_disasm}: {decode_err}")
|
||||
|
||||
# Asm test: our disasm -> LLVM asm -> compare bytes with original
|
||||
if our_disasm is None:
|
||||
asm_skipped += 1
|
||||
elif idx in asm_llvm_map:
|
||||
llvm_bytes, orig = asm_llvm_map[idx]
|
||||
if llvm_bytes == orig[:len(llvm_bytes)]:
|
||||
asm_passed += 1
|
||||
else:
|
||||
asm_failed += 1
|
||||
asm_failures.append(f"K{ki}@{offset}: '{our_disasm}': llvm={llvm_bytes.hex()} orig={orig[:len(llvm_bytes)].hex()}")
|
||||
else:
|
||||
asm_skipped += 1
|
||||
|
||||
# Disasm comparison test
|
||||
if our_disasm is None:
|
||||
disasm_skipped += 1
|
||||
elif idx in disasm_llvm_map:
|
||||
llvm_disasm_str = disasm_llvm_map[idx]
|
||||
if our_disasm == llvm_disasm_str:
|
||||
disasm_passed += 1
|
||||
else:
|
||||
disasm_failed += 1
|
||||
disasm_failures.append(f"K{ki}@{offset}: ours='{our_disasm}' llvm='{llvm_disasm_str}'")
|
||||
else:
|
||||
disasm_skipped += 1
|
||||
|
||||
print(f"[{arch}] decode roundtrip: {decode_passed} passed, {decode_failed} failed, {decode_skipped} skipped")
|
||||
print(f"[{arch}] asm via llvm: {asm_passed} passed, {asm_failed} failed, {asm_skipped} skipped")
|
||||
print(f"[{arch}] disasm vs llvm: {disasm_passed} passed, {disasm_failed} failed, {disasm_skipped} skipped")
|
||||
self.assertEqual(decode_failed, 0, "Decode failures:\n" + "\n".join(decode_failures[:20]))
|
||||
self.assertEqual(asm_failed, 0, "Asm failures:\n" + "\n".join(asm_failures[:20]))
|
||||
# Note: disasm string comparison is informational only - formatting differences between LLVM versions are expected
|
||||
|
||||
# Basic unary ops
|
||||
def test_neg(self): self._test_kernel_roundtrip(lambda T: -T([1.0, -2.0, 3.0, -4.0]))
|
||||
def test_relu(self): self._test_kernel_roundtrip(lambda T: T([-1.0, 0.0, 1.0, 2.0]).relu())
|
||||
def test_exp(self): self._test_kernel_roundtrip(lambda T: T([0.0, 1.0, 2.0]).exp())
|
||||
def test_log(self): self._test_kernel_roundtrip(lambda T: T([1.0, 2.0, 3.0]).log())
|
||||
def test_sin(self): self._test_kernel_roundtrip(lambda T: T([0.0, 1.0, 2.0]).sin())
|
||||
def test_sqrt(self): self._test_kernel_roundtrip(lambda T: T([1.0, 4.0, 9.0]).sqrt())
|
||||
def test_recip(self): self._test_kernel_roundtrip(lambda T: T([1.0, 2.0, 4.0]).reciprocal())
|
||||
|
||||
# Binary ops
|
||||
def test_add(self): self._test_kernel_roundtrip(lambda T: T([1.0, 2.0]) + T([3.0, 4.0]))
|
||||
def test_sub(self): self._test_kernel_roundtrip(lambda T: T([5.0, 6.0]) - T([1.0, 2.0]))
|
||||
def test_mul(self): self._test_kernel_roundtrip(lambda T: T([2.0, 3.0]) * T([4.0, 5.0]))
|
||||
def test_div(self): self._test_kernel_roundtrip(lambda T: T([10.0, 20.0]) / T([2.0, 4.0]))
|
||||
def test_max_binary(self): self._test_kernel_roundtrip(lambda T: T([1.0, 5.0]).maximum(T([3.0, 2.0])))
|
||||
|
||||
# Reductions
|
||||
def test_sum_reduce(self): self._test_kernel_roundtrip(lambda T: T.empty(64).sum())
|
||||
def test_max_reduce(self): self._test_kernel_roundtrip(lambda T: T.empty(64).max())
|
||||
def test_mean_reduce(self): self._test_kernel_roundtrip(lambda T: T.empty(32).mean())
|
||||
|
||||
# Matmul
|
||||
def test_gemm_4x4(self): self._test_kernel_roundtrip(lambda T: T.empty(4, 4) @ T.empty(4, 4))
|
||||
def test_gemv(self): self._test_kernel_roundtrip(lambda T: T.empty(1, 16) @ T.empty(16, 16))
|
||||
|
||||
# Complex ops
|
||||
def test_softmax(self): self._test_kernel_roundtrip(lambda T: T.empty(16).softmax())
|
||||
def test_layernorm(self): self._test_kernel_roundtrip(lambda T: T.empty(8, 8).layernorm())
|
||||
|
||||
# Memory patterns
|
||||
def test_contiguous(self): self._test_kernel_roundtrip(lambda T: T.empty(4, 4).permute(1, 0).contiguous())
|
||||
def test_reshape(self): self._test_kernel_roundtrip(lambda T: (T.empty(16) + 1).reshape(4, 4).contiguous())
|
||||
def test_expand(self): self._test_kernel_roundtrip(lambda T: T.empty(4, 1).expand(4, 4).contiguous())
|
||||
|
||||
# Cast ops
|
||||
def test_cast_int(self): self._test_kernel_roundtrip(lambda T: T.empty(16).int().float())
|
||||
def test_cast_half(self): self._test_kernel_roundtrip(lambda T: T.empty(16).half().float())
|
||||
|
||||
# Comparison ops
|
||||
def test_cmp_lt(self): self._test_kernel_roundtrip(lambda T: (T.empty(64) < T.empty(64)).where(T.empty(64), T.empty(64)))
|
||||
def test_where(self): self._test_kernel_roundtrip(lambda T: (T.empty(64) > 0).where(T.empty(64), T.empty(64)))
|
||||
|
||||
# Fused ops
|
||||
def test_fma(self): self._test_kernel_roundtrip(lambda T: (T([1.0, 2.0]) * T([3.0, 4.0]) + T([5.0, 6.0])))
|
||||
|
||||
class TestTinygradKernelRoundtripRDNA4(TestTinygradKernelRoundtrip): arch = 'rdna4'
|
||||
|
||||
@unittest.skip("CDNA decode roundtrip not yet supported")
|
||||
class TestTinygradKernelRoundtripCDNA(TestTinygradKernelRoundtrip): arch = 'cdna'
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
108
tinygrad_repo/test/amd/test_sqtt_encoder.py
Normal file
108
tinygrad_repo/test/amd/test_sqtt_encoder.py
Normal file
@@ -0,0 +1,108 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests for SQTT encoder: verifies the emulator produces correct SQTT traces for known kernels.
|
||||
|
||||
Run with: DEV=MOCK+AMD python -m pytest test/amd/test_sqtt_encoder.py -v
|
||||
"""
|
||||
import ctypes, unittest
|
||||
from tinygrad.helpers import Context
|
||||
from tinygrad.renderer.amd.sqtt import decode, LAYOUT_HEADER, WAVESTART, WAVEEND, INST, IMMEDIATE, VALUINST, InstOp
|
||||
from tinygrad.runtime.autogen.amd.rdna3.ins import *
|
||||
|
||||
def _run_kernel(instructions: list, lx=1, ly=1, lz=1, gx=1, gy=1, gz=1, args_ptr=0) -> bytes:
|
||||
"""Assemble instructions, run on emulator with PROFILE=1, return the SQTT blob."""
|
||||
from test.mockgpu.amd.emu import run_asm, sqtt_traces
|
||||
code = b''.join(inst.to_bytes() for inst in instructions)
|
||||
buf = (ctypes.c_char * len(code))(*code)
|
||||
lib = ctypes.addressof(buf)
|
||||
sqtt_traces.clear()
|
||||
with Context(PROFILE=1):
|
||||
run_asm(lib, len(code), gx, gy, gz, lx, ly, lz, args_ptr)
|
||||
assert len(sqtt_traces) == 1, f"expected 1 trace, got {len(sqtt_traces)}"
|
||||
return sqtt_traces.pop()
|
||||
|
||||
class TestSQTTEncoder(unittest.TestCase):
|
||||
|
||||
def test_simple_salu(self):
|
||||
"""A simple s_mov + s_endpgm kernel emits SALU INST packet."""
|
||||
blob = _run_kernel([s_mov_b32(s[0], 42), s_endpgm()])
|
||||
packets = list(decode(blob))
|
||||
inst_pkts = [p for p in packets if isinstance(p, INST)]
|
||||
self.assertEqual(len(inst_pkts), 1)
|
||||
self.assertEqual(inst_pkts[0].op, InstOp.SALU)
|
||||
|
||||
def test_valu_emits_valuinst(self):
|
||||
"""Regular VALU ops emit VALUINST packets."""
|
||||
blob = _run_kernel([v_mov_b32_e32(v[0], 0), v_add_f32_e32(v[1], v[0], v[0]), s_endpgm()])
|
||||
packets = list(decode(blob))
|
||||
valu_pkts = [p for p in packets if isinstance(p, VALUINST)]
|
||||
self.assertEqual(len(valu_pkts), 2)
|
||||
# no INST packets for regular VALU
|
||||
self.assertEqual(len([p for p in packets if isinstance(p, INST)]), 0)
|
||||
|
||||
def test_waitcnt_emits_immediate(self):
|
||||
"""s_waitcnt and s_nop emit IMMEDIATE packets."""
|
||||
blob = _run_kernel([s_nop(simm16=0), s_waitcnt(simm16=0), s_endpgm()])
|
||||
imm_pkts = [p for p in decode(blob) if isinstance(p, IMMEDIATE)]
|
||||
self.assertEqual(len(imm_pkts), 2) # s_nop + s_waitcnt
|
||||
|
||||
def test_endpgm_skipped(self):
|
||||
"""s_endpgm does not emit any packet."""
|
||||
blob = _run_kernel([s_endpgm()])
|
||||
packets = list(decode(blob))
|
||||
self.assertEqual(len([p for p in packets if isinstance(p, INST)]), 0)
|
||||
self.assertEqual(len([p for p in packets if isinstance(p, IMMEDIATE)]), 0)
|
||||
|
||||
def test_wave_lifecycle(self):
|
||||
"""Every WAVESTART has a matching WAVEEND."""
|
||||
blob = _run_kernel([s_mov_b32(s[0], 0), s_endpgm()])
|
||||
packets = list(decode(blob))
|
||||
self.assertEqual(sum(1 for p in packets if isinstance(p, WAVESTART)), sum(1 for p in packets if isinstance(p, WAVEEND)))
|
||||
|
||||
def test_layout_header(self):
|
||||
"""First packet is LAYOUT_HEADER with layout=3."""
|
||||
blob = _run_kernel([s_endpgm()])
|
||||
packets = list(decode(blob))
|
||||
self.assertIsInstance(packets[0], LAYOUT_HEADER)
|
||||
self.assertEqual(packets[0].layout, 3)
|
||||
|
||||
def test_blob_32byte_aligned(self):
|
||||
"""SQTT blob is 32-byte aligned."""
|
||||
blob = _run_kernel([s_mov_b32(s[0], 0), s_mov_b32(s[1], 1), s_endpgm()])
|
||||
self.assertEqual(len(blob) % 32, 0)
|
||||
|
||||
def test_multiple_waves(self):
|
||||
"""Multiple wavefronts each get their own WAVESTART/WAVEEND."""
|
||||
blob = _run_kernel([s_mov_b32(s[0], 0), s_endpgm()], lx=64) # 64 threads = 2 waves (WAVE_SIZE=32)
|
||||
packets = list(decode(blob))
|
||||
self.assertEqual(sum(1 for p in packets if isinstance(p, WAVESTART)), 2)
|
||||
self.assertEqual(sum(1 for p in packets if isinstance(p, WAVEEND)), 2)
|
||||
|
||||
def test_branch_taken_and_not_taken(self):
|
||||
"""A loop with s_cbranch_scc1 emits JUMP when taken, JUMP_NO on final iteration."""
|
||||
# s[0] = 2; loop: s[0] -= 1; cmp s[0] != 0 (SCC=1 if true); cbranch_scc1 loop; endpgm
|
||||
# iteration 1: s[0]=2→1, SCC=1 (1!=0), branch taken (JUMP)
|
||||
# iteration 2: s[0]=1→0, SCC=0 (0==0), branch not taken (JUMP_NO)
|
||||
blob = _run_kernel([s_mov_b32(s[0], 2), s_sub_u32(s[0], s[0], 1), s_cmp_lg_u32(s[0], 0), s_cbranch_scc1(simm16=-3), s_endpgm()])
|
||||
inst_pkts = [p for p in decode(blob) if isinstance(p, INST)]
|
||||
ops = [p.op for p in inst_pkts]
|
||||
self.assertIn(InstOp.JUMP, ops)
|
||||
self.assertIn(InstOp.JUMP_NO, ops)
|
||||
|
||||
def test_timestamps_monotonic(self):
|
||||
"""Timestamps are monotonically non-decreasing."""
|
||||
blob = _run_kernel([s_mov_b32(s[0], 0), s_mov_b32(s[1], 1), s_mov_b32(s[2], 2), s_endpgm()])
|
||||
times = [p._time for p in decode(blob)]
|
||||
self.assertEqual(times, sorted(times))
|
||||
|
||||
def test_no_trace_without_profile(self):
|
||||
"""No SQTT trace is emitted when PROFILE=0."""
|
||||
from test.mockgpu.amd.emu import run_asm, sqtt_traces
|
||||
code = s_endpgm().to_bytes()
|
||||
buf = (ctypes.c_char * len(code))(*code)
|
||||
sqtt_traces.clear()
|
||||
with Context(PROFILE=0):
|
||||
run_asm(ctypes.addressof(buf), len(code), 1, 1, 1, 1, 1, 1, 0)
|
||||
self.assertEqual(len(sqtt_traces), 0)
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
235
tinygrad_repo/test/amd/test_sqtt_examples.py
Normal file
235
tinygrad_repo/test/amd/test_sqtt_examples.py
Normal file
@@ -0,0 +1,235 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests for SQTT packet decoding using real captured examples."""
|
||||
import pickle, unittest, ctypes, threading
|
||||
from pathlib import Path
|
||||
from tinygrad.helpers import DEBUG
|
||||
from tinygrad.runtime.autogen import rocprof
|
||||
from tinygrad.runtime.support.elf import elf_loader
|
||||
from tinygrad.renderer.amd import decode_inst
|
||||
from tinygrad.runtime.autogen.amd.rdna3.ins import SOPP
|
||||
from tinygrad.runtime.autogen.amd.rdna3.enum import SOPPOp
|
||||
from tinygrad.renderer.amd.sqtt import (decode, LAYOUT_HEADER, WAVESTART, WAVESTART_RDNA4, WAVEEND, WAVEEND_RDNA4, INST, INST_RDNA4, VALUINST,
|
||||
IMMEDIATE, IMMEDIATE_MASK, PACKET_TYPES_RDNA3, PACKET_TYPES_RDNA4, PACKET_TYPES_CDNA, CDNA_WAVESTART,
|
||||
print_packets, CDNA_WAVEEND, CDNA_INST)
|
||||
from test.amd.helpers import TARGET_TO_ARCH
|
||||
from test.amd.test_sqttmap import needs_rocprof
|
||||
|
||||
import tinygrad
|
||||
EXAMPLES_DIR = Path(tinygrad.__file__).parent.parent / "extra/sqtt/examples"
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# ROCPROF DECODER
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
def run_rocprof_decoder(blobs: list[bytes], lib: bytes, base: int, target: str):
|
||||
"""Run rocprof decoder on SQTT blobs, returning raw occupancy and instruction records."""
|
||||
image, sections, _ = elf_loader(lib)
|
||||
text = next((sh for sh in sections if sh.name == ".text"), None)
|
||||
assert text is not None, "no .text section found"
|
||||
text_off, text_size = text.header.sh_addr, text.header.sh_size
|
||||
|
||||
blob_iter, current_blob = iter(blobs), [None] # type: ignore[var-annotated]
|
||||
occupancy_records: list[tuple[int, int, int, int, bool]] = [] # (wave_id, simd, cu, time, is_start)
|
||||
wave_insts: list[list[tuple[int, int]]] = [] # per-wave list of (time, stall)
|
||||
|
||||
@rocprof.rocprof_trace_decoder_se_data_callback_t
|
||||
def copy_cb(buf, buf_size, _): # type: ignore[no-untyped-def]
|
||||
blob = next(blob_iter, None)
|
||||
if blob is None: return 0
|
||||
current_blob[0] = (ctypes.c_ubyte * len(blob)).from_buffer_copy(blob) # type: ignore[call-overload]
|
||||
buf[0] = ctypes.cast(current_blob[0], ctypes.POINTER(ctypes.c_ubyte)) # type: ignore[arg-type]
|
||||
buf_size[0] = len(current_blob[0]) # type: ignore[arg-type]
|
||||
return len(current_blob[0]) # type: ignore[arg-type]
|
||||
|
||||
@rocprof.rocprof_trace_decoder_trace_callback_t
|
||||
def trace_cb(record_type, events_ptr, n, _):
|
||||
if record_type == rocprof.ROCPROFILER_THREAD_TRACE_DECODER_RECORD_OCCUPANCY:
|
||||
for ev in (rocprof.rocprofiler_thread_trace_decoder_occupancy_t * n).from_address(events_ptr):
|
||||
occupancy_records.append((ev.wave_id, ev.simd, ev.cu, ev.time, ev.start))
|
||||
elif record_type == rocprof.ROCPROFILER_THREAD_TRACE_DECODER_RECORD_WAVE:
|
||||
for ev in (rocprof.rocprofiler_thread_trace_decoder_wave_t * n).from_address(events_ptr):
|
||||
if ev.instructions_size > 0:
|
||||
sz = ev.instructions_size * ctypes.sizeof(rocprof.rocprofiler_thread_trace_decoder_inst_t)
|
||||
insts_blob = bytearray(sz)
|
||||
ctypes.memmove((ctypes.c_char * sz).from_buffer(insts_blob), ev.instructions_array, sz)
|
||||
insts = list((rocprof.rocprofiler_thread_trace_decoder_inst_t * ev.instructions_size).from_buffer(insts_blob))
|
||||
wave_insts.append([(inst.time, inst.stall) for inst in insts])
|
||||
return rocprof.ROCPROFILER_THREAD_TRACE_DECODER_STATUS_SUCCESS
|
||||
|
||||
arch = TARGET_TO_ARCH[target]
|
||||
@rocprof.rocprof_trace_decoder_isa_callback_t
|
||||
def isa_cb(instr_ptr, mem_size_ptr, size_ptr, pc, _):
|
||||
offset = pc.address - base
|
||||
if offset < text_off or offset >= text_off + text_size:
|
||||
mem_size_ptr[0] = 0
|
||||
return rocprof.ROCPROFILER_THREAD_TRACE_DECODER_STATUS_SUCCESS
|
||||
try:
|
||||
inst = decode_inst(image[offset:], arch=arch)
|
||||
mem_size_ptr[0] = inst._size()
|
||||
# this could be an error in our decode_inst
|
||||
except (ValueError, AssertionError):
|
||||
mem_size_ptr[0] = 0
|
||||
return rocprof.ROCPROFILER_THREAD_TRACE_DECODER_STATUS_SUCCESS
|
||||
if isinstance(inst, SOPP) and inst.op == SOPPOp.S_ENDPGM: mem_size_ptr[0] = 0
|
||||
# rocprof parses instruction string to determine type; v_nop works for all
|
||||
if (max_sz := size_ptr[0]) == 0: return rocprof.ROCPROFILER_THREAD_TRACE_DECODER_STATUS_ERROR_OUT_OF_RESOURCES
|
||||
ctypes.memmove(instr_ptr, b"v_nop", min(5, max_sz - 1))
|
||||
size_ptr[0] = min(5, max_sz - 1)
|
||||
return rocprof.ROCPROFILER_THREAD_TRACE_DECODER_STATUS_SUCCESS
|
||||
|
||||
exc = None
|
||||
def worker():
|
||||
nonlocal exc
|
||||
try: rocprof.rocprof_trace_decoder_parse_data(copy_cb, trace_cb, isa_cb, None)
|
||||
except Exception as e: exc = e
|
||||
(t:=threading.Thread(target=worker, daemon=True)).start()
|
||||
t.join(timeout=5)
|
||||
if exc is not None: raise exc
|
||||
if t.is_alive(): raise RuntimeError("rocprof decoder timeout")
|
||||
return occupancy_records, wave_insts
|
||||
|
||||
class SQTTExamplesTestBase(unittest.TestCase):
|
||||
target: str
|
||||
examples: dict
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
if cls is SQTTExamplesTestBase: raise unittest.SkipTest("base class")
|
||||
cls.examples = {}
|
||||
for pkl_path in sorted((EXAMPLES_DIR/cls.target).glob("*.pkl")):
|
||||
with open(pkl_path, "rb") as f:
|
||||
data = pickle.load(f)
|
||||
sqtt_events = [e for e in data if type(e).__name__ == "ProfileSQTTEvent"]
|
||||
prg = next((e for e in data if type(e).__name__ == "ProfileProgramEvent"), None)
|
||||
if sqtt_events and prg:
|
||||
cls.examples[pkl_path.stem] = (sqtt_events, prg.lib, prg.base)
|
||||
|
||||
def test_examples_loaded(self):
|
||||
self.assertGreater(len(self.examples), 0, "no example files found")
|
||||
|
||||
def test_decode_all_examples(self):
|
||||
for name, (events, *_) in self.examples.items():
|
||||
for i, event in enumerate(events):
|
||||
with self.subTest(example=name, event=i):
|
||||
packets = list(decode(event.blob))
|
||||
if DEBUG >= 2:
|
||||
print(f"\n=== {name} event {i} ===")
|
||||
print_packets(packets)
|
||||
self.assertGreater(len(packets), 0, f"no packets decoded from {name} event {i}")
|
||||
self.assertIsInstance(packets[0], LAYOUT_HEADER, f"first packet should be LAYOUT_HEADER in {name}")
|
||||
|
||||
def test_packet_types_valid(self):
|
||||
all_classes = set(PACKET_TYPES_RDNA3.values()) | set(PACKET_TYPES_RDNA4.values()) | set(PACKET_TYPES_CDNA.values())
|
||||
for name, (events, *_) in self.examples.items():
|
||||
for i, event in enumerate(events):
|
||||
with self.subTest(example=name, event=i):
|
||||
for pkt in decode(event.blob):
|
||||
# Use isinstance to handle layout-specific subclasses (e.g., WAVESTART_RDNA4)
|
||||
self.assertTrue(any(isinstance(pkt, cls) for cls in all_classes), f"unknown packet type {type(pkt)} in {name}")
|
||||
|
||||
def test_wave_lifecycle(self):
|
||||
for name, (events, *_) in self.examples.items():
|
||||
if "empty" in name: continue
|
||||
with self.subTest(example=name):
|
||||
all_packets = [p for e in events for p in decode(e.blob)]
|
||||
self.assertGreater(len([p for p in all_packets if isinstance(p, (WAVESTART, WAVESTART_RDNA4, CDNA_WAVESTART))]), 0, f"no WAVESTART in {name}")
|
||||
self.assertGreater(len([p for p in all_packets if isinstance(p, (WAVEEND, WAVEEND_RDNA4, CDNA_WAVEEND))]), 0, f"no WAVEEND in {name}")
|
||||
|
||||
def test_time_monotonic(self):
|
||||
for name, (events, *_) in self.examples.items():
|
||||
for i, event in enumerate(events):
|
||||
with self.subTest(example=name, event=i):
|
||||
times = [p._time for p in decode(event.blob)]
|
||||
self.assertEqual(times, sorted(times), f"timestamps not monotonic in {name}")
|
||||
|
||||
def test_gemm_has_instructions(self):
|
||||
for name, (events, *_) in self.examples.items():
|
||||
if "gemm" not in name: continue
|
||||
with self.subTest(example=name):
|
||||
all_packets = [p for e in events for p in decode(e.blob)]
|
||||
inst_packets = [p for p in all_packets if isinstance(p, (INST, INST_RDNA4, CDNA_INST))]
|
||||
self.assertGreater(len(inst_packets), 0, f"no INST packets in {name}")
|
||||
if isinstance(inst_packets[0], (INST, INST_RDNA4)):
|
||||
self.assertGreater(len([p for p in inst_packets if p.op.name.startswith("JUMP")]), 0, f"no JUMP packets in {name}")
|
||||
|
||||
expected: dict[str, list[int]] = {} # override in subclasses
|
||||
def test_packet_counts(self):
|
||||
if not self.expected: self.skipTest("no expected packet counts for this target")
|
||||
for name, (events, *_) in self.examples.items():
|
||||
with self.subTest(example=name):
|
||||
if not self.expected.get(name): continue
|
||||
counts = [len(list(decode(e.blob))) for e in events]
|
||||
self.assertEqual(counts, self.expected[name], f"packet count mismatch in {name}")
|
||||
|
||||
@needs_rocprof
|
||||
def test_rocprof_wave_times_match(self):
|
||||
"""Wave start/end times must match rocprof exactly."""
|
||||
for name, (events, lib, base) in self.examples.items():
|
||||
with self.subTest(example=name):
|
||||
occupancy, _ = run_rocprof_decoder([e.blob for e in events], lib, base, self.target)
|
||||
# extract from rocprof occupancy records
|
||||
roc_starts: dict[tuple[int, int, int], int] = {}
|
||||
roc_waves: list[tuple[int, int]] = []
|
||||
for wave_id, simd, cu, time, is_start in occupancy:
|
||||
key = (wave_id, simd, cu)
|
||||
if is_start: roc_starts[key] = time
|
||||
elif key in roc_starts: roc_waves.append((roc_starts.pop(key), time))
|
||||
# extract from our decoder
|
||||
our_waves: list[tuple[int, int]] = []
|
||||
for event in events:
|
||||
wave_starts: dict[tuple[int, int, int], int] = {}
|
||||
first_timestamp:int|None = None
|
||||
for p in decode(event.blob):
|
||||
if first_timestamp is None: first_timestamp = p._time
|
||||
if isinstance(p, (WAVESTART, CDNA_WAVESTART, WAVESTART_RDNA4)): wave_starts[(p.wave, p.simd, p.cu)] = p._time
|
||||
elif isinstance(p, (WAVEEND, WAVEEND_RDNA4, CDNA_WAVEEND)) and (key := (p.wave, p.simd, p.cu)) in wave_starts:
|
||||
our_waves.append((wave_starts[key], p._time))
|
||||
for st in wave_starts.values():
|
||||
self.assertGreater(st, first_timestamp, "wave start must be after the first packet")
|
||||
# rocprof fails non deterministically and gives inaccurate timestamps.
|
||||
#self.assertEqual(sorted(our_waves), sorted(roc_waves), f"wave times mismatch in {name}")
|
||||
for st, et in our_waves:
|
||||
self.assertGreater(et, st, "wave end must be after start")
|
||||
|
||||
@needs_rocprof
|
||||
def test_rocprof_inst_times_match(self):
|
||||
"""Instruction times must match rocprof exactly (excluding s_endpgm)."""
|
||||
for name, (events, lib, base) in self.examples.items():
|
||||
with self.subTest(example=name):
|
||||
_, wave_insts = run_rocprof_decoder([e.blob for e in events], lib, base, self.target)
|
||||
# skip last inst per wave (s_endpgm) - it needs special handling (time + duration instead of time + stall)
|
||||
roc_insts = [time + stall for insts in wave_insts for time, stall in insts[:-1]]
|
||||
# extract from our decoder
|
||||
our_insts: list[int] = []
|
||||
for event in events:
|
||||
for p in decode(event.blob):
|
||||
# INST ops for non-traced SIMDs (excluded from instruction count)
|
||||
if isinstance(p, (INST, INST_RDNA4)) and not p.op.name.startswith("OTHER_"): our_insts.append(p._time)
|
||||
elif isinstance(p, VALUINST): our_insts.append(p._time)
|
||||
elif isinstance(p, IMMEDIATE): our_insts.append(p._time)
|
||||
elif isinstance(p, IMMEDIATE_MASK):
|
||||
for _ in range(bin(p.mask).count('1')): our_insts.append(p._time)
|
||||
self.assertEqual(sorted(our_insts), sorted(roc_insts), f"instruction times mismatch in {name}")
|
||||
|
||||
class TestSQTTExamplesRDNA3(SQTTExamplesTestBase):
|
||||
target = "gfx1100"
|
||||
expected = {
|
||||
"profile_empty_run_0": [1880, 1867, 1920, 1971, 1998, 1904],
|
||||
"profile_empty_run_1": [1880, 1867, 1920, 1971, 1998, 1904],
|
||||
"profile_gemm_run_0": [3275, 3278, 2426, 2475, 2511, 2431],
|
||||
"profile_gemm_run_1": [3264, 3268, 2420, 2469, 2504, 2401],
|
||||
"profile_ops_run_0": [1944, 4903, 1984, 2035, 2062, 1968],
|
||||
"profile_ops_run_1": [1944, 4918, 1984, 2035, 2062, 1968],
|
||||
"profile_plus_run_0": [1938, 1932, 1978, 2029, 2056, 1962],
|
||||
"profile_plus_run_1": [1891, 1874, 1931, 1982, 2009, 1915],
|
||||
}
|
||||
|
||||
class TestSQTTExamplesRDNA4(SQTTExamplesTestBase): target = "gfx1200"
|
||||
|
||||
class TestSQTTExamplesCDNA(SQTTExamplesTestBase):
|
||||
target = "gfx950"
|
||||
def test_rocprof_wave_times_match(self): self.skipTest("TODO: requires timestamp patching")
|
||||
def test_rocprof_inst_times_match(self): self.skipTest("TODO: requires timestamp patching")
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
99
tinygrad_repo/test/amd/test_sqtt_profiler.py
Normal file
99
tinygrad_repo/test/amd/test_sqtt_profiler.py
Normal file
@@ -0,0 +1,99 @@
|
||||
import unittest, contextlib
|
||||
from tinygrad import Device, Tensor, Context, TinyJit
|
||||
from tinygrad.device import Compiled, ProfileProgramEvent, ProfileDeviceEvent
|
||||
from tinygrad.engine.realize import run_linear
|
||||
from tinygrad.codegen import to_program
|
||||
from tinygrad.viz.serve import load_amd_counters, VizData
|
||||
|
||||
@contextlib.contextmanager
|
||||
def save_sqtt():
|
||||
data = VizData()
|
||||
yield data.ctxs
|
||||
Device[Device.DEFAULT].synchronize()
|
||||
Device[Device.DEFAULT]._at_profile_finalize()
|
||||
load_amd_counters(data, Compiled.profile_events)
|
||||
data.ctxs[:] = [r for r in data.ctxs if r["name"].startswith("SQTT")]
|
||||
|
||||
@unittest.skipUnless(Device.DEFAULT == "AMD", "only runs on AMD")
|
||||
class TestSQTTProfiler(unittest.TestCase):
|
||||
# TODO: can we enable SQTT profiling in context?
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
if not Device[Device.DEFAULT].sqtt_enabled: raise unittest.SkipTest("device must be in SQTT profiling mode")
|
||||
|
||||
def setUp(self):
|
||||
Device[Device.DEFAULT].synchronize()
|
||||
Compiled.profile_events[:] = [e for e in Compiled.profile_events if isinstance(e, (ProfileProgramEvent, ProfileDeviceEvent))]
|
||||
|
||||
def test_simple(self):
|
||||
t = Tensor.empty(1) + 1
|
||||
with save_sqtt() as sqtt:
|
||||
linear = t.schedule_linear()
|
||||
run_linear(linear)
|
||||
fn_name = to_program(linear.src[0].src[0], renderer=Device[Device.DEFAULT].renderer).arg.function_name
|
||||
self.assertEqual(len(sqtt), 1)
|
||||
self.assertEqual(sqtt[0]["name"], f"SQTT {fn_name}")
|
||||
|
||||
def test_multiple_runs(self):
|
||||
t = Tensor.empty(1) + 1
|
||||
with save_sqtt() as sqtt:
|
||||
linear = t.schedule_linear()
|
||||
for _ in range(N:=3): run_linear(linear)
|
||||
fn_name = to_program(linear.src[0].src[0], renderer=Device[Device.DEFAULT].renderer).arg.function_name
|
||||
self.assertEqual(len(sqtt), N)
|
||||
for i in range(1, N):
|
||||
self.assertEqual(sqtt[i]["name"], f"SQTT {fn_name} n{i+1}")
|
||||
|
||||
def test_multiple_kernels(self):
|
||||
t = ((Tensor.empty(1) + 1).contiguous() + 2)
|
||||
linear = t.schedule_linear()
|
||||
with save_sqtt() as sqtt:
|
||||
run_linear(linear)
|
||||
self.assertEqual(len(sqtt), len(linear.src))
|
||||
for i,call in enumerate(linear.src):
|
||||
fn_name = to_program(call.src[0], renderer=Device[Device.DEFAULT].renderer).arg.function_name
|
||||
self.assertEqual(sqtt[i]["name"], f"SQTT {fn_name}")
|
||||
|
||||
def test_multiple_kernels_lower(self):
|
||||
t = ((Tensor.empty(1) + 1).contiguous() + 2)
|
||||
linear = t.schedule_linear()
|
||||
with save_sqtt() as sqtt:
|
||||
run_linear(linear)
|
||||
self.assertEqual(len(sqtt), len(linear.src))
|
||||
for i,call in enumerate(linear.src):
|
||||
fn_name = to_program(call.src[0], renderer=Device[Device.DEFAULT].renderer).arg.function_name
|
||||
self.assertEqual(sqtt[i]["name"], f"SQTT {fn_name}")
|
||||
|
||||
def test_jit(self):
|
||||
@TinyJit
|
||||
def f(a): return a + 1
|
||||
t = Tensor.empty(1)
|
||||
with save_sqtt() as sqtt:
|
||||
for _ in range(N:=5):
|
||||
f(t).realize()
|
||||
self.assertEqual(len(sqtt), N)
|
||||
kernel_name = sqtt[0]["name"]
|
||||
for i,s in enumerate(sqtt[1:], start=1): self.assertEqual(s["name"], f"{kernel_name} n{i+1}")
|
||||
|
||||
# TODO: can we trace SQTT for graphed kernels?
|
||||
def test_jit_graph(self, kernel_count=3*1):
|
||||
@TinyJit
|
||||
def f(a): return ((a + 1).contiguous() + 2).contiguous().sum()
|
||||
t = Tensor.empty(32)
|
||||
with save_sqtt() as sqtt:
|
||||
for _ in range(5):
|
||||
f(t).realize()
|
||||
names = [s["name"] for s in sqtt]
|
||||
k0, k1, k2 = names[:3]
|
||||
for i in range(3, len(sqtt), 3):
|
||||
n = (i // 3)+1
|
||||
self.assertEqual(names[i], f"{k0} n{n}")
|
||||
self.assertEqual(names[i+1], f"{k1} n{n}")
|
||||
self.assertEqual(names[i+2], f"{k2} n{n}")
|
||||
self.assertEqual(len(sqtt), kernel_count)
|
||||
|
||||
@Context(JIT=2)
|
||||
def test_jit_multiple_kernels(self): self.test_jit_graph(kernel_count=3*5)
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
174
tinygrad_repo/test/amd/test_sqttmap.py
Normal file
174
tinygrad_repo/test/amd/test_sqttmap.py
Normal file
@@ -0,0 +1,174 @@
|
||||
# test to compare every packet with the rocprof decoder
|
||||
import unittest, pickle, functools, json
|
||||
from typing import Iterator
|
||||
from pathlib import Path
|
||||
from tinygrad.helpers import DEBUG, getenv, temp, ansistrip, Context
|
||||
from tinygrad.renderer.amd.sqtt import print_packets, map_insts
|
||||
from tinygrad.runtime.autogen.amd.rdna3.ins import s_endpgm
|
||||
from tinygrad.viz.serve import sqtt_timeline, amd_decode
|
||||
from test.amd.disasm import disasm
|
||||
from test.null.test_viz import run_cli
|
||||
|
||||
import tinygrad
|
||||
EXAMPLES_DIR = Path(tinygrad.__file__).parent.parent / "extra/sqtt/examples"
|
||||
|
||||
def needs_rocprof(fn):
|
||||
@functools.wraps(fn)
|
||||
def wrapper(self, *args, **kwargs):
|
||||
# check if latest rocprof is available, if not, skip rocprof comparison tests
|
||||
# rocprof doesn't have a version string, decode a known pickle to validate it's the latest
|
||||
try:
|
||||
from extra.sqtt.roc import decode as roc_decode
|
||||
with open(EXAMPLES_DIR/"gfx1200"/"profile_plus_run_0.pkl", "rb") as f:
|
||||
data = pickle.load(f)
|
||||
sqtt = [e for e in data if type(e).__name__ == "ProfileSQTTEvent"][1]
|
||||
kern = {e.tag:e for e in data if type(e).__name__ == "ProfileProgramEvent"}[sqtt.kern]
|
||||
rctx = roc_decode([sqtt], {kern.tag:{addr+kern.base:inst for addr,inst in amd_decode(kern.lib, "gfx1200").items()}})
|
||||
insts = [e.time for e in list(rctx.inst_execs.values())[0][0].unpack_insts()]
|
||||
self.assertListEqual(insts, [28178, 28179, 28180, 28181, 28182, 29882, 29883, 29884, 29885, 30966, 30983, 30985, 30992, 30993])
|
||||
except Exception as e: self.skipTest(f"latest rocprof not available, install with extra/sqtt/install_rocprof_decoder.py: {e}")
|
||||
return fn(self, *args, **kwargs)
|
||||
return wrapper
|
||||
|
||||
def rocprof_inst_traces_match(sqtt, prg, target):
|
||||
from extra.sqtt.roc import decode as roc_decode, InstExec
|
||||
addr_table = amd_decode(prg.lib, target)
|
||||
disasm_map = {addr+prg.base:inst for addr,inst in addr_table.items()}
|
||||
rctx = roc_decode([sqtt], {prg.tag:disasm_map})
|
||||
rwaves = rctx.inst_execs.get((sqtt.kern, sqtt.exec_tag), [])
|
||||
rwaves_iter:dict[int, list[Iterator[InstExec]]] = {} # wave unit (0-15) -> list of inst trace iterators for all executions on that unit
|
||||
for w in rwaves: rwaves_iter.setdefault(w.wave_id, []).append(w.unpack_insts())
|
||||
|
||||
if not rwaves: return 0, 0, 0
|
||||
|
||||
passed_insts = 0
|
||||
for pkt, info in map_insts(sqtt.blob, prg.lib, target):
|
||||
if DEBUG >= 2: print_packets([(pkt, info)])
|
||||
if info is None: continue
|
||||
if DEBUG >= 2: print(f"{' '*29}{disasm(info.inst)}")
|
||||
rocprof_inst = next(rwaves_iter[info.wave][0])
|
||||
ref_pc = rocprof_inst.pc-prg.base
|
||||
# always check pc matches
|
||||
assert ref_pc == info.pc, f"pc mismatch {ref_pc}:{disasm_map[rocprof_inst.pc]} != {info.pc}:{disasm(info.inst)}"
|
||||
# special handling for s_endpgm, it marks the wave completion.
|
||||
if info.inst == s_endpgm():
|
||||
completed_wave = list(rwaves_iter[info.wave].pop(0))
|
||||
assert len(completed_wave) == 0, f"incomplete instructions in wave {info.wave}"
|
||||
# otherwise the packet timestamp is time + "stall"
|
||||
else:
|
||||
assert pkt._time == rocprof_inst.time+rocprof_inst.stall
|
||||
passed_insts += 1
|
||||
|
||||
for k,v in rwaves_iter.items():
|
||||
assert len(v) == 0, f"incomplete wave {k}"
|
||||
|
||||
return passed_insts, len(rwaves), len(rwaves_iter)
|
||||
|
||||
class TestSQTTMapBase(unittest.TestCase):
|
||||
target: str
|
||||
examples: dict
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
if cls is TestSQTTMapBase: raise unittest.SkipTest("base class")
|
||||
cls.examples = {}
|
||||
for pkl_path in ([Path(temp("profile.pkl", append_user=True))] if getenv("LOAD_PROFILE") else sorted((EXAMPLES_DIR/cls.target).glob("*.pkl"))):
|
||||
with open(pkl_path, "rb") as f:
|
||||
data = pickle.load(f)
|
||||
sqtt_events = [e for e in data if type(e).__name__ == "ProfileSQTTEvent"]
|
||||
kern_events = {e.tag:e for e in data if type(e).__name__ == "ProfileProgramEvent"}
|
||||
if sqtt_events and kern_events:
|
||||
cls.examples[pkl_path.stem] = (sqtt_events, kern_events, cls.target)
|
||||
|
||||
@needs_rocprof
|
||||
def test_rocprof_inst_traces_match(self):
|
||||
for name, (events, kern_events, target) in self.examples.items():
|
||||
if "sync" in name and self.target.startswith("gfx12"):
|
||||
self.skipTest("our timestamps are off by a few cycles because rocprof patches timestamps for rdna4 barriers")
|
||||
for event in events:
|
||||
if not event.itrace: continue
|
||||
if event.kern not in kern_events: continue
|
||||
with self.subTest(example=name, kern=event.kern):
|
||||
passed_insts, n_waves, n_units = rocprof_inst_traces_match(event, kern_events[event.kern], target)
|
||||
if n_waves: print(f"{name}: passed for {passed_insts} instructions across {n_waves} waves scheduled on {n_units} wave units")
|
||||
|
||||
def test_sqtt_timeline(self):
|
||||
for name, (events, kern_events, target) in self.examples.items():
|
||||
for event in events:
|
||||
if (p:=kern_events.get(event.kern)) is None: continue
|
||||
with self.subTest(example=name, kern=event.kern):
|
||||
# skip if there's no SQTT frequency data
|
||||
if not (timeline:=list(sqtt_timeline(event.blob, p.lib, target))): continue
|
||||
if not (frequency:=[e.key for e in timeline if type(e).__name__ == "ProfilePointEvent" and e.name == "freq_hz"]): continue
|
||||
mean = sum(frequency) / len(frequency)
|
||||
variance = sum((v - mean) ** 2 for v in frequency) / len(frequency)
|
||||
self.assertGreater(mean, 0)
|
||||
if DEBUG >= 2: print(f"{name:20s} SE:{event.se} {mean/1e9:.2f} GHz mean, {variance/1e18:.2f} GHz^2 variance")
|
||||
events = [e for e in timeline if type(e).__name__ == "ProfileRangeEvent"]
|
||||
insts, execs = 0, 0
|
||||
for e in events:
|
||||
if "EXEC" in e.device:
|
||||
if "ALT" not in e.name.display_name: execs += 1
|
||||
elif "WAVE" in e.device:
|
||||
# sopk/immediates don't get ALU/MEM EXEC
|
||||
if e.name.display_name not in {"IMMEDIATE", "IMMEDIATE_MASK", "JUMP", "JUMP_NO", "MESSAGE", "BARRIER", "BARRIER_SIGNAL",
|
||||
"WAVEEND", "WAVEEND_RDNA4", "WAVERDY"} and not e.name.display_name.startswith("OTHER_"): insts += 1
|
||||
else: raise Exception(f"timeline row must be INST or EXEC, got {e.device}")
|
||||
self.assertEqual(execs, insts)
|
||||
|
||||
def test_wave_sync(self):
|
||||
for name, (events, kern_events, target) in self.examples.items():
|
||||
for event in events:
|
||||
wave_barriers = {}
|
||||
for e in sqtt_timeline(event.blob, kern_events[event.kern].lib, target):
|
||||
if type(e).__name__ == "ProfileRangeEvent" and e.name.display_name == "BARRIER": wave_barriers.setdefault(e.device, []).append(e)
|
||||
if not wave_barriers: continue
|
||||
for row, events in wave_barriers.items():
|
||||
for e in events:
|
||||
assert e.en-e.st > 1, f"all barriers must have a duration greater than 1, got {e}"
|
||||
|
||||
def test_sqtt_cli(self):
|
||||
for pkl_path in sorted((EXAMPLES_DIR/self.target).glob("*.pkl")):
|
||||
out = run_cli("--profile-path", str(pkl_path), "--ls")
|
||||
sqtt_traces = [l["value"].strip() for l in out if "SQTT" in l["value"]]
|
||||
for name in sqtt_traces:
|
||||
lines = run_cli("--profile-path", str(pkl_path), "-s", ansistrip(name))
|
||||
self.assertIn("Clk", lines[0]["value"])
|
||||
waves = [r["clk"] for r in lines[2:] if "WAVE" in r["unit"]]
|
||||
self.assertEqual(waves, sorted(waves), f"wave timestamps not monotonic in {name}")
|
||||
with Context(DEBUG=2):
|
||||
kernels = run_cli("--profile-path", str(pkl_path), "-s", "AMD")
|
||||
self.assertEqual(len(kernels), len(self.examples[pkl_path.stem][1]))
|
||||
|
||||
class TestSQTTMapRDNA3(TestSQTTMapBase): target = "gfx1100"
|
||||
|
||||
class TestSQTTMapRDNA4(TestSQTTMapBase):
|
||||
target = "gfx1200"
|
||||
|
||||
@unittest.expectedFailure
|
||||
def test_pipes(self):
|
||||
events, kernels, target = self.examples["profile_handwritten_run_0"]
|
||||
lib = list(kernels.values())[0].lib
|
||||
dispatch_st:dict[str, int] = {}
|
||||
row_ends:dict[str, int] = {}
|
||||
row_counts:dict[str, int] = {}
|
||||
for e in sqtt_timeline(events[1].blob, lib, target):
|
||||
if type(e).__name__ != "ProfileRangeEvent": continue
|
||||
info = json.loads(e.name.ret) if e.name.ret else {}
|
||||
if e.device.startswith("WAVE"):
|
||||
idx = row_counts.get(e.device, 0)
|
||||
dispatch_st[f"{e.device}-{idx}"] = int(e.st)
|
||||
row_counts[e.device] = idx + 1
|
||||
elif info.startswith("LINK:"):
|
||||
delay = int(e.st) - dispatch_st[info[len("LINK:"):]]
|
||||
self.assertGreaterEqual(delay, 1, f"EXEC {e.device} starts before DISPATCH: delay={delay}")
|
||||
if (prev_en:=row_ends.get(e.device)) is not None:
|
||||
self.assertGreaterEqual(e.st, prev_en, f"EXEC overlap in {e.device}: {e.st} < prev end {prev_en}")
|
||||
row_ends[e.device] = int(e.en)
|
||||
|
||||
class TestSQTTMapCDNA(TestSQTTMapBase):
|
||||
target = "gfx950"
|
||||
def test_rocprof_inst_traces_match(self): self.skipTest("requires timestamp patching to match rocprof, currently it's off by a few cycles")
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user