J3ff/IQ.Pilot
1
0
Fork 0
forked from IQ.Lvbs/IQ.Pilot
Code Pull Requests Activity

IQ.Pilot Prebuilt Release @ ab07000

Browse Source
This commit is contained in:
IQ.Lvbs history cleanup
2026-08-22 23:42:42 -05:00
commit 9f9c9a70cc
3729 changed files with 778697 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
tinygrad_repo/docs/CNAME Normal file
View File

@@ -0,0 +1 @@
docs.tinygrad.org

54
tinygrad_repo/docs/abstractions3.py Normal file
View File

@@ -0,0 +1,54 @@
# abstractions2 goes from back to front, here we will go from front to back
# *****
# 0. Load mnist on the device
from tinygrad.nn.datasets import mnist
X_train, Y_train, _, _ = mnist()
X_train = X_train.float()
X_train -= X_train.mean()
# *****
# 1. Define an MNIST model.
from tinygrad import Tensor
l1 = Tensor.kaiming_uniform(128, 784)
l2 = Tensor.kaiming_uniform(10, 128)
def model(x): return x.flatten(1).dot(l1.T).relu().dot(l2.T)
l1n, l2n = l1.numpy(), l2.numpy()
# *****
# 2. Choose a batch for training and do the backward pass.
from tinygrad.nn.optim import SGD
optim = SGD([l1, l2])
Tensor.training = True
X, Y = X_train[(samples:=Tensor.randint(128, high=X_train.shape[0]))], Y_train[samples]
optim.zero_grad()
model(X).sparse_categorical_crossentropy(Y).backward()
optim.schedule_step() # this will step the optimizer without running realize
# *****
# 3. Create a schedule (linear uop).
# The weight Tensors have been assigned to, but not yet realized. Everything is still lazy at this point
# l1.uop and l2.uop define a computation graph
from tinygrad.engine.realize import run_linear
linear = Tensor.schedule_linear(l1, l2)
print(f"The schedule contains {len(linear.src)} items.")
for call in linear.src: print(str(call)[:80])
# *****
# 4. Lower and run the schedule (linear uop).
run_linear(linear)
# *****
# 5. Print the weight change
print("first weight change\n", l1.numpy()-l1n)
print("second weight change\n", l2.numpy()-l2n)

253
tinygrad_repo/docs/abstractions4.py Normal file
View File

@@ -0,0 +1,253 @@
# tinygrad allows you to write kernels at many different abstractions levels.
# This is for RDNA3, but if you don't have one you can run with the emulator
# PYTHONPATH="." DEV=MOCKPCI+AMD
from tinygrad import Tensor, Context, GlobalCounters, UOp, Device
from tinygrad.helpers import DEV, DEBUG, getenv
from tinygrad.uop.ops import AxisType, KernelInfo, Ops
from tinygrad.dtype import AddrSpace, dtypes
from tinygrad.runtime.autogen.amd.rdna3.ins import *
def eval_harness(name, tensor, fxn, check=None):
print(f"***** {name}")
GlobalCounters.reset()
with Context(DEBUG=max(DEBUG.value, 2)): out = fxn(tensor).item()
assert check is None or abs(out - check) < abs(check) * 1e-3, f"out was wrong {out}, expected {check}, off by {out/check}x"
print(f"computed in {GlobalCounters.time_sum_s*1000:.2f} ms, {(a.nbytes()/1e9)/GlobalCounters.time_sum_s:.2f} GB/s")
return out
SZ = 256*1024 if DEV.interface.startswith("MOCK") else 1024*1024*1024
def example_2_hip(a:Tensor, correct):
GLOBALS = 1024
THREADS = 256
def hip_reduce_sum(out:UOp, buf:UOp) -> UOp:
assert SZ % (GLOBALS * THREADS) == 0
CHUNK = SZ // (GLOBALS * THREADS)
# NOTE: tinygrad doesn't populate HIP hidden kernargs, so blockDim.x/gridDim.x read as 0.
# We hardcode block/grid sizes as constexpr to avoid any dependency on those builtins.
code = f"""
#include <hip/hip_runtime.h>
constexpr unsigned int BLOCK = {THREADS};
constexpr unsigned int CHUNK = {CHUNK};
extern "C" __global__ void hip_reduce_sum_kernel(float* __restrict__ block_sums, const float* __restrict__ x) {{
__shared__ float sdata[BLOCK];
unsigned int tid = threadIdx.x;
unsigned int gid = blockIdx.x * BLOCK + tid;
// Each thread sums CHUNK consecutive elements from its own region
float sum = 0.0f;
const float* base = x + gid * CHUNK;
#pragma unroll 16
for (unsigned int k = 0; k < CHUNK; k++) {{
sum += base[k];
}}
sdata[tid] = sum;
__syncthreads();
// Block reduction in shared memory
for (unsigned int s = BLOCK / 2; s > 0; s >>= 1) {{
if (tid < s) {{
sdata[tid] += sdata[tid + s];
}}
__syncthreads();
}}
// One partial sum per block
if (tid == 0) {{
block_sums[blockIdx.x] = sdata[0];
}}
}}"""
# TODO: remove the need for the compiler here, you should just be able to remove Ops.BINARY
from tinygrad.runtime.support.compiler_amd import HIPCCCompiler
lib = HIPCCCompiler(Device[Device.DEFAULT].renderer.target.arch, []).compile_cached(code)
# the sink specifies the GLOBAL and LOCAL sizes, along with the input buffers and name
sink = UOp.sink(UOp.special(GLOBALS, 'gidx0'), UOp.special(THREADS, 'lidx0'), out, buf,
arg=KernelInfo(name="hip_reduce_sum_kernel"))
return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.DEVICE, arg=Device.DEFAULT),
UOp(Ops.LINEAR, src=(*sink.src, sink)), UOp(Ops.SOURCE, arg=code), UOp(Ops.BINARY, arg=lib)))
eval_harness("HIP kernel", a, lambda x: Tensor.empty(GLOBALS).custom_kernel(x, fxn=hip_reduce_sum)[0].sum(), check=correct)
def example_3_custom_uop(a:Tensor, correct):
# This GPU has 32 CUs, keep them all busy
CU_COUNT = 32
def custom_sum(out:UOp, buf:UOp) -> UOp:
LCLS = 256
buf = buf.reshape(CU_COUNT, -1, LCLS)
glbl = UOp.range(CU_COUNT, 0, AxisType.GLOBAL)
lane = UOp.range(LCLS, 1, AxisType.LOCAL)
# accumulate the globals into a per lane accumulator
reduce_loop = UOp.range(buf.shape[1], 2, AxisType.REDUCE)
acc = UOp.placeholder((1,), dtypes.float, slot=6, addrspace=AddrSpace.REG)
acc = acc.after(acc.store(0))
acc = acc.after(acc[0].store(acc.after(reduce_loop)[0] + buf[glbl, reduce_loop, lane]).end(reduce_loop))
# store all the per lane accumulators to LOCAL
local_accs = UOp.placeholder((LCLS,), dtypes.float, slot=0, addrspace=AddrSpace.LOCAL)
local_accs = local_accs.after(local_accs[lane].store(acc[0]).barrier())
# accumulate LOCALs into a single per CU accumulator
late_reduce_loop = UOp.range(LCLS, 3, AxisType.REDUCE)
acc2 = UOp.placeholder((1,), dtypes.float, slot=7, addrspace=AddrSpace.REG)
acc2 = acc2.after(acc2.store(0))
acc2 = acc2.after(acc2[0].store(acc2.after(late_reduce_loop)[0] + local_accs[late_reduce_loop]).end(late_reduce_loop))[0]
# store (NOTE: since the address doesn't depend on the warp, this will be automatically gated)
return out[glbl].store(acc2).end(lane, glbl).sink(arg=KernelInfo(opts_to_apply=()))
eval_harness("custom UOp kernel", a, lambda x: Tensor.empty(CU_COUNT).custom_kernel(x, fxn=custom_sum)[0].sum(), check=correct)
def example_5_custom_assembly(a:Tensor, correct):
# Kernel class copied from amd_asm_matmul
class Kernel:
def __init__(self): self.instructions, self.labels, self.pos = [], {}, 0
def label(self, name): self.labels[name] = self.pos
def emit(self, inst, target=None):
self.instructions.append(inst)
inst._target, inst._pos = target, self.pos
self.pos += inst.size()
return inst
def waitcnt(self, lgkm=None, vm=None):
# Wait for memory operations. lgkm=N waits until N lgkm ops remain, vm=N waits until N vmem ops remain.
vmcnt, lgkmcnt, expcnt = vm if vm is not None else 63, lgkm if lgkm is not None else 63, 7
waitcnt = (expcnt & 0x7) | ((lgkmcnt & 0x3f) << 4) | ((vmcnt & 0x3f) << 10)
self.emit(s_waitcnt(simm16=waitcnt))
def finalize(self, sink:UOp) -> UOp:
for inst in self.instructions:
if inst._target is None: continue
offset_dwords = (self.labels[inst._target] - inst._pos - inst.size()) // 4
if not -32768 <= offset_dwords <= 32767: raise ValueError(f"branch to '{inst._target}' offset {offset_dwords} exceeds simm16 range")
inst.simm16 = offset_dwords
return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.DEVICE, arg=Device.DEFAULT),
UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in self.instructions]))))
CU_COUNT = 32
LANES = 64
def asm_sum(out:UOp, buf:UOp) -> UOp:
V_LANE_ID = 0 # lane_id set on startup
S_WORKGROUP_X = 2 # workgroup_id_x
S_LOOP_CTR = 3
k = Kernel()
# mul lane id by 16 for offsets (4 for float, 4 for b128)
k.emit(v_mul_lo_u32(v[0], v[V_LANE_ID], 16))
k.emit(v_add_nc_u32_e32(v[1], 4096, v[0]))
k.emit(v_add_nc_u32_e32(v[2], 4096, v[1]))
k.emit(v_add_nc_u32_e32(v[3], 4096, v[2]))
# load both addresses
k.emit(s_load_b128(sdata=s[4:7], sbase=s[0:1], offset=0x0, soffset=NULL))
k.waitcnt(lgkm=0)
# offset buffer pointer by workgroup_id_x * chunk_size_bytes
k.emit(s_mul_i32(s[S_LOOP_CTR], s[S_WORKGROUP_X], buf.numel()*4//CU_COUNT))
k.emit(s_add_u32(s[6], s[6], s[S_LOOP_CTR]))
k.emit(s_addc_u32(s[7], s[7], 0))
# zero the accumulators
k.emit(VOPD(VOPDOp.V_DUAL_MOV_B32, VOPDOp.V_DUAL_MOV_B32, vdstx=v[4], vdsty=v[5], srcx0=0, srcy0=0))
k.emit(VOPD(VOPDOp.V_DUAL_MOV_B32, VOPDOp.V_DUAL_MOV_B32, vdstx=v[6], vdsty=v[7], srcx0=0, srcy0=0))
def emit_loads(base_vreg, reg_len):
assert reg_len%4 == 0
k.emit(s_clause(simm16=(reg_len//4)-1))
for i in range(reg_len//4):
offset = i*LANES*16
assert offset < 16384
k.emit(global_load_b128(vdst=v[base_vreg+i*4:base_vreg+i*4+3], addr=v[offset//4096], saddr=s[6:7], offset=offset%4096))
k.emit(s_add_u32(s[6], s[6], reg_len * LANES * 4))
k.emit(s_addc_u32(s[7], s[7], 0))
def tree_reduce_to_4567(base_vreg, reg_len):
assert reg_len%4 == 0
reg_len //= 4
while reg_len > 1:
half = reg_len // 2
for j in range(half):
a, b = base_vreg + j*4, base_vreg + (j+half)*4
# v[a+0](bank0) += v[b+2](bank2), v[a+1](bank1) += v[b+3](bank3) — src0 and src1 on different banks
k.emit(VOPD(VOPDOp.V_DUAL_ADD_F32, VOPDOp.V_DUAL_ADD_F32, vdstx=v[a], vdsty=v[a+1], srcx0=v[a], vsrcx1=v[b+2], srcy0=v[a+1], vsrcy1=v[b+3]))
# v[a+2](bank2) += v[b+0](bank0), v[a+3](bank3) += v[b+1](bank1) — src0 and src1 on different banks
k.emit(VOPD(VOPDOp.V_DUAL_ADD_F32, VOPDOp.V_DUAL_ADD_F32, vdstx=v[a+2], vdsty=v[a+3], srcx0=v[a+2], vsrcx1=v[b], srcy0=v[a+3], vsrcy1=v[b+1]))
reg_len = half
k.emit(VOPD(VOPDOp.V_DUAL_ADD_F32, VOPDOp.V_DUAL_ADD_F32, vdstx=v[4], vdsty=v[5], srcx0=v[4], vsrcx1=v[base_vreg], srcy0=v[5], vsrcy1=v[base_vreg+1]))
k.emit(VOPD(VOPDOp.V_DUAL_ADD_F32, VOPDOp.V_DUAL_ADD_F32, vdstx=v[6], vdsty=v[7], srcx0=v[6], vsrcx1=v[base_vreg+2], srcy0=v[7], vsrcy1=v[base_vreg+3]))
BASE_REG = 8
LOAD_UNROLL = 64
INNER_UNROLL = 2
assert buf.numel() % (CU_COUNT*LANES*LOAD_UNROLL*INNER_UNROLL) == 0
total_batches = buf.numel()//(CU_COUNT*LANES*LOAD_UNROLL*INNER_UNROLL)
k.emit(s_mov_b32(s[S_LOOP_CTR], total_batches-1))
k.label('LOOP')
for _ in range(INNER_UNROLL):
emit_loads(BASE_REG, reg_len=LOAD_UNROLL)
k.waitcnt(vm=0)
tree_reduce_to_4567(BASE_REG, reg_len=LOAD_UNROLL)
k.emit(s_sub_u32(s[S_LOOP_CTR], s[S_LOOP_CTR], 1))
k.emit(s_cbranch_scc0(), target='LOOP')
# add into v[4]
k.emit(v_add_f32_e32(v[4], v[4], v[5]))
k.emit(v_add_f32_e32(v[6], v[6], v[7]))
k.emit(v_add_f32_e32(v[4], v[4], v[6]))
# warp shuffle into v[4] on lane 0 using DPP row_shl within each 16-lane row
for shift in [1, 2, 4, 8]:
k.emit(v_add_f32_e32(v[4], DPP, v[4], vsrc0=v[4], dpp=0x100 | shift, row_mask=0xf, bank_mask=0xf, bc=1))
# combine rows: get lane 16's value to lane 0 via permlanex16
k.emit(v_permlanex16_b32(v[5], v[4], 0, 0))
k.emit(v_add_f32_e32(v[4], v[4], v[5]))
# atomic store (only on lane 0)
k.emit(s_mov_b32(EXEC_LO, 1))
k.emit(v_mov_b32_e32(v[0], 0))
k.emit(global_atomic_add_f32(addr=v[0], saddr=s[4:5], data=v[4]))
k.emit(s_sendmsg(simm16=3)) # DEALLOC_VGPRS
k.emit(s_endpgm())
return k.finalize(UOp.sink(UOp.special(CU_COUNT, 'gidx0'), UOp.special(LANES, 'lidx0'), out, buf, arg=KernelInfo(name="asm_reduce")))
out = Tensor.zeros(1,).contiguous().realize()
eval_harness("RDNA3 assembly kernel", a, lambda x: out.custom_kernel(x, fxn=asm_sum)[0], check=correct)
if __name__ == "__main__":
examples = [int(x) for x in getenv("EXAMPLES", "1,2,3,4,5").split(",")]
correct = None
# First define a Tensor and realize it. We will focus on a 1GB sum kernel on RDNA3
a = (Tensor.randn(SZ) if getenv("RAND") else Tensor.ones(SZ)).contiguous().realize()
if 1 in examples:
# *****
# This is the high level tinygrad way.
# Note that this is split into multiple kernels for speed.
correct = eval_harness("basic kernel", a, lambda x: x.sum())
if 2 in examples:
# *****
# You can import kernels from CUDA/HIP/Metal.
# ChatGPT is great at writing these Kernel
example_2_hip(a, correct)
if 3 in examples:
# *****
# Now we get to the lower abstraction layers of tinygrad.
# You can write a kernel in UOps, and it's 2.5x faster than normal.
example_3_custom_uop(a, correct)
if 4 in examples:
# *****
# You can also BEAM search stock tinygrad for a faster kernel.
# This does even better than all the kernels to date in this simple case.
with Context(BEAM=2):
eval_harness("BEAMed kernel", a, lambda x: x.sum(), check=correct)
if 5 in examples:
# *****
# If you really want to go crazy with speed, you can code in assembly.
# There's not too much to gain here over BEAM, but it's a few percent faster.
example_5_custom_assembly(a, correct)

39
tinygrad_repo/docs/developer/am.md Normal file
View File

@@ -0,0 +1,39 @@
# AM Driver
AM driver is a userspace driver targeting AMD's RDNA3/RDNA4. You only need tinygrad to send compute tasks to your GPU!
## How to run?
Make sure that amdgpu module is unloaded and just run tinygrad with `DEV=AMD`!
Optional requirements:
* System without IOMMU for P2P / SDMA support
* vfio-pci module for IRQ handling
## Environment Variables
| Variable | Possible Value(s) | Description |
|----------|------------------|-------------|
| AM_RESET | [1] | Performs a full GPU reset (reloading all firmware and IP blocks) |
| AM_DEBUG | [0-4] | Sets the level of additional debugging information |
## AM Driver Details
### Compute & SDMA Queues
AM binds compute queues directly to MEC (bypassing MES). Tinygrad uses only one compute queue, which is bound at `pipe=0 queue=0`. Similarly, the single SDMA queue is bound at `engine=0 queue=0`.
### Boot
The GPU being passed can be in one of several states:
1. Not initialized
2. Initialized by amdgpu
3. Initialized by AM
The first and second states require a full GPU setup since their states are unknown. The second state also requires a mode1 reset to reinitialize all components.
The third state can be set up partially to optimize boot time. In this case, only the GFX and SDMA IPs need to be initialized. To enable this, AM uses a separate boot memory that is guaranteed not to be overwritten. This physical memory is utilized for all blocks that are initialized only during the initial AM boot. To determine if the GPU is in the third state, AM uses `regSCRATCH_REG7` as a flag.
### VM Management
Each AM device sets up only a single `VMID=0` and one page directory. The page directory used is 3-level and thus supports up to 512GB of virtual addresses. All AM devices are located in one virtual address space.

46
tinygrad_repo/docs/developer/developer.md Normal file
View File

@@ -0,0 +1,46 @@
The tinygrad framework has four pieces
* a PyTorch like <b>frontend</b>.
* a <b>scheduler</b> which breaks the compute into kernels.
* a <b>lowering</b> engine which converts ASTs into code that can run on the accelerator.
* an <b>execution</b> engine which can run that code.
There is a good [bunch of tutorials](https://mesozoic-egg.github.io/tinygrad-notes/) by Di Zhu that go over tinygrad internals.
There's also a [doc describing speed](../developer/speed.md)
## Frontend
Everything in [Tensor](../tensor/index.md) is syntactic sugar around constructing a graph of [UOps](../developer/uop.md).
The `UOp` graph specifies the compute in terms of low level tinygrad ops. Not all UOps will actually become realized. There's two types of UOps, base and view. base contains compute into a contiguous buffer, and view is a view. Inputs to a base can be either base or view, inputs to a view can only be a single base.
## Scheduling
The [scheduler](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/schedule/__init__.py) converts the graph of UOps into a `LINEAR` UOp whose `src` is a list of `CALL` UOps. One `CALL` is one kernel on the GPU, and the scheduler is responsible for breaking the large compute graph into subgraphs that can fit in a kernel. The `CALL`'s `src[0]` (a `SINK` ast) specifies what compute to run, and the remaining `src` are the buffers to run it on.
## Lowering
The code in [realize](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/engine/realize.py) lowers each `CALL` by compiling its ast into a `PROGRAM` and running it.
::: tinygrad.engine.realize.run_linear
There's a ton of complexity hidden behind this, see the `codegen/` directory.
First we lower the AST to UOps, which is a linear list of the compute to be run. This is where the BEAM search happens.
Then we render the UOps into code with a `Renderer`, then we compile the code to binary with a `Compiler`.
## Execution
`run_linear` walks the `LINEAR` UOp, dispatching each `CALL` to a runner (kernel, copy, view, encdec, or graph).
## Runtime
Runtimes are responsible for device-specific interactions. They handle tasks such as initializing devices, allocating memory, loading/launching programs, and more. You can find more information about the runtimes API on the [runtime overview page](runtime.md).
All runtime implementations can be found in the [runtime directory](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime).
### HCQ Compatible Runtimes
HCQ API is a lower-level API for defining runtimes. Interaction with HCQ-compatible devices occurs at a lower level, with commands issued directly to hardware queues. Some examples of such backends are [NV](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_nv.py) and [AMD](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_amd.py), which are userspace drivers for NVIDIA and AMD devices respectively. You can find more information about the API on [HCQ overview page](hcq.md)

128
tinygrad_repo/docs/developer/hcq.md Normal file
View File

@@ -0,0 +1,128 @@
# HCQ Compatible Runtime
## Overview
The main aspect of HCQ-compatible runtimes is how they interact with devices. In HCQ, all interactions with devices occur in a hardware-friendly manner using [command queues](#command-queues). This approach allows commands to be issued directly to devices, bypassing runtime overhead such as HIP or CUDA. Additionally, by using the HCQ API, these runtimes can benefit from various optimizations and features, including [HCQGraph](#hcqgraph) and built-in profiling capabilities.
### Command Queues
To interact with devices you create a `HWQueue`. Some methods are required, like timestamp and synchronization methods like [signal](#tinygrad.runtime.support.hcq.HWQueue.signal) and [wait](#tinygrad.runtime.support.hcq.HWQueue.wait), while others are dependent on it being a compute or copy queue.
For example, the following Python code enqueues a wait, execute, and signal command on the HCQ-compatible device:
```python
HWQueue().wait(signal_to_wait, value_to_wait) \
.exec(program, args_state, global_dims, local_dims) \
.signal(signal_to_fire, value_to_fire) \
.submit(your_device)
```
Each runtime should implement the required functions that are defined in the `HWQueue` classes.
::: tinygrad.runtime.support.hcq.HWQueue
options:
members: [
"signal",
"wait",
"timestamp",
"bind",
"submit",
"memory_barrier",
"exec",
"copy",
]
show_source: false
### HCQ Compatible Device
The `HCQCompiled` class defines the API for HCQ-compatible devices. This class serves as an abstract base class that device-specific implementations should inherit from and implement.
::: tinygrad.runtime.support.hcq.HCQCompiled
options:
show_source: false
#### Signals
Signals are device-dependent structures used for synchronization and timing in HCQ-compatible devices. They should be designed to record both a `value` and a `timestamp` within the same signal. HCQ-compatible backend implementations should use `HCQSignal` as a base class.
::: tinygrad.runtime.support.hcq.HCQSignal
options:
members: [value, timestamp, wait]
show_source: false
The following Python code demonstrates the usage of signals:
```python
signal = your_device.new_signal(value=0)
HWQueue().timestamp(signal) \
.signal(signal, value_to_fire) \
.submit(your_device)
signal.wait(value_to_fire)
signaled_value = signal.value # should be the same as `value_to_fire`
timestamp = signal.timestamp
```
##### Synchronization signals
Each HCQ-compatible device must allocate two signals for global synchronization purposes. These signals are passed to the `HCQCompiled` base class during initialization: an active timeline signal `self.timeline_signal` and a shadow timeline signal `self._shadow_timeline_signal` which helps to handle signal value overflow issues. You can find more about synchronization in the [synchronization section](#synchronization)
### HCQ Compatible Allocator
The `HCQAllocator` base class simplifies allocator logic by leveraging [command queues](#command-queues) abstractions. This class efficiently handles copy and transfer operations, leaving only the alloc and free functions to be implemented by individual backends.
::: tinygrad.runtime.support.hcq.HCQAllocator
options:
members: [
"_alloc",
"_free",
]
show_source: false
#### HCQ Allocator Result Protocol
Backends must adhere to the `HCQBuffer` protocol when returning allocation results.
::: tinygrad.runtime.support.hcq.HCQBuffer
options:
members: true
show_source: false
### HCQ Compatible Program
`HCQProgram` is a base class for defining programs compatible with HCQ-enabled devices. It provides a flexible framework for handling different argument layouts (see `HCQArgsState`).
::: tinygrad.runtime.support.hcq.HCQProgram
options:
members: true
show_source: false
#### Arguments State
`HCQArgsState` is a base class for managing the argument state for HCQ programs. Backend implementations should create a subclass of `HCQArgsState` to manage arguments for the given program.
::: tinygrad.runtime.support.hcq.HCQArgsState
options:
members: true
show_source: false
**Lifetime**: The `HCQArgsState` is passed to `HWQueue.exec` and is guaranteed not to be freed until `HWQueue.submit` for the same queue is called.
### Synchronization
HCQ-compatible devices use a global timeline signal for synchronizing all operations. This mechanism ensures proper ordering and completion of tasks across the device. By convention, `self.timeline_value` points to the next value to signal. So, to wait for all previous operations on the device to complete, wait for `self.timeline_value - 1` value. The following Python code demonstrates the typical usage of signals to synchronize execution to other operations on the device:
```python
HWQueue().wait(your_device.timeline_signal, your_device.timeline_value - 1) \
.exec(...)
.signal(your_device.timeline_signal, your_device.next_timeline()) \
.submit(your_device)
# Optionally wait for execution
your_device.timeline_signal.wait(your_device.timeline_value - 1)
```
## HCQGraph
[HCQGraph](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/graph/hcq.py) is a core feature that implements `GraphRunner` for HCQ-compatible devices. `HCQGraph` builds static `HWQueue` for all operations per device. To optimize enqueue time, only the necessary parts of the queues are updated for each run using the symbolic variables, avoiding a complete rebuild.
Optionally, queues can implement a `bind` API, which allows further optimization by eliminating the need to copy the queues into the device ring.

60
tinygrad_repo/docs/developer/layout.md Normal file
View File

@@ -0,0 +1,60 @@
# tinygrad directory layout
This explains the flow of a big graph down to programs.
Directories are listed in order of how they are processed.
---
## tinygrad/schedule
Group UOps into kernels.
::: tinygrad.schedule.rangeify.get_kernel_graph
options:
members: false
show_labels: false
show_source: false
---
## tinygrad/codegen/opt
Transforms the ast into an optimized ast. This is where BEAM search and heuristics live.
---
## tinygrad/codegen
Transform the optimized ast into a linearized and rendered program.
::: tinygrad.codegen.to_program
options:
members: false
show_labels: false
show_source: false
---
## tinygrad/renderer
Transform the linearized list of UOps into a program, represented as a string.
::: tinygrad.renderer.Renderer
options:
members:
- render
show_labels: false
show_source: false
---
## tinygrad/engine
Abstracted high level interface to the runtimes.
::: tinygrad.engine.realize.to_program
options:
members: false
show_labels: false
show_source: false

51
tinygrad_repo/docs/developer/runtime.md Normal file
View File

@@ -0,0 +1,51 @@
# Runtime Overview
## Overview
A typical runtime consists of the following parts:
- [Compiled](#compiled)
- [Allocator](#allocator)
- [Program](#program)
- [Compiler](#compiler)
### Compiled
The `Compiled` class is responsible for initializing and managing a device.
::: tinygrad.device.Compiled
options:
members: [
"synchronize"
]
show_source: false
### Allocator
The `Allocator` class is responsible for managing memory on the device. There is also a version called the `LRUAllocator`, which caches allocated buffers to optimize performance.
::: tinygrad.device.Allocator
options:
members: true
show_source: false
::: tinygrad.device.LRUAllocator
options:
members: true
show_source: false
### Program
The `Program` class is created for each loaded program. It is responsible for executing the program on the device. As an example, here is a `CPUProgram` implementation which loads program and runs it.
::: tinygrad.runtime.ops_cpu.CPUProgram
options:
members: true
### Compiler
The `Compiler` class compiles the output from the `Renderer` and produces it in a device-specific format.
::: tinygrad.device.Compiler
options:
members: true

71
tinygrad_repo/docs/developer/speed.md Normal file
View File

@@ -0,0 +1,71 @@
# speed in tinygrad
## Overview
Speed refers to many different things. To break it down to four, there's:
- Compile Speed (Python)
- Execution Speed (driver)
- Model Speed (scheduler)
- Kernel Speed (codegen)
## Compile Speed (Python)
This is how long the first run of your model takes. It's limited largely by the runtime of the Python doing UOp rewrites. Currently it's a bit slow, but on par with torch.compile. It gets even slower if you are using BEAM, since that's compiling many variants of each kernel.
This will be improved by writing faster graph_rewrite, doing less graph_rewrite, and better parallelization.
## Execution Speed (driver)
After your model is compiled, you are often using the `TinyJIT`. tinygrad has the best execution speed of any framework because it usually bypasses the GPU driver and prebuilds the command queue. It's tons faster than normal CUDA, and often even faster than CUDA Graph.
There's very little to improve here, as this is almost never the bottleneck.
## Model Speed (scheduler)
The scheduler determines how operations are grouped into kernels and which Tensors are written to memory. This is currently a big bottleneck of training speed.
The decisions are often not obvious. For example, when is it worth recomputing an arithmetic operation instead of storing and loading from memory? Example:
```python
from tinygrad import Tensor
a = Tensor.rand(100)
b = Tensor.rand(100)
c = Tensor.rand(100)
d = Tensor.rand(100)
out1 = a+b+c
out2 = a+b+d
Tensor.realize(out1, out2)
```
The real answer is obvious, compute both `out1` and `out2` in the same kernel. But you can't always do that. If you can't, should `a+b` first be saved to a subbuffer? Or should both the `out1` and `out2` kernels recompute `a+b`?
In this case: with recompute (6 reads + 2 writes), no recompute (6 reads + 3 writes), so we should probably recompute. However, once you add movement ops and casts this is even harder to figure out. tinygrad doesn't yet have a systematic way to do it.
## Kernel Speed (codegen)
Given that you have decided how the model ops will be grouped and what will be written to memory, kernel speed determines how fast that operation is done. This is what BEAM changes, it searches over a set of equivalent kernels which all perform the same operation and finds the one which performs the task the fastest.
In `kernel.py` we have a set of `OptOps`, these control the parameters of the speed optimizations applied to the kernel.
### Memory
The main bottleneck in most kernels is accessing memory. In a freshman algorithms class, you'll learn about cache aware matrix multiplication, and this is all forms of that. While the same math is run, the order in which you run it can have large impacts on the speed depending on if the data you are loading. OptOps will change this order.
Memory, even cache, is often much slower than accessing the register file. The amount of times data is used in math is called the "arithmetic intensity". For operations like BS=1 GEMV, the arithmetic intensity is 1, but for GEMMs and convs it can be much higher. OptOps like UPCAST and UNROLL can increase this, but be careful of making them too large, as if there's too much register pressure on the GPU the warp scheduler may not be able to fit many warps, or even worse, it could be spilling to local memory.
4090s have 1 TB/s of ram bandwidth and ~160 TFLOPS of compute, so you need to use each loaded value ~100 times. The L1 cache has around 40 TB/s of bandwidth, so in order to get full compute utilization you need to use each value ~4 times.
A lot of work can still be done here. For example, we never copy the inputs to on chip SRAM, but this is often quite helpful for kernel speed. Also, we aren't doing a good job with L2 cache awareness (the locals handle L1 quite well)
### Tensor Cores
Many accelerators have Tensor Cores / MAC arrays / systolic arrays. The main value of these is that, since they are 2-D, they create an n^2 ratio between the compute and the input data.
GPUs use Tensor Cores instead of MAC arrays to fit better in the GPU warp paradigm. This is because the output of Tensor Cores is O(n) wrt the input, while the output of MAC arrays like the AMX is O(n^2)
We have a simple framework in tinygrad for adding these ALU blocks and achieving good performance from them.
### Indexing
Indexing determines the address of the memory we need to load. GPUs often have less integer math resources than floating point math, so this can sometimes be the bottleneck. We have a symbolic math engine in our rewrite rules to simplify indexing before it's emitted to the kernel. Newer NVIDIA GPUs have a "Tensor Memory Accelerator" to assist with fast indexing, however, this is not supported in tinygrad yet.

11
tinygrad_repo/docs/developer/uop.md Normal file
View File

@@ -0,0 +1,11 @@
::: tinygrad.uop.ops.UOp
options:
members: false
members_order: source
show_labels: false
::: tinygrad.uop.ops.Ops
options:
members: true
members_order: source
show_labels: false

9
tinygrad_repo/docs/dtypes.md Normal file
View File

@@ -0,0 +1,9 @@
::: tinygrad.dtype.DType
::: tinygrad.dtype.dtypes
options:
members: true
members_order: source
show_labels: false
::: tinygrad.dtype.ConstType

73
tinygrad_repo/docs/env_vars.md Normal file
View File

@@ -0,0 +1,73 @@
# List of environment variables that control tinygrad behavior.
This is a list of environment variable that control the runtime behavior of tinygrad and its examples.
Most of these are self-explanatory, and are usually used to set an option at runtime.
Example: `DEV=CL DEBUG=4 python3 -m pytest`
However you can also decorate a function to set a value only inside that function.
```python
# in tensor.py (probably only useful if you are a tinygrad developer)
@Context(DEBUG=4)
def numpy(self) -> ...
```
Or use contextmanager to temporarily set a value inside some scope:
```python
with Context(DEBUG=0):
a = Tensor.ones(10, 10)
a *= 2
```
## Global Variables
The columns of this list are are: Variable, Possible Value(s) and Description.
- A `#` means that the variable can take any integer value.
These control the behavior of core tinygrad even when used as a library.
Variable | Possible Value(s) | Description
---|---|---
DEBUG | [1-7] | enable debugging output (operations, timings, speed, generated code and more)
DEV | [AMD, NV, ...] | enable a specific backend, see [below](#dev-variable)
BEAM | [#] | number of beams in kernel beam search
DEFAULT_FLOAT | [HALF, ...]| specify the default float dtype (FLOAT32, HALF, BFLOAT16, FLOAT64, ...), default to FLOAT32
IMAGE | [1] | enable 2d specific optimizations
FLOAT16 | [1] | use float16 for images instead of float32
JIT | [0-2] | 0=disabled, 1=[jit enabled](quickstart.md#jit) (default), 2=jit enabled, but graphs are disabled
VIZ | [1] | 0=disabled, 1=[viz enabled](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/viz)
ALLOW_TF32 | [1] | enable TensorFloat-32 tensor cores on Ampere or newer GPUs.
WEBGPU_BACKEND | [WGPUBackendType_Metal, ...] | Force select a backend for WebGPU (Metal, DirectX, OpenGL, Vulkan...)
CUDA_PATH | str | Use `CUDA_PATH/include` for CUDA headers for CUDA and NV backends. If not set, TinyGrad will use `/usr/local/cuda/include`, `/usr/include` and `/opt/cuda/include`.
### DEV variable
The `DEV` variable deserves special note due to its more nuanced syntax.
`DEV` is used to specify the target device, target renderer and target architecture for said device, separated by colons.
Specifying the renderer and architecture is optional, omitting a preference will cause tinygrad to automatically determine a suitable setting.
The `DEV` variable may also be used to specify the interface through which to access the device (eg. `PCI`, `USB`). Interfaces may be specified preceding the target triple,
separated by a plus (eg. `DEV=USB+AMD:LLVM`). Similarly as above, the interface may be omitted. Example usage follows:
`DEV` contents | Interpretation
--- | ---
AMD | use the AMD device
AMD:LLVM | use the AMD device with the LLVM renderer
NV:CUDA:sm_70 | use the NV device with the CUDA renderer targetting sm_70
AMD::gfx950 | use the AMD device targetting gfx950
USB+AMD | use the AMD device over the USB interface
CPU:LLVM | use the CPU device with the LLVM renderer
CPU:LLVM:x86_64,znver2,avx2,-avx512f | use the CPU device with the LLVM renderer, with [additional arch flags](runtime.md#cpu-arch)
### Debug breakdown
Variable | Value | Description
---|---|---
DEBUG | >= 1 | Enables debugging and lists devices being used
DEBUG | >= 2 | Provides performance metrics for operations, including timing, memory usage, bandwidth for each kernel execution
DEBUG | >= 3 | Outputs the applied optimizations at a kernel level
DEBUG | >= 4 | Outputs the generated kernel code
DEBUG | >= 5 | Displays the intermediate representation of the computation UOps
DEBUG | >= 6 | Displays the intermediate representation of the computation UOps in a linearized manner, detailing the operation sequence
DEBUG | >= 7 | Outputs the assembly code generated for the target hardware

25
tinygrad_repo/docs/favicon.svg Normal file
View File

@@ -0,0 +1,25 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="-10 -10 150 70" shape-rendering="crispEdges">
<g id="logo">
<!-- t -->
<polygon points="10,40 10,20 0,20 0,10 10,10 10,0 20,0 20,10 30,10 30,20 20,20 20,30 30,30 30,40" />
<!-- i -->
<polygon points="40,40 40,20 50,20 50,40" />
<polygon points="40,10 40,0 50,0 50,10" />
<!-- n -->
<polygon points="60,40 60,10 80,10 80,40 90,40 90,20 70,20 70,40" />
<!-- y -->
<polygon points="100,50 100,40 130,40 130,10 120,10 120,20 110,20 110,10 100,10 100,30 120,30 120,50" />
</g>
<style>
@media (prefers-color-scheme: dark) {
#logo {
fill: #fff;
}
}
@media (prefers-color-scheme: light) {
#logo {
fill: #000;
}
}
</style>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 750 B

53
tinygrad_repo/docs/index.md Normal file
View File

@@ -0,0 +1,53 @@
# tinygrad documentation
Welcome to the docs for tinygrad. This page is for users of the tinygrad library. tinygrad is not 1.0 yet, but it will be soon. The API has been pretty stable for a while.
While you can `pip install tinygrad`, we encourage you to install from source:
```bash
git clone https://github.com/tinygrad/tinygrad.git
cd tinygrad
python3 -m pip install -e .
```
After you have installed tinygrad, try the [MNIST tutorial](mnist.md).
If you are new to tensor libraries, learn how to use them by solving puzzles from [tinygrad-tensor-puzzles](https://github.com/obadakhalili/tinygrad-tensor-puzzles).
We also have [developer docs](developer/developer.md), and Di Zhu has created a [bunch of tutorials](https://mesozoic-egg.github.io/tinygrad-notes/) to help understand how tinygrad works.
## tinygrad Usage
The main class you will interact with is [Tensor](tensor/index.md). It functions very similarly to PyTorch, but has a bit more of a functional style. tinygrad supports [many datatypes](dtypes.md). All operations in tinygrad are lazy, meaning they won't do anything until you realize.
* tinygrad has a built in [neural network library](nn.md) with some classes, optimizers, and load/save state management.
* tinygrad has a JIT to make things fast. Decorate your pure function with `TinyJit`
* tinygrad has amazing support for multiple GPUs, allowing you to shard your Tensors with `Tensor.shard`
To understand what training looks like in tinygrad, you should read `beautiful_mnist.py`
We have a [quickstart guide](quickstart.md) and a [showcase](showcase.md)
## tinygrad Stack
<img src="./tinygrad_vs_others.png" alt="Tinygrad vs others" style="max-width: 1000px; height: auto;" />
## Differences from PyTorch
If you are migrating from PyTorch, welcome. Most of the API is the same. We hope you will find tinygrad both familiar and somehow more "correct feeling"
### tinygrad doesn't have nn.Module
There's nothing special about a "Module" class in tinygrad, it's just a normal class. [`nn.state.get_parameters`](nn.md/#tinygrad.nn.state.get_parameters) can be used to recursively search normal classes for valid tensors. Instead of the `forward` method in PyTorch, tinygrad just uses `__call__`
### tinygrad is functional
In tinygrad, you can do [`x.conv2d(w, b)`](tensor/ops.md/#tinygrad.Tensor.conv2d) or [`x.sparse_categorical_crossentropy(y)`](tensor/ops.md/#tinygrad.Tensor.sparse_categorical_crossentropy). We do also have a [`Conv2D`](nn.md/#tinygrad.nn.Conv2d) class like PyTorch if you want a place to keep the state, but all stateless operations don't have classes.
### tinygrad is lazy
When you do `a+b` in tinygrad, nothing happens. It's not until you [`realize`](tensor/properties.md#tinygrad.Tensor.realize) the Tensor that the computation actually runs.
### tinygrad requires @TinyJit to be fast
PyTorch spends a lot of development effort to make dispatch very fast. tinygrad doesn't. We have a simple decorator that will replay the kernels used in the decorated function.

11
tinygrad_repo/docs/logo_tiny_dark.svg Normal file
View File

@@ -0,0 +1,11 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="-10 -10 150 70" shape-rendering="crispEdges" fill="#fff">
<!-- t -->
<polygon points="10,40 10,20 0,20 0,10 10,10 10,0 20,0 20,10 30,10 30,20 20,20 20,30 30,30 30,40" />
<!-- i -->
<polygon points="40,40 40,20 50,20 50,40" />
<polygon points="40,10 40,0 50,0 50,10" />
<!-- n -->
<polygon points="60,40 60,10 80,10 80,40 90,40 90,20 70,20 70,40" />
<!-- y -->
<polygon points="100,50 100,40 130,40 130,10 120,10 120,20 110,20 110,10 100,10 100,30 120,30 120,50" />
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 538 B

11
tinygrad_repo/docs/logo_tiny_light.svg Normal file
View File

@@ -0,0 +1,11 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="-10 -10 150 70" shape-rendering="crispEdges">
<!-- t -->
<polygon points="10,40 10,20 0,20 0,10 10,10 10,0 20,0 20,10 30,10 30,20 20,20 20,30 30,30 30,40" />
<!-- i -->
<polygon points="40,40 40,20 50,20 50,40" />
<polygon points="40,10 40,0 50,0 50,10" />
<!-- n -->
<polygon points="60,40 60,10 80,10 80,40 90,40 90,20 70,20 70,40" />
<!-- y -->
<polygon points="100,50 100,40 130,40 130,10 120,10 120,20 110,20 110,10 100,10 100,30 120,30 120,50" />
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 526 B

185
tinygrad_repo/docs/mnist.md Normal file
View File

@@ -0,0 +1,185 @@
# MNIST Tutorial
After you have installed tinygrad, this is a great first tutorial.
Start up a notebook locally, or use [colab](https://colab.research.google.com/). tinygrad is very lightweight, so it's easy to install anywhere and doesn't need a special colab image, but for speed we recommend a T4 GPU image.
### One-liner to install tinygrad in colab
```python
!pip install git+https://github.com/tinygrad/tinygrad.git
```
### What's the default device?
```python
from tinygrad import Device
print(Device.DEFAULT)
```
You will see `CUDA` here on a GPU instance, or `CPU` here on a CPU instance.
## A simple model
We'll use the model from [the Keras tutorial](https://keras.io/examples/vision/mnist_convnet/).
```python
from tinygrad import Tensor, nn
class Model:
def __init__(self):
self.l1 = nn.Conv2d(1, 32, kernel_size=(3,3))
self.l2 = nn.Conv2d(32, 64, kernel_size=(3,3))
self.l3 = nn.Linear(1600, 10)
def __call__(self, x:Tensor) -> Tensor:
x = self.l1(x).relu().max_pool2d((2,2))
x = self.l2(x).relu().max_pool2d((2,2))
return self.l3(x.flatten(1).dropout(0.5))
```
Two key differences from PyTorch:
* Only the stateful layers are declared in `__init__`
* There's no `nn.Module` class or `forward` function, just a normal class and `__call__`
### Getting the dataset
```python
from tinygrad.nn.datasets import mnist
X_train, Y_train, X_test, Y_test = mnist()
print(X_train.shape, X_train.dtype, Y_train.shape, Y_train.dtype)
# (60000, 1, 28, 28) dtypes.uchar (60000,) dtypes.uchar
```
tinygrad includes MNIST, it only adds four lines. Feel free to read the [function](https://github.com/tinygrad/tinygrad/blob/master/tinygrad/nn/datasets.py).
## Using the model
MNIST is small enough that the `mnist()` function copies the dataset to the default device.
So creating the model and evaluating it is a matter of:
```python
model = Model()
acc = (model(X_test).argmax(axis=1) == Y_test).mean()
# NOTE: tinygrad is lazy, and hasn't actually run anything by this point
print(acc.item()) # ~10% accuracy, as expected from a random model
```
### Training the model
We'll use the Adam optimizer. The `nn.state.get_parameters` will walk the model class and pull out the parameters for the optimizer. Also, in tinygrad, it's typical to write a function to do the training step so it can be jitted.
```python
optim = nn.optim.Adam(nn.state.get_parameters(model))
batch_size = 128
def step():
Tensor.training = True # makes dropout work
samples = Tensor.randint(batch_size, high=X_train.shape[0])
X, Y = X_train[samples], Y_train[samples]
optim.zero_grad()
loss = model(X).sparse_categorical_crossentropy(Y).backward()
optim.step()
return loss
```
You can time a step with:
```python
import timeit
timeit.repeat(step, repeat=5, number=1)
#[0.08268719699981375,
# 0.07478952900009972,
# 0.07714716600003158,
# 0.07785399599970333,
# 0.07605237000007037]
```
So around 75 ms on T4 colab.
If you want to see a breakdown of the time by kernel:
```python
from tinygrad import GlobalCounters, Context
GlobalCounters.reset()
with Context(DEBUG=2): step()
```
### Why so slow?
Unlike PyTorch, tinygrad isn't designed to be fast like that. While 75 ms for one step is plenty fast for debugging, it's not great for training. Here, we introduce the first quintessentially tinygrad concept, the `TinyJit`.
```python
from tinygrad import TinyJit
jit_step = TinyJit(step)
```
NOTE: It can also be used as a decorator `@TinyJit`
Now when we time it:
```python
import timeit
timeit.repeat(jit_step, repeat=5, number=1)
# [0.2596786549997887,
# 0.08989566299987928,
# 0.0012115650001760514,
# 0.001010227999813651,
# 0.0012164899999334011]
```
1.0 ms is 75x faster! Note that we aren't syncing the GPU, so GPU time may be slower.
The first two runs of the function execute normally, with the JIT capturing the kernels. Starting from the third run, only the tinygrad operations are replayed, removing the overhead by skipping Python code execution. So be aware that any non-tinygrad Python values affecting the kernels will be "frozen" from the second run. Note that `Tensor` randomness functions work as expected.
Unlike other JITs, we JIT everything, including the optimizer. Think of it as a dumb replay on different data.
## Putting it together
Since we are just randomly sampling from the dataset, there's no real concept of an epoch. We have a batch size of 128, so the Keras example is taking about 7000 steps.
```python
for step in range(7000):
loss = jit_step()
if step%100 == 0:
Tensor.training = False
acc = (model(X_test).argmax(axis=1) == Y_test).mean().item()
print(f"step {step:4d}, loss {loss.item():.2f}, acc {acc*100.:.2f}%")
```
It doesn't take long to reach 98%, and it usually reaches 99%.
```
step 0, loss 4.03, acc 71.43%
step 100, loss 0.34, acc 93.86%
step 200, loss 0.23, acc 95.97%
step 300, loss 0.18, acc 96.32%
step 400, loss 0.18, acc 96.76%
step 500, loss 0.13, acc 97.46%
step 600, loss 0.14, acc 97.45%
step 700, loss 0.10, acc 97.27%
step 800, loss 0.23, acc 97.49%
step 900, loss 0.13, acc 97.51%
step 1000, loss 0.13, acc 97.88%
step 1100, loss 0.11, acc 97.72%
step 1200, loss 0.14, acc 97.65%
step 1300, loss 0.12, acc 98.04%
step 1400, loss 0.25, acc 98.17%
step 1500, loss 0.11, acc 97.86%
step 1600, loss 0.21, acc 98.21%
step 1700, loss 0.14, acc 98.34%
...
```
## From here?
tinygrad is yours to play with now. It's pure Python and short, so unlike PyTorch, fixing library bugs is well within your abilities.
- It's two lines to add multiGPU support to this example (can you find them?). You have to `.shard` the model to all GPUs, and `.shard` the dataset by batch.
- `with Context(DEBUG=2)` shows the running kernels, `DEBUG=4` shows the code. All `Context` variables can also be environment variables.
- `with Context(BEAM=2)` will do a BEAM search on the kernels, searching many possible implementations for what runs the fastest on your hardware. After this search, tinygrad is usually speed competitive with PyTorch, and the results are cached so you won't have to search next time.
[Join our Discord](https://discord.gg/ZjZadyC7PK) for help, and if you want to be a tinygrad developer. Please read the Discord rules when you get there.
[Follow us on Twitter](https://twitter.com/__tinygrad__) to keep up with the project.

40
tinygrad_repo/docs/nn.md Normal file
View File

@@ -0,0 +1,40 @@
## Neural Network classes
::: tinygrad.nn.BatchNorm
::: tinygrad.nn.Conv1d
::: tinygrad.nn.Conv2d
::: tinygrad.nn.ConvTranspose1d
::: tinygrad.nn.ConvTranspose2d
::: tinygrad.nn.Linear
::: tinygrad.nn.GroupNorm
::: tinygrad.nn.InstanceNorm
::: tinygrad.nn.LayerNorm
::: tinygrad.nn.LayerNorm2d
::: tinygrad.nn.RMSNorm
::: tinygrad.nn.Embedding
::: tinygrad.nn.LSTMCell
## Optimizers
::: tinygrad.nn.optim.SGD
::: tinygrad.nn.optim.LARS
::: tinygrad.nn.optim.AdamW
::: tinygrad.nn.optim.Adam
::: tinygrad.nn.optim.LAMB
## Load/Save
::: tinygrad.nn.state.safe_load
::: tinygrad.nn.state.safe_save
::: tinygrad.nn.state.get_state_dict
::: tinygrad.nn.state.get_parameters
::: tinygrad.nn.state.load_state_dict
::: tinygrad.nn.state.tar_extract
options:
show_signature: false
separate_signature: false
::: tinygrad.nn.state.torch_load
options:
show_signature: false
separate_signature: false
::: tinygrad.llm.gguf.gguf_load

305
tinygrad_repo/docs/quickstart.md Normal file
View File

@@ -0,0 +1,305 @@
# Quick Start Guide
This guide assumes no prior knowledge of pytorch or any other deep learning framework, but does assume some basic knowledge of neural networks.
It is intended to be a very quick overview of the high level API that tinygrad provides.
This guide is also structured as a tutorial which at the end of it you will have a working model that can classify handwritten digits.
We need some imports to get started:
```python
import numpy as np
from tinygrad.helpers import Timing
```
## Tensors
Tensors are the base data structure in tinygrad. They can be thought of as a multidimensional array of a specific data type.
All high level operations in tinygrad operate on these tensors.
The tensor class can be imported like so:
```python
from tinygrad import Tensor
```
Tensors can be created from an existing data structure like a python list or numpy ndarray:
```python
t1 = Tensor([1, 2, 3, 4, 5])
na = np.array([1, 2, 3, 4, 5])
t2 = Tensor(na)
```
Tensors can also be created using one of the many factory methods:
```python
full = Tensor.full(shape=(2, 3), fill_value=5) # create a tensor of shape (2, 3) filled with 5
zeros = Tensor.zeros(2, 3) # create a tensor of shape (2, 3) filled with 0
ones = Tensor.ones(2, 3) # create a tensor of shape (2, 3) filled with 1
full_like = Tensor.full_like(full, fill_value=2) # create a tensor of the same shape as `full` filled with 2
zeros_like = Tensor.zeros_like(full) # create a tensor of the same shape as `full` filled with 0
ones_like = Tensor.ones_like(full) # create a tensor of the same shape as `full` filled with 1
eye = Tensor.eye(3) # create a 3x3 identity matrix
arange = Tensor.arange(start=0, stop=10, step=1) # create a tensor of shape (10,) filled with values from 0 to 9
rand = Tensor.rand(2, 3) # create a tensor of shape (2, 3) filled with random values from a uniform distribution
randn = Tensor.randn(2, 3) # create a tensor of shape (2, 3) filled with random values from a standard normal distribution
uniform = Tensor.uniform(2, 3, low=0, high=10) # create a tensor of shape (2, 3) filled with random values from a uniform distribution between 0 and 10
```
There are even more of these factory methods, you can find them in the [Tensor Creation](tensor/creation.md) file.
All the tensors creation methods can take a `dtype` argument to specify the data type of the tensor, find the supported `dtype` in [dtypes](dtypes.md).
```python
from tinygrad import dtypes
t3 = Tensor([1, 2, 3, 4, 5], dtype=dtypes.int32)
```
Tensors allow you to perform operations on them like so:
```python
t4 = Tensor([1, 2, 3, 4, 5])
t5 = (t4 + 1) * 2
t6 = (t5 * t4).relu().log_softmax()
```
All of these operations are lazy and are only executed when you realize the tensor using `.realize()` or `.numpy()`.
```python
print(t6.numpy())
# [-56. -48. -36. -20. 0.]
```
There are a lot more operations that can be performed on tensors, you can find them in the [Tensor Ops](tensor/ops.md) file.
Additionally reading through [abstractions2.py](https://github.com/tinygrad/tinygrad/blob/master/docs/abstractions2.py) will help you understand how operations on these tensors make their way down to your hardware.
## Models
Neural networks in tinygrad are really just represented by the operations performed on tensors.
These operations are commonly grouped into the `__call__` method of a class which allows modularization and reuse of these groups of operations.
These classes do not need to inherit from any base class, in fact if they don't need any trainable parameters they don't even need to be a class!
An example of this would be the `nn.Linear` class which represents a linear layer in a neural network.
```python
class Linear:
def __init__(self, in_features, out_features, bias=True, initialization: str='kaiming_uniform'):
self.weight = getattr(Tensor, initialization)(out_features, in_features)
self.bias = Tensor.zeros(out_features) if bias else None
def __call__(self, x):
return x.linear(self.weight.transpose(), self.bias)
```
There are more neural network modules already implemented in [nn](nn.md), and you can also implement your own.
We will be implementing a simple neural network that can classify handwritten digits from the MNIST dataset.
Our classifier will be a simple 2 layer neural network with a Leaky ReLU activation function.
It will use a hidden layer size of 128 and an output layer size of 10 (one for each digit) with no bias on either Linear layer.
```python
class TinyNet:
def __init__(self):
self.l1 = Linear(784, 128, bias=False)
self.l2 = Linear(128, 10, bias=False)
def __call__(self, x):
x = self.l1(x)
x = x.leaky_relu()
x = self.l2(x)
return x
net = TinyNet()
```
We can see that the forward pass of our neural network is just the sequence of operations performed on the input tensor `x`.
We can also see that functional operations like `leaky_relu` are not defined as classes and instead are just methods we can just call.
Finally, we just initialize an instance of our neural network, and we are ready to start training it.
## Training
Now that we have our neural network defined we can start training it.
Training neural networks in tinygrad is super simple.
All we need to do is define our neural network, define our loss function, and then call `.backward()` on the loss function to compute the gradients.
They can then be used to update the parameters of our neural network using one of the many [Optimizers](nn.md#optimizers).
For our loss function we will be using sparse categorical cross entropy loss. The implementation below is taken from [tensor.py](https://github.com/tinygrad/tinygrad/blob/master/tinygrad/tensor.py), it's copied below to highlight an important detail of tinygrad.
```python
def sparse_categorical_crossentropy(self, Y, ignore_index=-1) -> Tensor:
loss_mask = Y != ignore_index
y_counter = Tensor.arange(self.shape[-1], dtype=dtypes.int32, device=self.device).unsqueeze(0).expand(Y.numel(), self.shape[-1])
y = ((y_counter == Y.flatten().reshape(-1, 1)).where(-1.0, 0) * loss_mask.reshape(-1, 1)).reshape(*Y.shape, self.shape[-1])
return self.log_softmax().mul(y).sum() / loss_mask.sum()
```
As we can see in this implementation of cross entropy loss, there are certain operations that tinygrad does not support natively.
Load/store ops are not supported in tinygrad natively because they add complexity when trying to port to different backends, 90% of the models out there don't use/need them, and they can be implemented like it's done above with an `arange` mask.
For our optimizer we will be using the traditional stochastic gradient descent optimizer with a learning rate of 3e-4.
```python
from tinygrad.nn.optim import SGD
opt = SGD([net.l1.weight, net.l2.weight], lr=3e-4)
```
We can see that we are passing in the parameters of our neural network to the optimizer.
This is due to the fact that the optimizer needs to know which parameters to update.
There is a simpler way to do this just by using `get_parameters(net)` from `tinygrad.nn.state` which will return a list of all the parameters in the neural network.
The parameters are just listed out explicitly here for clarity.
Now that we have our network, loss function, and optimizer defined all we are missing is the data to train on!
There are a couple of dataset loaders in tinygrad located in [/extra/datasets](https://github.com/tinygrad/tinygrad/blob/master/extra/datasets).
We will be using the MNIST dataset loader.
```python
from extra.datasets import fetch_mnist
```
Now we have everything we need to start training our neural network.
We will be training for 1000 steps with a batch size of 64.
We use `with Tensor.train()` to set the internal flag `Tensor.training` to `True` during training.
Upon exit, the flag is restored to its previous value by the context manager.
```python
X_train, Y_train, X_test, Y_test = fetch_mnist()
with Tensor.train():
for step in range(1000):
# random sample a batch
samp = np.random.randint(0, X_train.shape[0], size=(64))
batch = Tensor(X_train[samp])
# get the corresponding labels
labels = Tensor(Y_train[samp])
# forward pass
out = net(batch)
# compute loss
loss = sparse_categorical_crossentropy(out, labels)
# zero gradients
opt.zero_grad()
# backward pass
loss.backward()
# update parameters
opt.step()
# calculate accuracy
pred = out.argmax(axis=-1)
acc = (pred == labels).mean()
if step % 100 == 0:
print(f"Step {step+1} | Loss: {loss.numpy()} | Accuracy: {acc.numpy()}")
```
## Evaluation
Now that we have trained our neural network we can evaluate it on the test set.
We will be using the same batch size of 64 and will be evaluating for 1000 of those batches.
```python
with Timing("Time: "):
avg_acc = 0
for step in range(1000):
# random sample a batch
samp = np.random.randint(0, X_test.shape[0], size=(64))
batch = Tensor(X_test[samp])
# get the corresponding labels
labels = Y_test[samp]
# forward pass
out = net(batch)
# calculate accuracy
pred = out.argmax(axis=-1).numpy()
avg_acc += (pred == labels).mean()
print(f"Test Accuracy: {avg_acc / 1000}")
```
## And that's it
Highly recommend you check out the [examples/](https://github.com/tinygrad/tinygrad/blob/master/examples) folder for more examples of using tinygrad.
Reading the source code of tinygrad is also a great way to learn how it works.
Specifically the tests in [test/](https://github.com/tinygrad/tinygrad/blob/master/test) are a great place to see how to use and the semantics of the different operations.
There are also a bunch of models implemented in [models/](https://github.com/tinygrad/tinygrad/blob/master/extra/models) that you can use as a reference.
Additionally, feel free to ask questions in the `#learn-tinygrad` channel on the [discord](https://discord.gg/beYbxwxVdx). Don't ask to ask, just ask!
## Extras
### JIT
Additionally, it is possible to speed up the computation of certain neural networks by using the JIT.
Currently, this does not support models with varying input sizes and non tinygrad operations.
To use the JIT we just need to add a function decorator to the forward pass of our neural network and ensure that the input and output are realized tensors.
Or in this case we will create a wrapper function and decorate the wrapper function to speed up the evaluation of our neural network.
```python
from tinygrad import TinyJit
@TinyJit
def jit(x):
return net(x).realize()
with Timing("Time: "):
avg_acc = 0
for step in range(1000):
# random sample a batch
samp = np.random.randint(0, X_test.shape[0], size=(64))
batch = Tensor(X_test[samp])
# get the corresponding labels
labels = Y_test[samp]
# forward pass with jit
out = jit(batch)
# calculate accuracy
pred = out.argmax(axis=-1).numpy()
avg_acc += (pred == labels).mean()
print(f"Test Accuracy: {avg_acc / 1000}")
```
You will find that the evaluation time is much faster than before and that your accelerator utilization is much higher.
### Saving and Loading Models
The standard weight format for tinygrad is [safetensors](https://github.com/huggingface/safetensors). This means that you can load the weights of any model also using safetensors into tinygrad.
There are functions in [state.py](https://github.com/tinygrad/tinygrad/blob/master/tinygrad/nn/state.py) to save and load models to and from this format.
```python
from tinygrad.nn.state import safe_save, safe_load, get_state_dict, load_state_dict
# first we need the state dict of our model
state_dict = get_state_dict(net)
# then we can just save it to a file
safe_save(state_dict, "model.safetensors")
# and load it back in
state_dict = safe_load("model.safetensors")
load_state_dict(net, state_dict)
```
Many of the models in the [models/](https://github.com/tinygrad/tinygrad/tree/master/extra/models) folder have a `load_from_pretrained` method that will download and load the weights for you. These usually are pytorch weights meaning that you would need pytorch installed to load them.
### Environment Variables
There exist a bunch of environment variables that control the runtime behavior of tinygrad.
Some of the commons ones are `DEBUG` and the different backend enablement variables.
You can find a full list and their descriptions in [env_vars.md](env_vars.md).
### Visualizing the Computation Graph
It is possible to visualize the computation graph of a neural network using VIZ=1.

91
tinygrad_repo/docs/runtime.md Normal file
View File

@@ -0,0 +1,91 @@
# Runtimes
tinygrad supports various runtimes, enabling your code to scale across a wide range of devices. The default runtime can be automatically selected based on the available hardware, or you can force a specific runtime to be default using environment variables (e.g., `DEV=CPU`).
| Runtime | Description | Compiler Options | Requirements |
|---------|-------------|------------------|--------------|
| [NV](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_nv.py) | Provides acceleration for NVIDIA GPUs | nvrtc (default)<br>PTX (`DEV=NV:PTX`) | Ampere/Ada/Blackwell series GPUs.<br>You can select an interface via [the `DEV` variable](env_vars.md#dev-variable). See [NV interfaces](#nv-interfaces) for details. |
| [AMD](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_amd.py) | Provides acceleration for AMD GPUs | LLVM (`DEV=AMD:LLVM`)<br>HIP/COMGR (`DEV=AMD:HIP`) | CDNA3, CDNA4, RDNA3 or RDNA4 GPUs.<br>You can select an interface via [the `DEV` variable](env_vars.md#dev-variable). See [AMD interfaces](#amd-interfaces) for details. |
| [QCOM](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_qcom.py) | Provides acceleration for QCOM GPUs | - | 6xx series GPUs |
| [METAL](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_metal.py) | Utilizes Metal for acceleration on Apple devices | - | M1+ Macs; Metal 3.0+ for `bfloat` support |
| [CUDA](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_cuda.py) | Utilizes CUDA for acceleration on NVIDIA GPUs | nvrtc (default)<br> PTX (`DEV=CUDA:PTX`) | NVIDIA GPU with CUDA support |
| [CL](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_cl.py) | Accelerates computations using OpenCL on GPUs | - | OpenCL 2.0 compatible device |
| [CPU](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_cpu.py) | Runs on CPU using the clang or llvm compiler | Clang JIT (default)<br>LLVM IR (`DEV=CPU:LLVM`) | `clang` compiler in system `PATH`<br>You can specify additional arch parameters via [the `DEV` variable](env_vars.md#dev-variable). See [CPU arch](#cpu-arch) for details. |
| [WEBGPU](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_webgpu.py) | Runs on GPU using the Dawn WebGPU engine (used in Google Chrome) | - | Dawn library installed and discoverable. Binaries: [pydawn v0.3.0](https://github.com/wpmed92/pydawn/releases/tag/v0.3.0) |
## Interoperability
tinygrad provides interoperability with OpenCL and PyTorch, allowing efficient tensor data sharing between frameworks through the `Tensor.from_blob` API. This enables zero-copy operations by working directly with external memory pointers.
**Important**: When using external memory pointers with tinygrad tensors, you must ensure these pointers remain valid throughout the entire lifetime of the tinygrad tensor to prevent memory corruption.
### `CUDA`/`METAL` PyTorch Interoperability
You can seamlessly work with CUDA/MPS tensors between PyTorch and tinygrad without data copying:
```python
from tinygrad.dtype import _from_torch_dtype
tensor1 = torch.tensor([1.0, 2.0, 3.0], device=torch.device("cuda"))
tiny_tensor1 = Tensor.from_blob(tensor1.data_ptr(), tensor1.shape, dtype=_from_torch_dtype(tensor1.dtype), device='CUDA')
# Before tinygrad calculations, mps needs to be synchronized to make sure data is valid.
if data.device.type == "mps": torch.mps.synchronize()
else: torch.cuda.synchronize()
x = (tiny_tensor1 + 1).realize()
```
### `QCOM` OpenCL Interoperability
tinygrad supports OpenCL interoperability on `QCOM` backend.
Buffer interop allows direct access to OpenCL memory buffers:
```python
# create raw opencl buffer.
cl_buf = cl.clCreateBuffer(cl_context, cl.CL_MEM_READ_WRITE, 0x100, None, status := ctypes.c_int32())
# extract pointers
cl_buf_desc_ptr = to_mv(ctypes.addressof(cl_buf), 8).cast('Q')[0]
rawbuf_ptr = to_mv(cl_buf_desc_ptr, 0x100).cast('Q')[20] # offset 0xA0 is a raw gpu pointer.
# create tiny tensor
tiny = Tensor.from_blob(rawbuf_ptr, (8, 8), dtype=dtypes.int, device='QCOM')
```
And the same for the images:
```python
# create cl image.
cl_img = cl.clCreateImage2D(cl_context, cl.CL_MEM_READ_WRITE, cl.cl_image_format(cl.CL_RGBA, cl.CL_FLOAT), w, h, 0, None, status := ctypes.c_int32())
# extract pointers
cl_buf_desc_ptr = to_mv(ctypes.addressof(cl_img), 8).cast('Q')[0]
rawbuf_ptr = to_mv(cl_buf_desc_ptr, 0x100).cast('Q')[20] # offset 0xA0 is a raw gpu pointer.
# create tiny tensor
tiny = Tensor.from_blob(rawbuf_ptr, (h*w*4,), dtype=dtypes.imagef((h,w)), device='QCOM')
```
## AMD Interfaces
AMD backend supports several interfaces for communicating with devices:
* `KFD`: uses the amdgpu driver
* `PCI`: uses the [AM driver](developer/am.md)
* `USB`: USB3 interface for asm24xx chips.
You can force an interface by setting the interface component of [the `DEV` environment variable](env_vars.md#dev-variable) to one of these values. When set to `PCI`, this may unbind your GPU from the amdgpu driver.
## NV Interfaces
NV backend supports several interfaces for communicating with devices:
* `NVK`: uses the nvidia driver
* `PCI`: uses the [NV driver](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/support/nv/nvdev.py)
## CPU Arch
The CPU renderers may be additionally configured using the arch component of [the `DEV` environment variable](env_vars.md#dev-variable).
CPU arch should be specified as a comma-separated list of parameters, and must contain at least two values: the architecture family (ie. x86_64, arm64, or riscv64) and the cpu type (as accepted by `clang`'s `-march`).
If native is specified as the cpu type, tinygrad (or delegate compiler) will query the host cpu type. Additional comma-separated values may be specified as follows:
* `AMX`: emit Apple silicon AMX instructions
All other additional values are interpreted as cpu feature flags. When a value is preceded by a `-` character, the corresponding feature flag will be disabled, otherwise the flag will be enabled.
Note that enabled feature flags should not be preceded by a `+`.

62
tinygrad_repo/docs/showcase.md Normal file
View File

@@ -0,0 +1,62 @@
# Showcase
Despite being a tiny library, tinygrad is capable of doing a lot of things. From state-of-the-art [vision](https://arxiv.org/abs/1905.11946) to state-of-the-art [language](https://arxiv.org/abs/1706.03762) models.
## Vision
### EfficientNet
You can either pass in the URL of a picture to discover what it is:
```sh
python3 examples/efficientnet.py ./test/models/efficientnet/Chicken.jpg
```
Or, if you have a camera and OpenCV installed, you can detect what is in front of you:
```sh
python3 examples/efficientnet.py webcam
```
### YOLOv8
Take a look at [yolov8.py](https://github.com/tinygrad/tinygrad/tree/master/examples/yolov8.py).
![yolov8 by tinygrad](https://github.com/tinygrad/tinygrad/blob/master/docs/showcase/yolov8_showcase_image.png?raw=true)
## Audio
### Whisper
Take a look at [whisper.py](https://github.com/tinygrad/tinygrad/tree/master/examples/whisper.py). You need pyaudio and torchaudio installed.
```sh
SMALL=1 python3 examples/whisper.py
```
## Generative
### Stable Diffusion
```sh
python3 examples/stable_diffusion.py
```
![a horse sized cat eating a bagel](https://github.com/tinygrad/tinygrad/blob/master/docs/showcase/stable_diffusion_by_tinygrad.jpg?raw=true)
*"a horse sized cat eating a bagel"*
### LLaMA
You will need to download and put the weights into the `weights/LLaMA` directory, which may need to be created.
Then you can have a chat with Stacy:
```sh
python3 examples/llama.py
```
### Conversation
Make sure you have espeak installed and `PHONEMIZER_ESPEAK_LIBRARY` set.
Then you can talk to Stacy:
```sh
python3 examples/conversation.py
```

BIN
tinygrad_repo/docs/showcase/mnist_by_tinygrad.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 15 KiB

BIN
tinygrad_repo/docs/showcase/stable_diffusion_by_tinygrad.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 87 KiB

BIN
tinygrad_repo/docs/showcase/yolo_by_tinygrad.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 131 KiB

BIN
tinygrad_repo/docs/showcase/yolov8_showcase_image.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.1 MiB

33
tinygrad_repo/docs/tensor/creation.md Normal file
View File

@@ -0,0 +1,33 @@
## Creation (basic)
::: tinygrad.Tensor.empty
::: tinygrad.Tensor.zeros
::: tinygrad.Tensor.ones
::: tinygrad.Tensor.full
::: tinygrad.Tensor.arange
::: tinygrad.Tensor.linspace
::: tinygrad.Tensor.eye
::: tinygrad.Tensor.full_like
::: tinygrad.Tensor.zeros_like
::: tinygrad.Tensor.ones_like
## Creation (external)
::: tinygrad.Tensor.from_blob
::: tinygrad.Tensor.from_url
## Creation (random)
::: tinygrad.Tensor.manual_seed
::: tinygrad.Tensor.rand
::: tinygrad.Tensor.rand_like
::: tinygrad.Tensor.randn
::: tinygrad.Tensor.randn_like
::: tinygrad.Tensor.randint
::: tinygrad.Tensor.randperm
::: tinygrad.Tensor.normal
::: tinygrad.Tensor.uniform
::: tinygrad.Tensor.scaled_uniform
::: tinygrad.Tensor.glorot_uniform
::: tinygrad.Tensor.kaiming_uniform
::: tinygrad.Tensor.kaiming_normal

95
tinygrad_repo/docs/tensor/elementwise.md Normal file
View File

@@ -0,0 +1,95 @@
Elementwise ops operate on a per element basis. They don't change the shape of the tensor.
## Unary Ops (math)
::: tinygrad.Tensor.logical_not
::: tinygrad.Tensor.neg
::: tinygrad.Tensor.log
::: tinygrad.Tensor.log2
::: tinygrad.Tensor.log10
::: tinygrad.Tensor.exp
::: tinygrad.Tensor.exp2
::: tinygrad.Tensor.sqrt
::: tinygrad.Tensor.rsqrt
::: tinygrad.Tensor.sin
::: tinygrad.Tensor.cos
::: tinygrad.Tensor.tan
::: tinygrad.Tensor.asin
::: tinygrad.Tensor.acos
::: tinygrad.Tensor.atan
::: tinygrad.Tensor.trunc
::: tinygrad.Tensor.ceil
::: tinygrad.Tensor.floor
::: tinygrad.Tensor.round
::: tinygrad.Tensor.isinf
::: tinygrad.Tensor.isnan
::: tinygrad.Tensor.isfinite
::: tinygrad.Tensor.lerp
::: tinygrad.Tensor.square
::: tinygrad.Tensor.clamp
::: tinygrad.Tensor.clip
::: tinygrad.Tensor.sign
::: tinygrad.Tensor.abs
::: tinygrad.Tensor.reciprocal
## Unary Ops (activation)
::: tinygrad.Tensor.relu
::: tinygrad.Tensor.sigmoid
::: tinygrad.Tensor.logsigmoid
::: tinygrad.Tensor.hardsigmoid
::: tinygrad.Tensor.elu
::: tinygrad.Tensor.celu
::: tinygrad.Tensor.selu
::: tinygrad.Tensor.swish
::: tinygrad.Tensor.silu
::: tinygrad.Tensor.relu6
::: tinygrad.Tensor.hardswish
::: tinygrad.Tensor.tanh
::: tinygrad.Tensor.sinh
::: tinygrad.Tensor.cosh
::: tinygrad.Tensor.atanh
::: tinygrad.Tensor.asinh
::: tinygrad.Tensor.acosh
::: tinygrad.Tensor.hardtanh
::: tinygrad.Tensor.erf
::: tinygrad.Tensor.gelu
::: tinygrad.Tensor.quick_gelu
::: tinygrad.Tensor.leaky_relu
::: tinygrad.Tensor.mish
::: tinygrad.Tensor.softplus
::: tinygrad.Tensor.softsign
## Elementwise Ops (broadcasted)
::: tinygrad.Tensor.add
::: tinygrad.Tensor.sub
::: tinygrad.Tensor.mul
::: tinygrad.Tensor.div
::: tinygrad.Tensor.mod
::: tinygrad.Tensor.fmod
::: tinygrad.Tensor.bitwise_xor
::: tinygrad.Tensor.bitwise_and
::: tinygrad.Tensor.bitwise_or
::: tinygrad.Tensor.bitwise_not
::: tinygrad.Tensor.lshift
::: tinygrad.Tensor.rshift
::: tinygrad.Tensor.pow
::: tinygrad.Tensor.maximum
::: tinygrad.Tensor.minimum
::: tinygrad.Tensor.where
::: tinygrad.Tensor.copysign
::: tinygrad.Tensor.logaddexp
## Casting Ops
::: tinygrad.Tensor.cast
::: tinygrad.Tensor.bitcast
::: tinygrad.Tensor.float
::: tinygrad.Tensor.half
::: tinygrad.Tensor.int
::: tinygrad.Tensor.bool
::: tinygrad.Tensor.bfloat16
::: tinygrad.Tensor.double
::: tinygrad.Tensor.long
::: tinygrad.Tensor.short

7
tinygrad_repo/docs/tensor/index.md Normal file
View File

@@ -0,0 +1,7 @@
# Tensor
::: tinygrad.Tensor
options:
heading_level: 2
members: false
show_source: false

32
tinygrad_repo/docs/tensor/movement.md Normal file
View File

@@ -0,0 +1,32 @@
## Movement (low level)
::: tinygrad.Tensor.view
::: tinygrad.Tensor.reshape
::: tinygrad.Tensor.expand
::: tinygrad.Tensor.permute
::: tinygrad.Tensor.flip
::: tinygrad.Tensor.shrink
::: tinygrad.Tensor.pad
## Movement (high level)
::: tinygrad.Tensor.__getitem__
::: tinygrad.Tensor.gather
::: tinygrad.Tensor.cat
::: tinygrad.Tensor.stack
::: tinygrad.Tensor.repeat
::: tinygrad.Tensor.repeat_interleave
::: tinygrad.Tensor.split
::: tinygrad.Tensor.chunk
::: tinygrad.Tensor.unfold
::: tinygrad.Tensor.meshgrid
::: tinygrad.Tensor.squeeze
::: tinygrad.Tensor.unsqueeze
::: tinygrad.Tensor.T
::: tinygrad.Tensor.transpose
::: tinygrad.Tensor.flatten
::: tinygrad.Tensor.unflatten
::: tinygrad.Tensor.diag
::: tinygrad.Tensor.diagonal
::: tinygrad.Tensor.roll
::: tinygrad.Tensor.rearrange

68
tinygrad_repo/docs/tensor/ops.md Normal file
View File

@@ -0,0 +1,68 @@
## Reduce
::: tinygrad.Tensor.sum
::: tinygrad.Tensor.prod
::: tinygrad.Tensor.max
::: tinygrad.Tensor.min
::: tinygrad.Tensor.any
::: tinygrad.Tensor.all
::: tinygrad.Tensor.isclose
::: tinygrad.Tensor.allclose
::: tinygrad.Tensor.mean
::: tinygrad.Tensor.var
::: tinygrad.Tensor.var_mean
::: tinygrad.Tensor.std
::: tinygrad.Tensor.std_mean
::: tinygrad.Tensor.softmax
::: tinygrad.Tensor.log_softmax
::: tinygrad.Tensor.logsumexp
::: tinygrad.Tensor.logcumsumexp
::: tinygrad.Tensor.argmax
::: tinygrad.Tensor.argmin
## Processing
::: tinygrad.Tensor.avg_pool2d
::: tinygrad.Tensor.max_pool2d
::: tinygrad.Tensor.max_unpool2d
::: tinygrad.Tensor.conv2d
::: tinygrad.Tensor.conv_transpose2d
::: tinygrad.Tensor.dot
::: tinygrad.Tensor.matmul
::: tinygrad.Tensor.einsum
::: tinygrad.Tensor.cumsum
::: tinygrad.Tensor.cumprod
::: tinygrad.Tensor.cummax
::: tinygrad.Tensor.cummin
::: tinygrad.Tensor.triu
::: tinygrad.Tensor.tril
::: tinygrad.Tensor.interpolate
::: tinygrad.Tensor.scatter
::: tinygrad.Tensor.scatter_reduce
::: tinygrad.Tensor.masked_select
::: tinygrad.Tensor.masked_fill
::: tinygrad.Tensor.nonzero
::: tinygrad.Tensor.sort
::: tinygrad.Tensor.argsort
::: tinygrad.Tensor.topk
::: tinygrad.Tensor.multinomial
## Neural Network (functional)
::: tinygrad.Tensor.linear
::: tinygrad.Tensor.sequential
::: tinygrad.Tensor.layernorm
::: tinygrad.Tensor.batchnorm
::: tinygrad.Tensor.dropout
::: tinygrad.Tensor.one_hot
::: tinygrad.Tensor.scaled_dot_product_attention
::: tinygrad.Tensor.binary_crossentropy
::: tinygrad.Tensor.binary_crossentropy_logits
::: tinygrad.Tensor.sparse_categorical_crossentropy
::: tinygrad.Tensor.cross_entropy
::: tinygrad.Tensor.nll_loss
## Linear Algebra
::: tinygrad.Tensor.qr
::: tinygrad.Tensor.svd

39
tinygrad_repo/docs/tensor/properties.md Normal file
View File

@@ -0,0 +1,39 @@
## Basic
::: tinygrad.Tensor.shape
::: tinygrad.Tensor.dtype
::: tinygrad.Tensor.device
::: tinygrad.Tensor.ndim
::: tinygrad.Tensor.numel
::: tinygrad.Tensor.element_size
::: tinygrad.Tensor.nbytes
::: tinygrad.Tensor.is_floating_point
::: tinygrad.Tensor.size
## Data Access
::: tinygrad.Tensor.data
::: tinygrad.Tensor.item
::: tinygrad.Tensor.tolist
::: tinygrad.Tensor.numpy
## tinygrad ops
::: tinygrad.Tensor.linear_with_vars
::: tinygrad.Tensor.schedule_linear
::: tinygrad.Tensor.realize
::: tinygrad.Tensor.replace
::: tinygrad.Tensor.assign
::: tinygrad.Tensor.detach
::: tinygrad.Tensor.clone
::: tinygrad.Tensor.to
::: tinygrad.Tensor.to_
::: tinygrad.Tensor.shard
::: tinygrad.Tensor.shard_
::: tinygrad.Tensor.contiguous
::: tinygrad.Tensor.contiguous_backward
## Gradient
::: tinygrad.Tensor.gradient
::: tinygrad.Tensor.backward

54
tinygrad_repo/docs/tinybox.md Normal file
View File

@@ -0,0 +1,54 @@
# tinybox
Although these docs live in tinygrad, they pertain to deep learning hardware sold by the tiny corp. tinyboxes are used heavily in tinygrad's CI, and are the best tested platform to use tinygrad with. They appeared running tinygrad on [MLPerf Training 4.0](https://public.tableau.com/views/MLCommons-Training_16993769118290/MLCommons-Training)
If you don't have a tinybox and you want one, see [tinygrad.org](https://tinygrad.org). If you don't want one, that's okay too.
## Welcome
Welcome to your tinybox! The tinybox is the universal system purpose-built for all AI infrastructure and workloads, from training to inference. The red box includes six 7900XTX GPUs, the green box includes six 4090 GPUs, and the green v2 box includes four 5090 GPUs. Whether you bought a red one or a green one, we want you to love it.
We don't have a stupid cloud service, you don't have to create a tiny account to set it up, and we aren't tracking how you use the box. We're just happy you bought one. This petaflop is your petaflop.
## Plugging it in
tinybox has two 1600W PSUs, which together exceed the capacity of most 120V household circuits. Fortunately, it comes with two plugs. You'll want to plug each plug into a different circuit. You can verify that they are different circuits by flipping the breaker and seeing what turns off. If you have at least a 120V 30A or 220V 20A circuit, you are welcome to use only that one.
You'll also want to connect the Ethernet port without a rubber stopper to your home network.
While it's designed primarily for the home or office, the tinybox is 12U rack mountable using [these rails](https://rackmountmart.store.turbify.net/26slidrailfo.html).
## Power limiting the box
While a tinybox should ideally be run without power limits, there are cases where you might want to run the box off of a single outlet.
In such cases, it is possible to power limit the box using the provided `power-limit` script, which will power limit all of the GPUs to a specified wattage.
`sudo power-limit 150` should be good to run off of a single 120V 15A outlet.
## Connecting to the box
tinybox ships with a relatively basic install of Ubuntu 22.04. To do initial setup, you can either plug in a VGA monitor and keyboard, or you can connect remotely to the machine using the BMC. The BMC IP and password are displayed on the screen.
`ipmitool -H <BMC IP> -U admin -P <BMC PW> -I lanplus sol activate`
The default username is `tiny` and the default password is `tiny`. Once you are logged in, you can add an SSH key to authorized keys to connect over SSH (on the normal IP). Exit `ipmitool` with `~.` after a newline.
The BMC also has a web interface you can use if you find that easier.
## Changing the BMC password
It is recommended that you change the BMC password after setting up the box, as the password on the screen is only the initial password.
If you do decide to change the BMC password and no longer want the initial password to be displayed, remove the `/root/.bmc_password` file.
Reboot after making these changes or restart the `tinybox-display.service` service.
## What do I use it for?
The [default tinybox image](https://github.com/tinygrad/tinyos) ships with tinygrad and PyTorch. While we develop tinygrad, the box is universal hardware. Use whatever framework you desire, run notebooks, download demos, install more things, train, inference, live, laugh, love, you aren't paying per hour for this box so the only limit is your imagination.
## Building the OS image
The OS image is built using `ubuntu-image` from <https://github.com/tinygrad/tinyos>.
After cloning, run `make green` or `make red` to build a tinybox green or tinybox red image respectively.

61
tinygrad_repo/docs/tinygpu.md Normal file
View File

@@ -0,0 +1,61 @@
# TinyGPU
TinyGPU app lets you use AMD and NVIDIA GPUs on macOS over USB4/Thunderbolt with tinygrad.
## Requirements
- macOS (13.0+)
- USB4/Thunderbolt port
- A supported GPU (AMD RDNA3+ or NVIDIA Ampere+)
## Setup
### 1. Connect your GPU
Plug the supported GPU into your Mac over USB4/Thunderbolt.
### 2. Initiate the driver install
> **Note:** If tinygrad is cloned but not installed, run commands with `PYTHONPATH=.`
```bash
curl -fsSL https://raw.githubusercontent.com/tinygrad/tinygrad/master/extra/setup_tinygpu_osx.sh | sh
```
This downloads TinyGPU.app and triggers a system prompt to install the driver extension.
### 3. Enable the driver
You should see a system prompt: **"TinyGPU" would like to use a new driver extension**. Click **Open System Settings** and toggle TinyGPU on.
If you missed the prompt, go to **System Settings > General > Login Items & Extensions > Driver Extensions** and toggle TinyGPU on.
### 4. Compiler Setup
#### AMD
```bash
curl -fsSL https://raw.githubusercontent.com/tinygrad/tinygrad/master/extra/setup_hipcomgr_osx.sh | sh
```
#### NV
Install [Docker Desktop](https://www.docker.com/products/docker-desktop/) if you don't have it.
```bash
curl -fsSL https://raw.githubusercontent.com/tinygrad/tinygrad/master/extra/setup_nvcc_osx.sh | sh
```
Make sure `~/.local/bin` is on your `PATH`:
```bash
export PATH="$HOME/.local/bin:$PATH"
```
### 5. Use it!
```bash
DEV={AMD|NV} python3 -m tinygrad.llm
```
**Note:** Use `JITBEAM=2` to search for faster kernels (one-time search cost, results cached).

BIN
tinygrad_repo/docs/tinygrad_intro.pdf Normal file
View File

Binary file not shown.

BIN
tinygrad_repo/docs/tinygrad_vs_others.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 63 KiB