IQ.Pilot Release Commit @ bec7652

artifacts/package_sources/tinygrad/extra/hcqfuzz/tests/allocator.py

@@ -0,0 +1,16 @@
+from extra.hcqfuzz.spec import TestSpec
+import random
+
+class TLSFAllocator(TestSpec):
+  def prepare(self, dev, seed):
+    random.seed(seed)
+
+    self.env = {
+        "SEED": seed,
+        "ITERS": random.randint(10000, 1000000),
+    }
+
+    self.cmd = "python3 test/external/external_fuzz_tlsf.py"
+    self.timeout = 60 * 60 # 60 minutes
+
+  def get_exec_state(self): return self.env, self.cmd, self.timeout

artifacts/package_sources/tinygrad/extra/hcqfuzz/tests/allreduce.py

@@ -0,0 +1,17 @@
+from extra.hcqfuzz.spec import TestSpec
+import random
+
+class RingAllreduce(TestSpec):
+  def prepare(self, dev, seed):
+    random.seed(seed)
+
+    self.env = {
+      "GPUS": random.choice([2, 3, 4, 5, 6]),
+      "ITERS": random.randint(10, 1000),
+      "DEBUG": 2,
+    }
+
+    self.cmd = "python3 test/external/external_benchmark_multitensor_allreduce.py"
+    self.timeout = 10 * 60 # 10 minutes
+
+  def get_exec_state(self): return self.env, self.cmd, self.timeout

artifacts/package_sources/tinygrad/extra/hcqfuzz/tests/bert.py

@@ -0,0 +1,81 @@
+from extra.hcqfuzz.spec import TestSpec
+import random
+
+bert_train_params = {
+  "DEFAULT_FLOAT": "HALF",
+  "SUM_DTYPE": "HALF",
+  "GPUS": 6,
+  "BS": 96,
+  "EVAL_BS": 96,
+  "BASEDIR": "/raid/datasets/wiki",
+}
+
+class TrainBert(TestSpec):
+  def prepare(self, dev, seed):
+    random.seed(seed)
+
+    self.env = {
+      **bert_train_params,
+      "IGNORE_BEAM_CACHE": 1,
+      "BEAM": 5,
+      "BEAM_UOPS_MAX": 10000,
+      "BEAM_UPCAST_MAX": 256,
+      "BEAM_LOCAL_MAX": 1024,
+      "BEAM_MIN_PROGRESS": 5,
+      "IGNORE_JIT_FIRST_BEAM": 1,
+      "LOGMLPERF": 0,
+      "SEED": seed,
+    }
+
+    self.cmd = "python3 examples/mlperf/model_train.py"
+    self.timeout = 7 * 60 * 60 # 7 hours
+
+  def get_exec_state(self): return self.env, self.cmd, self.timeout
+
+class TrainBertShort(TestSpec):
+  def prepare(self, dev, seed):
+    random.seed(seed)
+
+    self.env = {
+      **bert_train_params,
+      "IGNORE_BEAM_CACHE": 1,
+      "BEAM": 5,
+      "BEAM_UOPS_MAX": 10000,
+      "BEAM_UPCAST_MAX": 256,
+      "BEAM_LOCAL_MAX": 1024,
+      "BEAM_MIN_PROGRESS": 5,
+      "IGNORE_JIT_FIRST_BEAM": 1,
+      "SEED": seed,
+      "BENCHMARK": 4096,
+      "JIT": 2
+    }
+
+    self.cmd = "python3 examples/mlperf/model_train.py"
+    self.timeout = 2 * 60 * 60 # 2 hours
+
+  def get_exec_state(self): return self.env, self.cmd, self.timeout
+
+class BertBeam(TestSpec):
+  def prepare(self, dev, seed):
+    random.seed(seed)
+
+    self.env = {
+      **bert_train_params,
+      "IGNORE_BEAM_CACHE": 1,
+      "BEAM": random.choice([1, 2, 3, 4, 5]),
+      "BEAM_UOPS_MAX": 10000,
+      "BEAM_UPCAST_MAX": 256,
+      "BEAM_LOCAL_MAX": 1024,
+      "BEAM_MIN_PROGRESS": 5,
+      "IGNORE_JIT_FIRST_BEAM": 1,
+      "SEED": seed,
+      "RESET_STEP": 1,
+      "BENCHMARK": 10,
+      "BERT_LAYERS": 2,
+      "SEED": seed,
+    }
+
+    self.cmd = "python3 examples/mlperf/model_train.py"
+    self.timeout = 1 * 60 * 60 # 1 hour
+
+  def get_exec_state(self): return self.env, self.cmd, self.timeout

artifacts/package_sources/tinygrad/extra/hcqfuzz/tests/hcq.py

@@ -0,0 +1,35 @@
+from extra.hcqfuzz.spec import TestSpec
+import random
+
+class HCQSignalFuzzer(TestSpec):
+  def prepare(self, dev, seed):
+    random.seed(seed)
+
+    self.env = {
+      "GPUS": random.choice([2, 3, 4, 5, 6]),
+      "ITERS": random.randint(1000000, 10000000),
+      "SEED": seed,
+    }
+
+    self.cmd = "python3 test/external/external_fuzz_hcq_signals.py"
+    self.timeout = 30 * 60 # 30 minutes
+
+  def get_exec_state(self): return self.env, self.cmd, self.timeout
+
+class HCQGraphFuzzer(TestSpec):
+  def prepare(self, dev, seed):
+    random.seed(seed)
+
+    self.env = {
+      "FUZZ_GRAPH_SPLIT_RUNS": random.randint(48, 64),
+      "FUZZ_GRAPH_MAX_SPLITS": random.randint(4, 16),
+      "FUZZ_GRAPH_SPLIT_RETRY_RUNS": random.randint(4, 8),
+      "MAX_KERNELS": random.randint(32, 512),
+      "MAX_DEVICES": random.choice([2, 3, 4, 5, 6]),
+      "ITERS": random.randint(100, 1000),
+    }
+
+    self.cmd = "python3 test/external/fuzz_graph.py"
+    self.timeout = 60 * 60 # 60 minutes
+
+  def get_exec_state(self): return self.env, self.cmd, self.timeout

artifacts/package_sources/tinygrad/extra/hcqfuzz/tests/resnet.py

@@ -0,0 +1,66 @@
+from extra.hcqfuzz.spec import TestSpec
+import random
+
+resnet_train_params = {
+  "DEFAULT_FLOAT": "HALF",
+  "SUM_DTYPE": "HALF",
+  "GPUS": 6,
+  "BS": 1536,
+  "EVAL_BS": 192,
+  "TRAIN_BEAM": 4,
+  "IGNORE_JIT_FIRST_BEAM": 1,
+  "BEAM_UOPS_MAX": 2000,
+  "BEAM_UPCAST_MAX": 96,
+  "BEAM_LOCAL_MAX": 1024,
+  "BEAM_MIN_PROGRESS": 5,
+  "BEAM_PADTO": 0,
+  "EVAL_START_EPOCH": 3,
+  "EVAL_FREQ": 4
+}
+
+class TrainResnet(TestSpec):
+  def prepare(self, dev, seed):
+    random.seed(seed)
+
+    self.env = {
+      **resnet_train_params,
+      "IGNORE_BEAM_CACHE": 1,
+      "SEED": seed,
+    }
+
+    self.cmd = "python3 examples/mlperf/model_train.py"
+    self.timeout = 4 * 60 * 60 # 7 hours
+
+  def get_exec_state(self): return self.env, self.cmd, self.timeout
+
+class TrainResnetShort(TestSpec):
+  def prepare(self, dev, seed):
+    random.seed(seed)
+
+    self.env = {
+      **resnet_train_params,
+      "SEED": seed,
+      "BENCHMARK": 4096,
+      "JIT": 2,
+    }
+
+    self.cmd = "python3 examples/mlperf/model_train.py"
+    self.timeout = 2 * 60 * 60 # 2 hours
+
+  def get_exec_state(self): return self.env, self.cmd, self.timeout
+
+class ResnetBeam(TestSpec):
+  def prepare(self, dev, seed):
+    random.seed(seed)
+
+    self.env = {
+      **resnet_train_params,
+      "IGNORE_BEAM_CACHE": 1,
+      "BENCHMARK": 10,
+      "SEED": seed,
+    }
+
+    self.cmd = "python3 examples/mlperf/model_train.py"
+    self.timeout = 1 * 60 * 60 # 1 hour
+
+  def get_exec_state(self): return self.env, self.cmd, self.timeout