IQ.Pilot Release Commit @ f2a861c

2026-09-02 15:07:09 -05:00
parent b42569dbca
commit e8748fd704
5497 changed files with 316070 additions and 179848 deletions
--- a/artifacts/package_sources/tinygrad/extra/datasets/preprocess_imagenet.py
+++ b/artifacts/package_sources/tinygrad/extra/datasets/preprocess_imagenet.py
@@ -0,0 +1,21 @@
+from tinygrad import Tensor, dtypes
+from extra.datasets.imagenet import iterate, get_val_files
+
+if __name__ == "__main__":
+  #sz = len(get_val_files())
+  sz = 32*100
+  X,Y = None, None
+
+  idx = 0
+  for x,y in iterate(shuffle=False):
+    print(x.shape, y.shape, x.dtype, y.dtype)
+    assert x.shape[0] == y.shape[0]
+    bs = x.shape[0]
+    if X is None:
+      X = Tensor.empty(sz, *x.shape[1:], device="disk:/tmp/imagenet_x", dtype=dtypes.uint8)
+      Y = Tensor.empty(sz, *y.shape[1:], device="disk:/tmp/imagenet_y", dtype=dtypes.int64)
+      print(X.shape, Y.shape)
+    X[idx:idx+bs].assign(x)
+    Y[idx:idx+bs].assign(y)
+    idx += bs
+    if idx >= sz: break