Add Pre-LayerNorm 10M TinyStories - 5 epochs

metrics.json

@@ -0,0 +1,56 @@
+{
+  "norm_type": "Pre-LayerNorm",
+  "vocab_size": 8192,
+  "max_seq_len": 128,
+  "d_model": 176,
+  "n_heads": 8,
+  "ffn_dim": 704,
+  "num_blocks": 8,
+  "epochs": 5,
+  "train_losses": [
+    2.7801686714166847,
+    2.222144907208902,
+    2.069231568489224,
+    1.9887459733063775,
+    1.9367085697778228
+  ],
+  "val_losses": [
+    2.371724769221994,
+    2.1539471743307708,
+    2.060667110974099,
+    2.008517362957909,
+    1.9709118760985769
+  ],
+  "train_top1": [
+    0.43638668426173,
+    0.501614362949496,
+    0.5221109322154497,
+    0.5334409678465053,
+    0.5410586588701694
+  ],
+  "train_top5": [
+    0.6825656910253973,
+    0.7609837558461504,
+    0.7833337738495424,
+    0.7951137859557944,
+    0.802671929917193
+  ],
+  "val_top1": [
+    0.4821287964004499,
+    0.5104976060684722,
+    0.5232406045398171,
+    0.5306650370626749,
+    0.5368395056387182
+  ],
+  "val_top5": [
+    0.7390574976204898,
+    0.771331948891004,
+    0.7851424821897263,
+    0.7932223616278734,
+    0.7987990683856826
+  ],
+  "test_loss": 1.9709118760985769,
+  "test_top1": 0.5368395056387182,
+  "test_top5": 0.7987990683856826,
+  "total_params": 5444480
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34ef29031c83b677b5cbc24b0e95c0148ac4ac0cafec9d63c6bc5a1f28d7736f
+size 21879080