Cubic Hier 150M: weights, tokenizer and benchmark comparison

README.md

@@ -40,14 +40,24 @@ informative below roughly 1B parameters.
 
 ## Results
 
-| Model | Params | hellaswag | arc_easy | arc_challenge | winogrande | openbookqa | sciq | boolq | lambada_openai | mmlu | Avg |
-|---|---|---|---|---|---|---|---|---|---|---|---|
-| CubicHierLM-157M (ours) | 157M | 30.1 | 39.1 | 25.3 | 50.8 | 25.2 | 70.4 | 61.8 | 20.1 | 27.1 | 38.9 |
-| _random chance_ | — | 25.0 | 25.0 | 25.0 | 50.0 | 25.0 | 25.0 | 50.0 | 0.0 | 25.0 | — |
+| Model | Params | hellaswag | arc_easy | arc_challenge | piqa | winogrande | openbookqa | sciq | boolq | lambada_openai | mmlu | Avg | Δ chance |
+|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
+| CubicHierLM-157M (ours) | 157M | 30.1 | 39.1 | 25.3 | 57.8 | 50.8 | 25.2 | 70.4 | 61.8 | 20.1 | 27.1 | 40.8 | +10.8 |
+| _random chance_ | — | 25.0 | 25.0 | 25.0 | 50.0 | 50.0 | 25.0 | 25.0 | 50.0 | 0.0 | 25.0 | — | +0.0 |
 
 Headline metric per task: `acc_norm` for multiple choice with unequal-length
 options, `acc` otherwise. Evaluated on an evenly strided subsample of up to 1500 examples per task.
 
+**Read `Δ chance`, not `Avg`.** The tasks have different random-guess floors —
+0% for LAMBADA, 25% for the four-way questions, 50% for the binary ones — so a
+plain average across them is not a meaningful quantity. `Δ chance` is the mean
+margin over the random baseline and is the column that actually compares.
+
+Two further caveats. BoolQ's validation set is about 62% "yes", so any score
+near 62 means the model is answering yes to everything and carries no signal.
+ARC-Challenge, OpenBookQA, WinoGrande and MMLU sit at chance for every model
+of this size and should not be read as differences.
+
 ## Architecture
 
 - 16 layers, dim 800, 10 heads, SwiGLU 2176, 32k byte-level BPE with digit splitting
@@ -81,4 +91,4 @@ It will confabulate. Do not use it for anything factual, safety-relevant or
 mathematical without independent verification. Bilingual EN/RU by design, but
 Russian coverage is 13% of pretraining and correspondingly weaker.
 
-Raw metrics: hellaswag=30.1, arc_easy=39.1, arc_challenge=25.3, winogrande=50.8, openbookqa=25.2, sciq=70.4, boolq=61.8, lambada_openai=20.1, mmlu=27.1
+Raw metrics: hellaswag=30.1, arc_easy=39.1, arc_challenge=25.3, piqa=57.8, winogrande=50.8, openbookqa=25.2, sciq=70.4, boolq=61.8, lambada_openai=20.1, mmlu=27.1

benchmark_results.json

@@ -1,5 +1,5 @@
 {
-  "generated_at": "2026-07-26T13:15:27+0000",
+  "generated_at": "2026-07-26T13:20:20+0000",
   "checkpoint": {
     "checkpoint": "/marimo/cubic_hier_150m_gemma4/checkpoints/final.pt",
     "phase_index": 4,
@@ -23,7 +23,7 @@
     "CubicHierLM-157M (ours)": {
       "hellaswag": {
         "examples": 1500,
-        "seconds": 9.8,
+        "seconds": 9.7,
         "acc": 26.8,
         "acc_norm": 30.066666666666666,
         "headline_metric": "acc_norm",
@@ -31,7 +31,7 @@
       },
       "arc_easy": {
         "examples": 1500,
-        "seconds": 9.2,
+        "seconds": 9.1,
         "acc": 41.2,
         "acc_norm": 39.06666666666667,
         "headline_metric": "acc_norm",
@@ -39,15 +39,23 @@
       },
       "arc_challenge": {
         "examples": 1172,
-        "seconds": 6.8,
+        "seconds": 7.2,
         "acc": 20.90443686006826,
         "acc_norm": 25.255972696245735,
         "headline_metric": "acc_norm",
         "headline": 25.255972696245735
       },
+      "piqa": {
+        "examples": 1500,
+        "seconds": 4.4,
+        "acc": 59.266666666666666,
+        "acc_norm": 57.8,
+        "headline_metric": "acc_norm",
+        "headline": 57.8
+      },
       "winogrande": {
         "examples": 1267,
-        "seconds": 3.7,
+        "seconds": 3.8,
         "acc": 50.828729281767956,
         "acc_norm": 50.828729281767956,
         "headline_metric": "acc",
@@ -55,7 +63,7 @@
       },
       "openbookqa": {
         "examples": 500,
-        "seconds": 2.7,
+        "seconds": 2.8,
         "acc": 15.8,
         "acc_norm": 25.2,
         "headline_metric": "acc_norm",
@@ -63,7 +71,7 @@
       },
       "sciq": {
         "examples": 1000,
-        "seconds": 7.2,
+        "seconds": 7.3,
         "acc": 73.3,
         "acc_norm": 70.4,
         "headline_metric": "acc_norm",
@@ -79,7 +87,7 @@
       },
       "lambada_openai": {
         "examples": 1500,
-        "seconds": 2.5,
+        "seconds": 2.4,
         "acc": 20.066666666666666,
         "acc_norm": 20.066666666666666,
         "headline_metric": "acc",
@@ -87,13 +95,14 @@
       },
       "mmlu": {
         "examples": 1500,
-        "seconds": 10.5,
+        "seconds": 10.3,
         "acc": 27.066666666666666,
         "acc_norm": 27.066666666666666,
         "headline_metric": "acc",
         "headline": 27.066666666666666
       },
-      "_average": 38.86
+      "_average": 40.76,
+      "_average_above_chance": 10.76
     }
   }
 }

benchmark_table.md

@@ -1,4 +1,4 @@
-| Model | Params | hellaswag | arc_easy | arc_challenge | winogrande | openbookqa | sciq | boolq | lambada_openai | mmlu | Avg |
-|---|---|---|---|---|---|---|---|---|---|---|---|
-| CubicHierLM-157M (ours) | 157M | 30.1 | 39.1 | 25.3 | 50.8 | 25.2 | 70.4 | 61.8 | 20.1 | 27.1 | 38.9 |
-| _random chance_ | — | 25.0 | 25.0 | 25.0 | 50.0 | 25.0 | 25.0 | 50.0 | 0.0 | 25.0 | — |
+| Model | Params | hellaswag | arc_easy | arc_challenge | piqa | winogrande | openbookqa | sciq | boolq | lambada_openai | mmlu | Avg | Δ chance |
+|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
+| CubicHierLM-157M (ours) | 157M | 30.1 | 39.1 | 25.3 | 57.8 | 50.8 | 25.2 | 70.4 | 61.8 | 20.1 | 27.1 | 40.8 | +10.8 |
+| _random chance_ | — | 25.0 | 25.0 | 25.0 | 50.0 | 50.0 | 25.0 | 25.0 | 50.0 | 0.0 | 25.0 | — | +0.0 |