Add five-task Jev comparison reports

reports/benchmark-jev-coreml.json

@@ -0,0 +1,103 @@
+{
+  "chip" : "Apple M5 Pro",
+  "elapsed_s" : 2.7073659896850586,
+  "latency_ms" : {
+    "p50" : 4.4866669999999997,
+    "p95" : 9.6365829999999999
+  },
+  "lengths" : [
+    128,
+    256,
+    512,
+    1024
+  ],
+  "load_s" : 2.0176939964294434,
+  "questions" : 500,
+  "suites" : {
+    "emotion" : {
+      "accuracy" : 0.57999999999999996,
+      "buckets" : {
+        "L128" : 100
+      },
+      "dropped" : 0,
+      "latency_ms" : {
+        "p50" : 4.0849580000000003,
+        "p95" : 4.5919999999999996
+      },
+      "max_probability_delta_vs_reference" : 0.014793992042541504,
+      "n" : 100,
+      "reference_argmax_agreement" : 1,
+      "reference_compared" : 100,
+      "state_truncated" : 0
+    },
+    "news_topic" : {
+      "accuracy" : 0.96999999999999997,
+      "buckets" : {
+        "L128" : 86,
+        "L256" : 14
+      },
+      "dropped" : 0,
+      "latency_ms" : {
+        "p50" : 5.5224159999999998,
+        "p95" : 9.2874169999999996
+      },
+      "max_probability_delta_vs_reference" : 0.010490596294403076,
+      "n" : 100,
+      "reference_argmax_agreement" : 1,
+      "reference_compared" : 100,
+      "state_truncated" : 0
+    },
+    "prompt_injection" : {
+      "accuracy" : 0.65000000000000002,
+      "buckets" : {
+        "L128" : 96,
+        "L256" : 4
+      },
+      "dropped" : 0,
+      "latency_ms" : {
+        "p50" : 4.3983749999999997,
+        "p95" : 6.5922919999999996
+      },
+      "max_probability_delta_vs_reference" : 0.02795100212097168,
+      "n" : 100,
+      "reference_argmax_agreement" : 0.98999999999999999,
+      "reference_compared" : 100,
+      "state_truncated" : 0
+    },
+    "review_stars" : {
+      "accuracy" : 0.34999999999999998,
+      "buckets" : {
+        "L128" : 28,
+        "L256" : 35,
+        "L512" : 26,
+        "L1024" : 11
+      },
+      "dropped" : 0,
+      "latency_ms" : {
+        "p50" : 5.9012919999999998,
+        "p95" : 18.642083
+      },
+      "max_probability_delta_vs_reference" : 0.0017490386962890625,
+      "n" : 100,
+      "reference_argmax_agreement" : 1,
+      "reference_compared" : 100,
+      "state_truncated" : 0
+    },
+    "sms_spam" : {
+      "accuracy" : 0.57999999999999996,
+      "buckets" : {
+        "L128" : 100
+      },
+      "dropped" : 0,
+      "latency_ms" : {
+        "p50" : 4.1685420000000004,
+        "p95" : 5.0857080000000003
+      },
+      "max_probability_delta_vs_reference" : 0.020002603530883789,
+      "n" : 100,
+      "reference_argmax_agreement" : 1,
+      "reference_compared" : 100,
+      "state_truncated" : 0
+    }
+  }
+}

reports/benchmark-jev-reference.json

@@ -0,0 +1,100 @@
+{
+  "protocol": "Reproduce the five-task \"Laya vs Jev, measured\" table (brainfunctioncollapse.com/laya) on device.\n\nThe post gives task names, 100 labelled examples per task, laya 0.3.4 English checkpoint on an M1 Max\nGPU, and Jev's accuracy, but not its datasets, sampling or question wording. This script uses the\nobvious public dataset for each task, the first 100 rows of its test split (train for SMS spam, which\nhas no test split), and laya's own question wording where one exists. Jev is a closed API and is not\nmeasured here; its column is copied from the post. Both laya checkpoints are scored with the\nunmodified PyTorch runtime so the English numbers can be compared with the post and the\nmultilingual rows give the reference for the Core ML buckets (`FluidUseLaya benchmark`).\n\nOutputs:\n  benchmark/jev-suites.jsonl           the 500 questions with serialized states and gold labels\n  benchmark/jev-reference-rows.jsonl   multilingual PyTorch answers per row\n  reports/benchmark-jev-reference.json accuracy per task for both checkpoints",
+  "n_per_task": 100,
+  "post": {
+    "news_topic": {
+      "laya_english_m1max": 0.93,
+      "jev": 0.92
+    },
+    "sms_spam": {
+      "laya_english_m1max": 0.96,
+      "jev": 0.96
+    },
+    "emotion": {
+      "laya_english_m1max": 0.45,
+      "jev": 0.53
+    },
+    "review_stars": {
+      "laya_english_m1max": 0.35,
+      "jev": 0.7
+    },
+    "prompt_injection": {
+      "laya_english_m1max": 0.65,
+      "jev": 0.71
+    },
+    "all": {
+      "laya_english_m1max": 0.668,
+      "jev": 0.764
+    }
+  },
+  "checkpoints": {
+    "multilingual": {
+      "max_len": 1024,
+      "seconds": 21.3,
+      "ms_per_question": 42.55,
+      "emotion": {
+        "n": 100,
+        "accuracy": 0.58,
+        "max_tokens": 82
+      },
+      "news_topic": {
+        "n": 100,
+        "accuracy": 0.97,
+        "max_tokens": 226
+      },
+      "prompt_injection": {
+        "n": 100,
+        "accuracy": 0.64,
+        "max_tokens": 177
+      },
+      "review_stars": {
+        "n": 100,
+        "accuracy": 0.35,
+        "max_tokens": 782
+      },
+      "sms_spam": {
+        "n": 100,
+        "accuracy": 0.58,
+        "max_tokens": 118
+      },
+      "all": {
+        "n": 500,
+        "accuracy": 0.624
+      }
+    },
+    "english": {
+      "max_len": 512,
+      "seconds": 48.9,
+      "ms_per_question": 97.87,
+      "emotion": {
+        "n": 100,
+        "accuracy": 0.65,
+        "max_tokens": 83
+      },
+      "news_topic": {
+        "n": 100,
+        "accuracy": 0.95,
+        "max_tokens": 232
+      },
+      "prompt_injection": {
+        "n": 100,
+        "accuracy": 0.79,
+        "max_tokens": 252
+      },
+      "review_stars": {
+        "n": 100,
+        "accuracy": 0.39,
+        "max_tokens": 512
+      },
+      "sms_spam": {
+        "n": 100,
+        "accuracy": 0.88,
+        "max_tokens": 116
+      },
+      "all": {
+        "n": 500,
+        "accuracy": 0.732
+      }
+    }
+  }
+}