Urdatorn Claude Fable 5.1 commited on
Commit
a9c1cfe
·
1 Parent(s): 1e15eec

Land the post-remake refresh pieces held back by the quota

Browse files

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Files changed (1) hide show
  1. scripts/publish_atomic_results.py +50 -0
scripts/publish_atomic_results.py CHANGED
@@ -295,6 +295,29 @@ def method_specs(root: Path, family: str, tasks: tuple[str, ...]) -> dict[str, t
295
  return out
296
 
297
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
298
  def dimensions_for_specs(specs: list[tuple[str, Path, Path, float]]) -> list[int]:
299
  return [feature_count(read_json(metrics_path)) for _, _, metrics_path, _ in specs]
300
 
@@ -385,6 +408,31 @@ def make_rows(results_root: Path, sphragis_data: Path, metre_data: Path):
385
  "kainobert-syntax", "kainobert-char-tfidf", "kainobert-tfidf-alm",
386
  "kainobert-burrows-alm", "kainobert-burrows-char", "kainobert-char-alm",
387
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
388
  method_main = method_specs(results_root, "sphragis-main", MAIN_TASKS)
389
  method_source = method_specs(results_root, "sphragis-source-held-out", SOURCE_TASKS)
390
  method_sphragis: dict[str, tuple] = {}
@@ -644,6 +692,7 @@ def make_rows(results_root: Path, sphragis_data: Path, metre_data: Path):
644
  )
645
  for pair in pairs
646
  ),
 
647
  *(
648
  build_row(
649
  model=pair_model_name(combo),
@@ -759,6 +808,7 @@ def make_rows(results_root: Path, sphragis_data: Path, metre_data: Path):
759
  )
760
  for pair in metre_pairs if pair in metre_pair_data
761
  ),
 
762
  *(
763
  build_row(
764
  model=pair_model_name(combo),
 
295
  return out
296
 
297
 
298
+ def alm_specs(root: Path, family: str, tasks: tuple[str, ...]):
299
+ """Specs for the remade authorial ensemble, or None until it exists.
300
+
301
+ ``scripts/alm_attribution_rows.py`` writes one predictions/metrics pair per
302
+ task from the remade models' scores. The directory is absent until the
303
+ remake has landed, and a leaderboard build before then simply carries no
304
+ ensemble row rather than a stale one.
305
+ """
306
+ family_root = root.parent / "alm-remake-20260902" / "leaderboard" / family
307
+ if not family_root.is_dir():
308
+ return None
309
+ specs, models = [], set()
310
+ for task in tasks:
311
+ metrics_path = family_root / task / "metrics.json"
312
+ if not metrics_path.is_file():
313
+ return None
314
+ metrics = read_json(metrics_path)
315
+ models.update(metrics["models"])
316
+ specs.append((task, family_root / task / "predictions.jsonl", metrics_path,
317
+ float(metrics["test_metrics"]["macro_f1"])))
318
+ return specs, len(models)
319
+
320
+
321
  def dimensions_for_specs(specs: list[tuple[str, Path, Path, float]]) -> list[int]:
322
  return [feature_count(read_json(metrics_path)) for _, _, metrics_path, _ in specs]
323
 
 
408
  "kainobert-syntax", "kainobert-char-tfidf", "kainobert-tfidf-alm",
409
  "kainobert-burrows-alm", "kainobert-burrows-char", "kainobert-char-alm",
410
  )
411
+ alm_main = alm_specs(results_root, "sphragis-main", MAIN_TASKS)
412
+ alm_source = alm_specs(results_root, "sphragis-source-held-out", SOURCE_TASKS)
413
+ alm_metre = alm_specs(results_root, "sphragis-metre", VERSE_TASKS)
414
+ alm_status = (
415
+ "One OLMo-1B model further pretrained per author on the current release with "
416
+ "validation-loss early stopping; attribution is the lowest mean per-token surprisal; "
417
+ "no base-model or epoch search; test evaluated once"
418
+ )
419
+ alm_rows = []
420
+ if alm_main and alm_source:
421
+ alm_rows.append(build_row(
422
+ model="OLMo-1B authorial language models",
423
+ url="https://github.com/Urdatorn/sphragis_models",
424
+ dimensionality=f"1.18B parameters per author; {alm_main[1]} and {alm_source[1]} models",
425
+ classifier="Lowest mean per-token surprisal", status=alm_status,
426
+ task_specs=alm_main[0] + alm_source[0], dataset_roots=roots))
427
+ alm_metre_rows = []
428
+ if alm_metre:
429
+ alm_metre_rows.append(build_row(
430
+ model="OLMo-1B authorial language models",
431
+ url="https://github.com/Urdatorn/sphragis_models",
432
+ dimensionality=f"1.18B parameters per author; {alm_metre[1]} models",
433
+ classifier="Lowest mean per-token surprisal", status=alm_status,
434
+ task_specs=alm_metre[0], dataset_roots=roots))
435
+
436
  method_main = method_specs(results_root, "sphragis-main", MAIN_TASKS)
437
  method_source = method_specs(results_root, "sphragis-source-held-out", SOURCE_TASKS)
438
  method_sphragis: dict[str, tuple] = {}
 
692
  )
693
  for pair in pairs
694
  ),
695
+ *alm_rows,
696
  *(
697
  build_row(
698
  model=pair_model_name(combo),
 
808
  )
809
  for pair in metre_pairs if pair in metre_pair_data
810
  ),
811
+ *alm_metre_rows,
812
  *(
813
  build_row(
814
  model=pair_model_name(combo),