Spaces:
Running
Running
Land the post-remake refresh pieces held back by the quota
Browse filesCo-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
scripts/publish_atomic_results.py
CHANGED
|
@@ -295,6 +295,29 @@ def method_specs(root: Path, family: str, tasks: tuple[str, ...]) -> dict[str, t
|
|
| 295 |
return out
|
| 296 |
|
| 297 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 298 |
def dimensions_for_specs(specs: list[tuple[str, Path, Path, float]]) -> list[int]:
|
| 299 |
return [feature_count(read_json(metrics_path)) for _, _, metrics_path, _ in specs]
|
| 300 |
|
|
@@ -385,6 +408,31 @@ def make_rows(results_root: Path, sphragis_data: Path, metre_data: Path):
|
|
| 385 |
"kainobert-syntax", "kainobert-char-tfidf", "kainobert-tfidf-alm",
|
| 386 |
"kainobert-burrows-alm", "kainobert-burrows-char", "kainobert-char-alm",
|
| 387 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 388 |
method_main = method_specs(results_root, "sphragis-main", MAIN_TASKS)
|
| 389 |
method_source = method_specs(results_root, "sphragis-source-held-out", SOURCE_TASKS)
|
| 390 |
method_sphragis: dict[str, tuple] = {}
|
|
@@ -644,6 +692,7 @@ def make_rows(results_root: Path, sphragis_data: Path, metre_data: Path):
|
|
| 644 |
)
|
| 645 |
for pair in pairs
|
| 646 |
),
|
|
|
|
| 647 |
*(
|
| 648 |
build_row(
|
| 649 |
model=pair_model_name(combo),
|
|
@@ -759,6 +808,7 @@ def make_rows(results_root: Path, sphragis_data: Path, metre_data: Path):
|
|
| 759 |
)
|
| 760 |
for pair in metre_pairs if pair in metre_pair_data
|
| 761 |
),
|
|
|
|
| 762 |
*(
|
| 763 |
build_row(
|
| 764 |
model=pair_model_name(combo),
|
|
|
|
| 295 |
return out
|
| 296 |
|
| 297 |
|
| 298 |
+
def alm_specs(root: Path, family: str, tasks: tuple[str, ...]):
|
| 299 |
+
"""Specs for the remade authorial ensemble, or None until it exists.
|
| 300 |
+
|
| 301 |
+
``scripts/alm_attribution_rows.py`` writes one predictions/metrics pair per
|
| 302 |
+
task from the remade models' scores. The directory is absent until the
|
| 303 |
+
remake has landed, and a leaderboard build before then simply carries no
|
| 304 |
+
ensemble row rather than a stale one.
|
| 305 |
+
"""
|
| 306 |
+
family_root = root.parent / "alm-remake-20260902" / "leaderboard" / family
|
| 307 |
+
if not family_root.is_dir():
|
| 308 |
+
return None
|
| 309 |
+
specs, models = [], set()
|
| 310 |
+
for task in tasks:
|
| 311 |
+
metrics_path = family_root / task / "metrics.json"
|
| 312 |
+
if not metrics_path.is_file():
|
| 313 |
+
return None
|
| 314 |
+
metrics = read_json(metrics_path)
|
| 315 |
+
models.update(metrics["models"])
|
| 316 |
+
specs.append((task, family_root / task / "predictions.jsonl", metrics_path,
|
| 317 |
+
float(metrics["test_metrics"]["macro_f1"])))
|
| 318 |
+
return specs, len(models)
|
| 319 |
+
|
| 320 |
+
|
| 321 |
def dimensions_for_specs(specs: list[tuple[str, Path, Path, float]]) -> list[int]:
|
| 322 |
return [feature_count(read_json(metrics_path)) for _, _, metrics_path, _ in specs]
|
| 323 |
|
|
|
|
| 408 |
"kainobert-syntax", "kainobert-char-tfidf", "kainobert-tfidf-alm",
|
| 409 |
"kainobert-burrows-alm", "kainobert-burrows-char", "kainobert-char-alm",
|
| 410 |
)
|
| 411 |
+
alm_main = alm_specs(results_root, "sphragis-main", MAIN_TASKS)
|
| 412 |
+
alm_source = alm_specs(results_root, "sphragis-source-held-out", SOURCE_TASKS)
|
| 413 |
+
alm_metre = alm_specs(results_root, "sphragis-metre", VERSE_TASKS)
|
| 414 |
+
alm_status = (
|
| 415 |
+
"One OLMo-1B model further pretrained per author on the current release with "
|
| 416 |
+
"validation-loss early stopping; attribution is the lowest mean per-token surprisal; "
|
| 417 |
+
"no base-model or epoch search; test evaluated once"
|
| 418 |
+
)
|
| 419 |
+
alm_rows = []
|
| 420 |
+
if alm_main and alm_source:
|
| 421 |
+
alm_rows.append(build_row(
|
| 422 |
+
model="OLMo-1B authorial language models",
|
| 423 |
+
url="https://github.com/Urdatorn/sphragis_models",
|
| 424 |
+
dimensionality=f"1.18B parameters per author; {alm_main[1]} and {alm_source[1]} models",
|
| 425 |
+
classifier="Lowest mean per-token surprisal", status=alm_status,
|
| 426 |
+
task_specs=alm_main[0] + alm_source[0], dataset_roots=roots))
|
| 427 |
+
alm_metre_rows = []
|
| 428 |
+
if alm_metre:
|
| 429 |
+
alm_metre_rows.append(build_row(
|
| 430 |
+
model="OLMo-1B authorial language models",
|
| 431 |
+
url="https://github.com/Urdatorn/sphragis_models",
|
| 432 |
+
dimensionality=f"1.18B parameters per author; {alm_metre[1]} models",
|
| 433 |
+
classifier="Lowest mean per-token surprisal", status=alm_status,
|
| 434 |
+
task_specs=alm_metre[0], dataset_roots=roots))
|
| 435 |
+
|
| 436 |
method_main = method_specs(results_root, "sphragis-main", MAIN_TASKS)
|
| 437 |
method_source = method_specs(results_root, "sphragis-source-held-out", SOURCE_TASKS)
|
| 438 |
method_sphragis: dict[str, tuple] = {}
|
|
|
|
| 692 |
)
|
| 693 |
for pair in pairs
|
| 694 |
),
|
| 695 |
+
*alm_rows,
|
| 696 |
*(
|
| 697 |
build_row(
|
| 698 |
model=pair_model_name(combo),
|
|
|
|
| 808 |
)
|
| 809 |
for pair in metre_pairs if pair in metre_pair_data
|
| 810 |
),
|
| 811 |
+
*alm_metre_rows,
|
| 812 |
*(
|
| 813 |
build_row(
|
| 814 |
model=pair_model_name(combo),
|