benchmarks/aggregate_results.py

#!/usr/bin/env python3
import argparse
import csv
import json
from pathlib import Path

p = argparse.ArgumentParser()
p.add_argument("results_dir", nargs="?", default="results")
p.add_argument("--out", default="results/leaderboard.csv")
p.add_argument(
    "--manifest",
    default=None,
    help="Optional text file listing result JSON filenames to include.",
)
a = p.parse_args()

rows = []
results_dir = Path(a.results_dir)

if a.manifest:
    manifest = Path(a.manifest)
    names = [
        line.strip()
        for line in manifest.read_text().splitlines()
        if line.strip() and not line.lstrip().startswith("#")
    ]
    result_files = [results_dir / name for name in names]
else:
    result_files = sorted(results_dir.glob("*.json"))

for f in result_files:
    try:
        x = json.loads(f.read_text())
        c = x.get("config", {})
        s = x.get("system", {})

        rows.append({
            "file": f.name,
            "candidate": x.get("candidate"),
            "seed": c.get("seed"),
            "candidate_seed": c.get("candidate_seed"),
            "n_slices": c.get("n_slices"),
            "n_experts": c.get("n_experts"),
            "rank": c.get("rank"),
            "m_coarse": c.get("m_coarse"),
            "m_focused": c.get("m_focused"),
            "items_per_row": c.get("items_per_row"),
            "bootstrap_reps": c.get("bootstrap_reps"),
            "lambda_l1": c.get("lambda_l1"),
            "lambda_group": c.get("lambda_group"),
            "gpu_count": s.get("gpu_count"),
            "gpus": "; ".join(s.get("gpus", []) or []),
            "wall_seconds": x.get("wall_seconds"),
            "pilot_residual": x.get("pilot_residual"),
            "dense_fallback": x.get("dense_fallback"),
            "gate_decision": x.get("gate_decision"),
            "support_f1": x.get("support_f1"),
            "normalized_delta_error": x.get("normalized_delta_error"),
            "regression_recall": x.get("regression_recall"),
            "items_probe": x.get("items_probe"),
            "items_sense": x.get("items_sense"),
            "items_anchor": x.get("items_anchor"),
            "true_experts": ";".join(
                map(str, x.get("true_experts", []) or [])
            ),
            "nominated_experts": ";".join(
                map(str, x.get("nominated_experts", []) or [])
            ),
            "recovered_experts": ";".join(
                map(str, x.get("recovered_experts", []) or [])
            ),
            "bootstrap_frequencies": ";".join(
                map(str, x.get("bootstrap_frequencies", []) or [])
            ),
        })
    except Exception as e:
        print(f"warning: skipped {f}: {e}")

Path(a.out).parent.mkdir(parents=True, exist_ok=True)

if rows:
    with open(a.out, "w", newline="") as h:
        w = csv.DictWriter(h, fieldnames=rows[0].keys())
        w.writeheader()
        w.writerows(rows)

print(f"wrote {len(rows)} rows to {a.out}")