{
 "chart": "calibration",
 "metric": "49-suite macro NLL / Brier / ECE, as deployed (lower is better)",
 "protocol_label": "v3: mixed (49 suites) / Laya: mixed (49 suites). Laya numbers: official checkpoints re-run by us on identical rows, shipped temperatures.",
 "rows": [
  {
   "panel": "NLL",
   "metric": "49-suite macro NLL, as deployed",
   "n_rows": 17416,
   "v3": 0.4928839178581892,
   "best_laya": 2.212565130608199,
   "best_laya_checkpoint": "multilingual",
   "all_laya_checkpoints": {
    "english": 9.669232280968933,
    "typed": 7.344998151307391,
    "multilingual": 2.212565130608199
   },
   "ratio_best_laya_over_v3": 4.48901871301224,
   "diff_v3_minus_best": -1.7196812127500098,
   "diff_ci95": [
    -1.7632826815264786,
    -1.6751892907984907
   ],
   "diff_ci_method": "case-cluster bootstrap within suites, 2000 resamples (scoreboard n_boot)",
   "source_path": "runs/macjev/report_r2/scoreboard/scoreboard.json",
   "fields": [
    "metrics['t4.macro_nll'].ours_value",
    "metrics['t4.macro_nll'].laya_best"
   ],
   "entries": [
    "t4.macro_nll.v3",
    "t4.macro_nll.laya_best"
   ],
   "claim": "t4.macro_nll_vs_best_laya"
  },
  {
   "panel": "Brier score",
   "metric": "49-suite macro Brier, as deployed",
   "n_rows": 17416,
   "v3": 0.23930092306086748,
   "best_laya": 0.7120664110846202,
   "best_laya_checkpoint": "multilingual",
   "all_laya_checkpoints": {
    "english": 1.0124090901595464,
    "typed": 0.9474950671291875,
    "multilingual": 0.7120664110846202
   },
   "ratio_best_laya_over_v3": 2.975610799894417,
   "diff_v3_minus_best": -0.47276548802375273,
   "diff_ci95": [
    -0.4843787573639837,
    -0.46074418926883626
   ],
   "diff_ci_method": "case-cluster bootstrap within suites, 2000 resamples (scoreboard n_boot)",
   "source_path": "runs/macjev/report_r2/scoreboard/scoreboard.json",
   "fields": [
    "metrics['t4.macro_brier'].ours_value",
    "metrics['t4.macro_brier'].laya_best"
   ],
   "entries": [
    "t4.macro_brier.v3",
    "t4.macro_brier.laya_best"
   ],
   "claim": "t4.macro_brier_vs_best_laya"
  },
  {
   "panel": "ECE",
   "metric": "49-suite macro ECE, as deployed",
   "n_rows": 17416,
   "v3": 0.05416919804670487,
   "best_laya": 0.2994167384382702,
   "best_laya_checkpoint": "multilingual",
   "all_laya_checkpoints": {
    "english": 0.4647782759946152,
    "typed": 0.40828649732151606,
    "multilingual": 0.2994167384382702
   },
   "ratio_best_laya_over_v3": 5.527435318132494,
   "diff_v3_minus_best": -0.24524754039156535,
   "diff_ci95": [
    -0.24448168599481393,
    -0.22893121774401398
   ],
   "diff_ci_method": "case-cluster bootstrap within suites, 2000 resamples (scoreboard n_boot)",
   "source_path": "runs/macjev/report_r2/scoreboard/scoreboard.json",
   "fields": [
    "metrics['t4.macro_ece_as_deployed'].ours_value",
    "metrics['t4.macro_ece_as_deployed'].laya_best"
   ],
   "entries": [
    "t4.macro_ece_as_deployed.v3",
    "t4.macro_ece_as_deployed.laya_best"
   ],
   "claim": "t4.macro_ece_as_deployed_vs_best_laya"
  }
 ],
 "footnote": "Macro average over 49 suites, 17,416 identical rows for both models. Protocol: mixed for both v3 and Laya (in-domain on some suites, held-out on others). As deployed: v3 with its supplied deployment temperatures; Laya numbers: official checkpoints re-run by us on identical rows with their shipped temperatures. Best Laya = best of the three official checkpoints (English, typed-decisions, multilingual) per metric; multilingual is best on all three. Paired case-cluster bootstrap 95% CIs (2,000 resamples) of every v3 minus Laya difference exclude zero. Plotted values: figures/calibration.data.json.",
 "not_plotted": "optional typed-decisions reliability diagram omitted (see agent report)"
}