{
 "format": "bf16",
 "temperature": 0.8278650620942867,
 "overall": {
  "n": 1000,
  "top1_agreement": 0.997,
  "top1_flips": 3,
  "max_abs_dscore": 0.13961565494537354,
  "mean_abs_dscore": 0.012822698334419146,
  "mean_kl": 7.013363207503902e-05,
  "n_gold": 1000,
  "nll_ref": 0.4436679457518203,
  "nll_cand": 0.4437944677940992,
  "dnll": 0.0001265220422789204,
  "mean_abs_row_dnll": 0.005641898326944814,
  "acc_ref": 0.808,
  "acc_cand": 0.805,
  "acc_drop_pp": 0.30000000000000027
 },
 "per_category": {
  "di_knowledge": {
   "n": 93,
   "top1_agreement": 1.0,
   "top1_flips": 0,
   "max_abs_dscore": 0.07215514779090881,
   "mean_abs_dscore": 0.014889913475799058,
   "mean_kl": 0.00013722336018850683,
   "n_gold": 93,
   "nll_ref": 0.7907936240501159,
   "nll_cand": 0.7927041608827371,
   "dnll": 0.0019105368326212124,
   "mean_abs_row_dnll": 0.01039648465267476,
   "acc_ref": 0.6559139784946236,
   "acc_cand": 0.6559139784946236,
   "acc_drop_pp": 0.0
  },
  "di_language": {
   "n": 93,
   "top1_agreement": 1.0,
   "top1_flips": 0,
   "max_abs_dscore": 0.0584871768951416,
   "mean_abs_dscore": 0.013199496934170362,
   "mean_kl": 4.376886766709004e-05,
   "n_gold": 93,
   "nll_ref": 0.43828192127009435,
   "nll_cand": 0.43613660599650544,
   "dnll": -0.0021453152735889103,
   "mean_abs_row_dnll": 0.005769073926221885,
   "acc_ref": 0.8494623655913979,
   "acc_cand": 0.8494623655913979,
   "acc_drop_pp": 0.0
  },
  "di_retrieval": {
   "n": 93,
   "top1_agreement": 1.0,
   "top1_flips": 0,
   "max_abs_dscore": 0.13961565494537354,
   "mean_abs_dscore": 0.011278054842327665,
   "mean_kl": 6.320183776936612e-05,
   "n_gold": 93,
   "nll_ref": 0.07565367364179643,
   "nll_cand": 0.07529180302084762,
   "dnll": -0.0003618706209488065,
   "mean_abs_row_dnll": 0.0018865181948961214,
   "acc_ref": 0.978494623655914,
   "acc_cand": 0.978494623655914,
   "acc_drop_pp": 0.0
  },
  "format": {
   "n": 75,
   "top1_agreement": 1.0,
   "top1_flips": 0,
   "max_abs_dscore": 0.07049310207366943,
   "mean_abs_dscore": 0.012186950511526283,
   "mean_kl": 1.866440251394625e-05,
   "n_gold": 75,
   "nll_ref": 0.09087868406020969,
   "nll_cand": 0.09027726831006917,
   "dnll": -0.0006014157501405132,
   "mean_abs_row_dnll": 0.0017343322558420642,
   "acc_ref": 0.9733333333333334,
   "acc_cand": 0.9733333333333334,
   "acc_drop_pp": 0.0
  },
  "general": {
   "n": 93,
   "top1_agreement": 1.0,
   "top1_flips": 0,
   "max_abs_dscore": 0.053872108459472656,
   "mean_abs_dscore": 0.011199172480311083,
   "mean_kl": 8.964417186979084e-06,
   "n_gold": 93,
   "nll_ref": 0.13061020512422436,
   "nll_cand": 0.12994049456566417,
   "dnll": -0.0006697105585601881,
   "mean_abs_row_dnll": 0.0015611860129022736,
   "acc_ref": 0.967741935483871,
   "acc_cand": 0.967741935483871,
   "acc_drop_pp": 0.0
  },
  "hard_short": {
   "n": 93,
   "top1_agreement": 0.989247311827957,
   "top1_flips": 1,
   "max_abs_dscore": 0.06802543997764587,
   "mean_abs_dscore": 0.014637837174057167,
   "mean_kl": 0.00013160361749015227,
   "n_gold": 93,
   "nll_ref": 0.7835153258109905,
   "nll_cand": 0.782783474188176,
   "dnll": -0.0007318516228145278,
   "mean_abs_row_dnll": 0.01093632719811279,
   "acc_ref": 0.6021505376344086,
   "acc_cand": 0.5913978494623656,
   "acc_drop_pp": 1.0752688172043001
  },
  "long": {
   "n": 92,
   "top1_agreement": 1.0,
   "top1_flips": 0,
   "max_abs_dscore": 0.1342533826828003,
   "mean_abs_dscore": 0.013823732117665294,
   "mean_kl": 6.168921301697641e-05,
   "n_gold": 92,
   "nll_ref": 0.10827285500173092,
   "nll_cand": 0.10768749311257943,
   "dnll": -0.000585361889151495,
   "mean_abs_row_dnll": 0.002564256755867923,
   "acc_ref": 0.967391304347826,
   "acc_cand": 0.967391304347826,
   "acc_drop_pp": 0.0
  },
  "mac": {
   "n": 92,
   "top1_agreement": 1.0,
   "top1_flips": 0,
   "max_abs_dscore": 0.04618799686431885,
   "mean_abs_dscore": 0.01143809092109618,
   "mean_kl": 2.6581458539860547e-05,
   "n_gold": 92,
   "nll_ref": 0.3262944821095264,
   "nll_cand": 0.3269180887247012,
   "dnll": 0.0006236066151747988,
   "mean_abs_row_dnll": 0.003772370527716574,
   "acc_ref": 0.8369565217391305,
   "acc_cand": 0.8369565217391305,
   "acc_drop_pp": 0.0
  },
  "public_ingests": {
   "n": 92,
   "top1_agreement": 1.0,
   "top1_flips": 0,
   "max_abs_dscore": 0.10585364699363708,
   "mean_abs_dscore": 0.015097456673118565,
   "mean_kl": 0.00015103258358910773,
   "n_gold": 92,
   "nll_ref": 0.7667467143475788,
   "nll_cand": 0.7682457652594957,
   "dnll": 0.0014990509119169326,
   "mean_abs_row_dnll": 0.010491535553020102,
   "acc_ref": 0.6521739130434783,
   "acc_cand": 0.6521739130434783,
   "acc_drop_pp": 0.0
  },
  "themes": {
   "n": 92,
   "top1_agreement": 0.9891304347826086,
   "top1_flips": 1,
   "max_abs_dscore": 0.04355478286743164,
   "mean_abs_dscore": 0.010605669458923132,
   "mean_kl": 2.4907957260022938e-05,
   "n_gold": 92,
   "nll_ref": 0.27025191463643466,
   "nll_cand": 0.2710970978349834,
   "dnll": 0.0008451831985487601,
   "mean_abs_row_dnll": 0.0033373064356656897,
   "acc_ref": 0.8695652173913043,
   "acc_cand": 0.8586956521739131,
   "acc_drop_pp": 1.0869565217391242
  },
  "typed_synthetic": {
   "n": 92,
   "top1_agreement": 0.9891304347826086,
   "top1_flips": 1,
   "max_abs_dscore": 0.04864954948425293,
   "mean_abs_dscore": 0.01256397343500986,
   "mean_kl": 9.395103279401365e-05,
   "n_gold": 92,
   "nll_ref": 1.0338530850662833,
   "nll_cand": 1.0353560613294195,
   "dnll": 0.0015029762631362242,
   "mean_abs_row_dnll": 0.008864003979572451,
   "acc_ref": 0.5652173913043478,
   "acc_cand": 0.5543478260869565,
   "acc_drop_pp": 1.0869565217391242
  }
 },
 "gates": {
  "coverage_finite_tokens": {
   "pass": true,
   "problems": {},
   "reference_missing_requests": 0
  },
  "top1": {
   "threshold": 0.99,
   "value": 0.997,
   "pass": true
  },
  "dnll": {
   "threshold": 0.02,
   "value": 0.0001265220422789204,
   "pass": true
  }
 },
 "verdict": "PASS",
 "resolution_pp_per_flip": 0.1,
 "note": "small fixtures cannot resolve 0.3pp accuracy gates; accuracy on a parity fixture is only indicative (synthetic long rows carry the base row's gold label).",
 "ref": "runs/macjev/runtime_v2_dev/reference_trained/gate_ref_block_fp32.jsonl",
 "cand": "runs/macjev/release_2b/parity/pred_mlx_bf16_gate.jsonl",
 "fixture": "runs/macjev/runtime_v2_dev/reference/gate_fixture_n1000.jsonl"
}