| { |
| "format": "8bit", |
| "temperature": 0.8278650620942867, |
| "overall": { |
| "n": 43, |
| "top1_agreement": 1.0, |
| "top1_flips": 0, |
| "max_abs_dscore": 0.4653351306915283, |
| "mean_abs_dscore": 0.023077200647273945, |
| "mean_kl": 0.0006975644872166873, |
| "n_gold": 43, |
| "nll_ref": 0.670433307194272, |
| "nll_cand": 0.6868139442506376, |
| "dnll": 0.01638063705636561, |
| "mean_abs_row_dnll": 0.030139399526949388, |
| "acc_ref": 0.7209302325581395, |
| "acc_cand": 0.7209302325581395, |
| "acc_drop_pp": 0.0 |
| }, |
| "per_category": { |
| "catalogue_overflow": { |
| "n": 3, |
| "top1_agreement": 1.0, |
| "top1_flips": 0, |
| "max_abs_dscore": 0.2735496163368225, |
| "mean_abs_dscore": 0.019241230377298316, |
| "mean_kl": 0.0012829640019479873, |
| "n_gold": 3, |
| "nll_ref": 0.5023960827418966, |
| "nll_cand": 0.46349011750257935, |
| "dnll": -0.038905965239317275, |
| "mean_abs_row_dnll": 0.039379824994875905, |
| "acc_ref": 0.6666666666666666, |
| "acc_cand": 0.6666666666666666, |
| "acc_drop_pp": 0.0 |
| }, |
| "catalogue_overflow_long_state": { |
| "n": 1, |
| "top1_agreement": 1.0, |
| "top1_flips": 0, |
| "max_abs_dscore": 0.09829854965209961, |
| "mean_abs_dscore": 0.02608197888001701, |
| "mean_kl": 0.00037418306666026774, |
| "n_gold": 1, |
| "nll_ref": 1.868953922917252, |
| "nll_cand": 1.8305656122814864, |
| "dnll": -0.03838831063576564, |
| "mean_abs_row_dnll": 0.03838831063576564, |
| "acc_ref": 0.0, |
| "acc_cand": 0.0, |
| "acc_drop_pp": 0.0 |
| }, |
| "long_16k": { |
| "n": 3, |
| "top1_agreement": 1.0, |
| "top1_flips": 0, |
| "max_abs_dscore": 0.4653351306915283, |
| "mean_abs_dscore": 0.04898447410160343, |
| "mean_kl": 0.005652873428140343, |
| "n_gold": 3, |
| "nll_ref": 1.098996986024778, |
| "nll_cand": 1.3040987437659517, |
| "dnll": 0.20510175774117378, |
| "mean_abs_row_dnll": 0.2051017577411738, |
| "acc_ref": 0.6666666666666666, |
| "acc_cand": 0.6666666666666666, |
| "acc_drop_pp": 0.0 |
| }, |
| "long_24k": { |
| "n": 2, |
| "top1_agreement": 1.0, |
| "top1_flips": 0, |
| "max_abs_dscore": 0.068747878074646, |
| "mean_abs_dscore": 0.03896120687325796, |
| "mean_kl": 0.0007690578648421433, |
| "n_gold": 2, |
| "nll_ref": 0.43411292245601785, |
| "nll_cand": 0.43100727732191557, |
| "dnll": -0.003105645134102275, |
| "mean_abs_row_dnll": 0.019953016950989222, |
| "acc_ref": 1.0, |
| "acc_cand": 1.0, |
| "acc_drop_pp": 0.0 |
| }, |
| "many_options": { |
| "n": 2, |
| "top1_agreement": 1.0, |
| "top1_flips": 0, |
| "max_abs_dscore": 0.06005406379699707, |
| "mean_abs_dscore": 0.01609148249030113, |
| "mean_kl": 3.102768330017338e-06, |
| "n_gold": 2, |
| "nll_ref": 0.012124599411053285, |
| "nll_cand": 0.012000711496263293, |
| "dnll": -0.00012388791478999163, |
| "mean_abs_row_dnll": 0.00012388791478999076, |
| "acc_ref": 1.0, |
| "acc_cand": 1.0, |
| "acc_drop_pp": 0.0 |
| }, |
| "multi_block_3k": { |
| "n": 2, |
| "top1_agreement": 1.0, |
| "top1_flips": 0, |
| "max_abs_dscore": 0.037885069847106934, |
| "mean_abs_dscore": 0.015739992260932922, |
| "mean_kl": 0.00011035902286322221, |
| "n_gold": 2, |
| "nll_ref": 0.10264503770560551, |
| "nll_cand": 0.09882922575086513, |
| "dnll": -0.0038158119547403863, |
| "mean_abs_row_dnll": 0.00429795147361977, |
| "acc_ref": 1.0, |
| "acc_cand": 1.0, |
| "acc_drop_pp": 0.0 |
| }, |
| "multi_block_5k": { |
| "n": 2, |
| "top1_agreement": 1.0, |
| "top1_flips": 0, |
| "max_abs_dscore": 0.0428202748298645, |
| "mean_abs_dscore": 0.02182019054889679, |
| "mean_kl": 4.99818269133305e-05, |
| "n_gold": 2, |
| "nll_ref": 0.41414880456154163, |
| "nll_cand": 0.40767085022183286, |
| "dnll": -0.006477954339708769, |
| "mean_abs_row_dnll": 0.006477954339708783, |
| "acc_ref": 1.0, |
| "acc_cand": 1.0, |
| "acc_drop_pp": 0.0 |
| }, |
| "multi_block_9k": { |
| "n": 2, |
| "top1_agreement": 1.0, |
| "top1_flips": 0, |
| "max_abs_dscore": 0.02951335906982422, |
| "mean_abs_dscore": 0.02005617693066597, |
| "mean_kl": 6.540709706121381e-06, |
| "n_gold": 2, |
| "nll_ref": 1.387121106483142, |
| "nll_cand": 1.3792866299819007, |
| "dnll": -0.00783447650124125, |
| "mean_abs_row_dnll": 0.00783447650124136, |
| "acc_ref": 0.5, |
| "acc_cand": 0.5, |
| "acc_drop_pp": 0.0 |
| }, |
| "multi_question": { |
| "n": 10, |
| "top1_agreement": 1.0, |
| "top1_flips": 0, |
| "max_abs_dscore": 0.06343865394592285, |
| "mean_abs_dscore": 0.018984448437889417, |
| "mean_kl": 0.00014264548450480086, |
| "n_gold": 10, |
| "nll_ref": 0.7606667410227165, |
| "nll_cand": 0.7742217565907421, |
| "dnll": 0.01355501556802563, |
| "mean_abs_row_dnll": 0.016602018627922027, |
| "acc_ref": 0.7, |
| "acc_cand": 0.7, |
| "acc_drop_pp": 0.0 |
| }, |
| "prefix_boundary": { |
| "n": 7, |
| "top1_agreement": 1.0, |
| "top1_flips": 0, |
| "max_abs_dscore": 0.07407116889953613, |
| "mean_abs_dscore": 0.02506211300690969, |
| "mean_kl": 0.000275167227242767, |
| "n_gold": 7, |
| "nll_ref": 0.9445710499909146, |
| "nll_cand": 0.9592024225354618, |
| "dnll": 0.014631372544547272, |
| "mean_abs_row_dnll": 0.026803385065686018, |
| "acc_ref": 0.5714285714285714, |
| "acc_cand": 0.5714285714285714, |
| "acc_drop_pp": 0.0 |
| }, |
| "qtype_choice": { |
| "n": 1, |
| "top1_agreement": 1.0, |
| "top1_flips": 0, |
| "max_abs_dscore": 0.039629340171813965, |
| "mean_abs_dscore": 0.01868373155593872, |
| "mean_kl": 9.758721027128357e-05, |
| "n_gold": 1, |
| "nll_ref": 1.2471588298811915, |
| "nll_cand": 1.2644290223938441, |
| "dnll": 0.017270192512652605, |
| "mean_abs_row_dnll": 0.017270192512652605, |
| "acc_ref": 0.0, |
| "acc_cand": 0.0, |
| "acc_drop_pp": 0.0 |
| }, |
| "qtype_noul": { |
| "n": 1, |
| "top1_agreement": 1.0, |
| "top1_flips": 0, |
| "max_abs_dscore": 0.04642695188522339, |
| "mean_abs_dscore": 0.02744242548942566, |
| "mean_kl": 0.00046207429765575576, |
| "n_gold": 1, |
| "nll_ref": 0.3643240477162341, |
| "nll_cand": 0.3445434412562239, |
| "dnll": -0.01978060646001023, |
| "mean_abs_row_dnll": 0.01978060646001023, |
| "acc_ref": 1.0, |
| "acc_cand": 1.0, |
| "acc_drop_pp": 0.0 |
| }, |
| "qtype_score": { |
| "n": 1, |
| "top1_agreement": 1.0, |
| "top1_flips": 0, |
| "max_abs_dscore": 0.04161477088928223, |
| "mean_abs_dscore": 0.015557724237442016, |
| "mean_kl": 1.4547309373278143e-05, |
| "n_gold": 1, |
| "nll_ref": 0.9393375234945696, |
| "nll_cand": 0.9372245712212975, |
| "dnll": -0.0021129522732720174, |
| "mean_abs_row_dnll": 0.0021129522732720174, |
| "acc_ref": 0.0, |
| "acc_cand": 0.0, |
| "acc_drop_pp": 0.0 |
| }, |
| "short_sb": { |
| "n": 6, |
| "top1_agreement": 1.0, |
| "top1_flips": 0, |
| "max_abs_dscore": 0.12482678890228271, |
| "mean_abs_dscore": 0.01820988009964663, |
| "mean_kl": 0.0005014431591724873, |
| "n_gold": 6, |
| "nll_ref": 0.11428482960768895, |
| "nll_cand": 0.123207743102961, |
| "dnll": 0.00892291349527205, |
| "mean_abs_row_dnll": 0.008996485578208901, |
| "acc_ref": 1.0, |
| "acc_cand": 1.0, |
| "acc_drop_pp": 0.0 |
| } |
| }, |
| "gates": { |
| "coverage_finite_tokens": { |
| "pass": true, |
| "problems": {}, |
| "reference_missing_requests": 0 |
| }, |
| "top1": { |
| "threshold": 0.98, |
| "value": 1.0, |
| "pass": true |
| }, |
| "dnll": { |
| "threshold": 0.02, |
| "value": 0.01638063705636561, |
| "pass": true |
| }, |
| "acc_drop_pp": { |
| "threshold": 0.3, |
| "value": 0.0, |
| "resolution_pp": 2.3255813953488373, |
| "pass": null, |
| "note": "UNRESOLVED: 43 gold questions -> one flip = 2.33pp > 0.3pp; this fixture cannot resolve the gate (use the gate fixture with >= 334 rows; statistical power needs far more)" |
| } |
| }, |
| "verdict": "INCONCLUSIVE", |
| "resolution_pp_per_flip": 2.3255813953488373, |
| "note": "small fixtures cannot resolve 0.3pp accuracy gates; accuracy on a parity fixture is only indicative (synthetic long rows carry the base row's gold label).", |
| "ref": "runs/macjev/runtime_v2_dev/reference_trained/ref_block_fp32.jsonl", |
| "cand": "runs/macjev/release_2b/parity/pred_mlx_affine8-g64_base.jsonl", |
| "fixture": "runs/macjev/runtime_v2_dev/reference_base/fixture.jsonl" |
| } |
