| { |
| "format": "8bit", |
| "temperature": 0.8278650620942867, |
| "overall": { |
| "n": 1000, |
| "top1_agreement": 0.996, |
| "top1_flips": 4, |
| "max_abs_dscore": 0.3184394836425781, |
| "mean_abs_dscore": 0.016680597170951345, |
| "mean_kl": 0.0002134925530053613, |
| "n_gold": 1000, |
| "nll_ref": 0.4436679457518203, |
| "nll_cand": 0.44417151398689614, |
| "dnll": 0.0005035682350758575, |
| "mean_abs_row_dnll": 0.008527335296640175, |
| "acc_ref": 0.808, |
| "acc_cand": 0.808, |
| "acc_drop_pp": 0.0 |
| }, |
| "per_category": { |
| "di_knowledge": { |
| "n": 93, |
| "top1_agreement": 0.978494623655914, |
| "top1_flips": 2, |
| "max_abs_dscore": 0.10516834259033203, |
| "mean_abs_dscore": 0.01826677256272753, |
| "mean_kl": 0.0001961899290394059, |
| "n_gold": 93, |
| "nll_ref": 0.7907936240501159, |
| "nll_cand": 0.7950259330695674, |
| "dnll": 0.004232309019451486, |
| "mean_abs_row_dnll": 0.012335049730692368, |
| "acc_ref": 0.6559139784946236, |
| "acc_cand": 0.6559139784946236, |
| "acc_drop_pp": 0.0 |
| }, |
| "di_language": { |
| "n": 93, |
| "top1_agreement": 1.0, |
| "top1_flips": 0, |
| "max_abs_dscore": 0.12582671642303467, |
| "mean_abs_dscore": 0.019231186946019475, |
| "mean_kl": 0.0002764365455637818, |
| "n_gold": 93, |
| "nll_ref": 0.43828192127009435, |
| "nll_cand": 0.4350339331964042, |
| "dnll": -0.0032479880736901445, |
| "mean_abs_row_dnll": 0.01105854314657161, |
| "acc_ref": 0.8494623655913979, |
| "acc_cand": 0.8494623655913979, |
| "acc_drop_pp": 0.0 |
| }, |
| "di_retrieval": { |
| "n": 93, |
| "top1_agreement": 1.0, |
| "top1_flips": 0, |
| "max_abs_dscore": 0.16489863395690918, |
| "mean_abs_dscore": 0.013032539303739148, |
| "mean_kl": 9.17728737128296e-05, |
| "n_gold": 93, |
| "nll_ref": 0.07565367364179643, |
| "nll_cand": 0.07596664403414172, |
| "dnll": 0.0003129703923452909, |
| "mean_abs_row_dnll": 0.0027333661229381077, |
| "acc_ref": 0.978494623655914, |
| "acc_cand": 0.978494623655914, |
| "acc_drop_pp": 0.0 |
| }, |
| "format": { |
| "n": 75, |
| "top1_agreement": 1.0, |
| "top1_flips": 0, |
| "max_abs_dscore": 0.15371152758598328, |
| "mean_abs_dscore": 0.014854268128133183, |
| "mean_kl": 8.706661370645814e-05, |
| "n_gold": 75, |
| "nll_ref": 0.09087868406020969, |
| "nll_cand": 0.09241995785860305, |
| "dnll": 0.001541273798393361, |
| "mean_abs_row_dnll": 0.003729830247715702, |
| "acc_ref": 0.9733333333333334, |
| "acc_cand": 0.9733333333333334, |
| "acc_drop_pp": 0.0 |
| }, |
| "general": { |
| "n": 93, |
| "top1_agreement": 1.0, |
| "top1_flips": 0, |
| "max_abs_dscore": 0.1003103256225586, |
| "mean_abs_dscore": 0.01342117658100456, |
| "mean_kl": 7.21948109281326e-05, |
| "n_gold": 93, |
| "nll_ref": 0.13061020512422436, |
| "nll_cand": 0.12993521810236933, |
| "dnll": -0.0006749870218550336, |
| "mean_abs_row_dnll": 0.003786342376918514, |
| "acc_ref": 0.967741935483871, |
| "acc_cand": 0.967741935483871, |
| "acc_drop_pp": 0.0 |
| }, |
| "hard_short": { |
| "n": 93, |
| "top1_agreement": 0.989247311827957, |
| "top1_flips": 1, |
| "max_abs_dscore": 0.12274712324142456, |
| "mean_abs_dscore": 0.020271506775084727, |
| "mean_kl": 0.00027143138298962177, |
| "n_gold": 93, |
| "nll_ref": 0.7835153258109905, |
| "nll_cand": 0.7861608190184016, |
| "dnll": 0.0026454932074111426, |
| "mean_abs_row_dnll": 0.014292107382686168, |
| "acc_ref": 0.6021505376344086, |
| "acc_cand": 0.5913978494623656, |
| "acc_drop_pp": 1.0752688172043001 |
| }, |
| "long": { |
| "n": 92, |
| "top1_agreement": 1.0, |
| "top1_flips": 0, |
| "max_abs_dscore": 0.3184394836425781, |
| "mean_abs_dscore": 0.020006766643251753, |
| "mean_kl": 0.0007467326030115854, |
| "n_gold": 92, |
| "nll_ref": 0.10827285500173092, |
| "nll_cand": 0.10447898892584594, |
| "dnll": -0.0037938660758849857, |
| "mean_abs_row_dnll": 0.008173341485227314, |
| "acc_ref": 0.967391304347826, |
| "acc_cand": 0.967391304347826, |
| "acc_drop_pp": 0.0 |
| }, |
| "mac": { |
| "n": 92, |
| "top1_agreement": 1.0, |
| "top1_flips": 0, |
| "max_abs_dscore": 0.05875861644744873, |
| "mean_abs_dscore": 0.013405789823635765, |
| "mean_kl": 3.8362254891210755e-05, |
| "n_gold": 92, |
| "nll_ref": 0.3262944821095264, |
| "nll_cand": 0.3269720723076003, |
| "dnll": 0.0006775901980738963, |
| "mean_abs_row_dnll": 0.0040734288260641455, |
| "acc_ref": 0.8369565217391305, |
| "acc_cand": 0.8369565217391305, |
| "acc_drop_pp": 0.0 |
| }, |
| "public_ingests": { |
| "n": 92, |
| "top1_agreement": 0.9891304347826086, |
| "top1_flips": 1, |
| "max_abs_dscore": 0.146925687789917, |
| "mean_abs_dscore": 0.020489049231511028, |
| "mean_kl": 0.00030913724098080613, |
| "n_gold": 92, |
| "nll_ref": 0.7667467143475788, |
| "nll_cand": 0.7701994879203283, |
| "dnll": 0.0034527735727495346, |
| "mean_abs_row_dnll": 0.016463952376233926, |
| "acc_ref": 0.6521739130434783, |
| "acc_cand": 0.6630434782608695, |
| "acc_drop_pp": -1.0869565217391242 |
| }, |
| "themes": { |
| "n": 92, |
| "top1_agreement": 1.0, |
| "top1_flips": 0, |
| "max_abs_dscore": 0.05321455001831055, |
| "mean_abs_dscore": 0.012158645316958427, |
| "mean_kl": 2.255066396046878e-05, |
| "n_gold": 92, |
| "nll_ref": 0.27025191463643466, |
| "nll_cand": 0.27031868131249703, |
| "dnll": 6.676667606236864e-05, |
| "mean_abs_row_dnll": 0.003036639894596646, |
| "acc_ref": 0.8695652173913043, |
| "acc_cand": 0.8695652173913043, |
| "acc_drop_pp": 0.0 |
| }, |
| "typed_synthetic": { |
| "n": 92, |
| "top1_agreement": 1.0, |
| "top1_flips": 0, |
| "max_abs_dscore": 0.07491475343704224, |
| "mean_abs_dscore": 0.018002478546206502, |
| "mean_kl": 0.00021491486269545232, |
| "n_gold": 92, |
| "nll_ref": 1.0338530850662833, |
| "nll_cand": 1.0343635982006703, |
| "dnll": 0.0005105131343869918, |
| "mean_abs_row_dnll": 0.0132145397374374, |
| "acc_ref": 0.5652173913043478, |
| "acc_cand": 0.5652173913043478, |
| "acc_drop_pp": 0.0 |
| } |
| }, |
| "gates": { |
| "coverage_finite_tokens": { |
| "pass": true, |
| "problems": {}, |
| "reference_missing_requests": 0 |
| }, |
| "top1": { |
| "threshold": 0.98, |
| "value": 0.996, |
| "pass": true |
| }, |
| "dnll": { |
| "threshold": 0.02, |
| "value": 0.0005035682350758575, |
| "pass": true |
| }, |
| "acc_drop_pp": { |
| "threshold": 0.3, |
| "value": 0.0, |
| "resolution_pp": 0.1, |
| "pass": true, |
| "note": null |
| } |
| }, |
| "verdict": "PASS", |
| "resolution_pp_per_flip": 0.1, |
| "note": "small fixtures cannot resolve 0.3pp accuracy gates; accuracy on a parity fixture is only indicative (synthetic long rows carry the base row's gold label).", |
| "ref": "runs/macjev/runtime_v2_dev/reference_trained/gate_ref_block_fp32.jsonl", |
| "cand": "runs/macjev/release_2b/parity/pred_mlx_affine8-g64_gate.jsonl", |
| "fixture": "runs/macjev/runtime_v2_dev/reference/gate_fixture_n1000.jsonl" |
| } |
