{
 "what": "the UNQUANTIZED bf16 model with relative Gaussian noise 0.001 on one tensor (layer 1 down_proj): the agreement any quantization of this model is bounded by",
 "note": "text_* fields are the deviation from the reference on documents rendered inside user turns; they are a fidelity number, not a quality metric",
 "klref": {
  "sequences": 28,
  "positions": 68370,
  "median_kl": 0.06028680503368378,
  "mean_kl": 0.7488869428634644,
  "p90": 2.2322335243225098,
  "p95": 4.25889778137207,
  "p99": 9.620244026184082,
  "kl_max": 26.808528900146484,
  "top1_pct": 71.58841597191751,
  "top5_pct": 85.82272926722247,
  "top10_pct": 89.08585636975282,
  "text_ppl_q": 67.33650207519531,
  "text_ppl_ref": 64.08563232421875,
  "text_ppl_ratio": 1.0507272481918335,
  "text_nll_delta_mean": 0.049482543021440506,
  "text_nll_delta_sem": 0.007424323470551016,
  "text_positions_compared_pct": 100.0,
  "text_ppl_unbiased": true,
  "text_top1_acc_q_pct": 44.4493198771391,
  "text_top1_acc_ref_pct": 44.99195553605382,
  "call_positions": 911,
  "call_nll_q": 1.9571926593780518,
  "call_nll_ref": 1.8504042625427246,
  "call_nll_delta": 0.10678845643997192,
  "call_nll_delta_sem": 0.07356708820417222,
  "call_top1_acc_q_pct": 72.44785949506037,
  "call_top1_acc_ref_pct": 74.86278814489572,
  "call_p_correct_q_geomean": 0.14125441014766693,
  "call_p_correct_ref_geomean": 0.15717361867427826,
  "per_domain": {
   "academic_mc": {
    "sequences": 2,
    "positions": 3123,
    "median_kl": 0.07163441926240921,
    "mean_kl": 0.4941253066062927,
    "p90": 1.2521964311599731,
    "p95": 2.463402509689331,
    "p99": 6.70340633392334,
    "kl_max": 21.34412956237793,
    "top1_pct": 73.07076528978547,
    "top5_pct": 89.94556516170348,
    "top10_pct": 93.01953250080052,
    "text_ppl_q": 77.51740264892578,
    "text_ppl_ref": 83.09352111816406,
    "text_ppl_ratio": 0.9328935146331787,
    "text_nll_delta_mean": -0.06946424394845963,
    "text_nll_delta_sem": 0.028082956470559688,
    "text_positions_compared_pct": 100.0,
    "text_ppl_unbiased": true,
    "text_top1_acc_q_pct": 38.52065321805956,
    "text_top1_acc_ref_pct": 38.58469420429075
   },
   "agentic": {
    "sequences": 4,
    "positions": 7411,
    "median_kl": 0.0958956927061081,
    "mean_kl": 0.8795613646507263,
    "p90": 2.719298839569092,
    "p95": 4.733425617218018,
    "p99": 9.887956619262695,
    "kl_max": 19.4514217376709,
    "top1_pct": 66.92754014303063,
    "top5_pct": 82.33706652273646,
    "top10_pct": 86.26366212386992,
    "text_ppl_q": 155.5294189453125,
    "text_ppl_ref": 144.7749786376953,
    "text_ppl_ratio": 1.0742839574813843,
    "text_nll_delta_mean": 0.07165432721376419,
    "text_nll_delta_sem": 0.023170038994200767,
    "text_positions_compared_pct": 100.0,
    "text_ppl_unbiased": true,
    "text_top1_acc_q_pct": 39.69774659290245,
    "text_top1_acc_ref_pct": 40.08905680744839,
    "call_positions": 315,
    "call_nll_q": 2.1134989261627197,
    "call_nll_ref": 1.8950586318969727,
    "call_nll_delta": 0.218440443277359,
    "call_nll_delta_sem": 0.126491185309146,
    "call_top1_acc_q_pct": 72.38095238095238,
    "call_top1_acc_ref_pct": 76.19047619047619,
    "call_p_correct_q_geomean": 0.1208145022392273,
    "call_p_correct_ref_geomean": 0.1503095179796219
   },
   "chinese": {
    "sequences": 2,
    "positions": 3158,
    "median_kl": 0.0375639982521534,
    "mean_kl": 0.540226936340332,
    "p90": 1.2133997678756714,
    "p95": 3.0907554626464844,
    "p99": 9.696211814880371,
    "kl_max": 19.559850692749023,
    "top1_pct": 77.83407219759341,
    "top5_pct": 90.75364154528182,
    "top10_pct": 93.35022165927802,
    "text_ppl_q": 146.9723358154297,
    "text_ppl_ref": 139.631103515625,
    "text_ppl_ratio": 1.052575707435608,
    "text_nll_delta_mean": 0.0512402206659317,
    "text_nll_delta_sem": 0.037088767409218,
    "text_positions_compared_pct": 100.0,
    "text_ppl_unbiased": true,
    "text_top1_acc_q_pct": 35.718809373020896,
    "text_top1_acc_ref_pct": 35.718809373020896
   },
   "coding": {
    "sequences": 5,
    "positions": 8249,
    "median_kl": 0.030682625249028206,
    "mean_kl": 0.7262228727340698,
    "p90": 2.2243595123291016,
    "p95": 4.3674492835998535,
    "p99": 9.655651092529297,
    "kl_max": 22.811620712280273,
    "top1_pct": 73.7059037459086,
    "top5_pct": 86.33773790762517,
    "top10_pct": 89.34416292883985,
    "text_ppl_q": 51.62739562988281,
    "text_ppl_ref": 48.28934097290039,
    "text_ppl_ratio": 1.0691256523132324,
    "text_nll_delta_mean": 0.06684114784002304,
    "text_nll_delta_sem": 0.02025469839937471,
    "text_positions_compared_pct": 100.0,
    "text_ppl_unbiased": true,
    "text_top1_acc_q_pct": 48.66044369014426,
    "text_top1_acc_ref_pct": 49.399927263910776
   },
   "cybersec": {
    "sequences": 4,
    "positions": 6653,
    "median_kl": 0.02382112666964531,
    "mean_kl": 0.6448559165000916,
    "p90": 1.9087097644805908,
    "p95": 4.029167652130127,
    "p99": 8.73491096496582,
    "kl_max": 23.606565475463867,
    "top1_pct": 76.41665414098902,
    "top5_pct": 88.20081166391101,
    "top10_pct": 90.80114234180068,
    "text_ppl_q": 44.303619384765625,
    "text_ppl_ref": 40.27240753173828,
    "text_ppl_ratio": 1.1000986099243164,
    "text_nll_delta_mean": 0.09539978206157684,
    "text_nll_delta_sem": 0.021676263560214893,
    "text_positions_compared_pct": 100.0,
    "text_ppl_unbiased": true,
    "text_top1_acc_q_pct": 51.66090485495265,
    "text_top1_acc_ref_pct": 52.72809258980911
   },
   "general": {
    "sequences": 3,
    "positions": 5311,
    "median_kl": 0.07391507178544998,
    "mean_kl": 0.6419679522514343,
    "p90": 1.8744210004806519,
    "p95": 3.544818878173828,
    "p99": 8.784656524658203,
    "kl_max": 19.794815063476562,
    "top1_pct": 71.49312747128602,
    "top5_pct": 86.44323102993786,
    "top10_pct": 89.68179250611938,
    "text_ppl_q": 134.31143188476562,
    "text_ppl_ref": 122.87806701660156,
    "text_ppl_ratio": 1.0930465459823608,
    "text_nll_delta_mean": 0.08896876871585846,
    "text_nll_delta_sem": 0.028080449858036344,
    "text_positions_compared_pct": 100.0,
    "text_ppl_unbiased": true,
    "text_top1_acc_q_pct": 38.7497646394276,
    "text_top1_acc_ref_pct": 39.20165693842967
   },
   "long_agentic": {
    "sequences": 2,
    "positions": 10190,
    "median_kl": 0.1518646776676178,
    "mean_kl": 1.174489974975586,
    "p90": 3.756044387817383,
    "p95": 6.036811351776123,
    "p99": 11.245322227478027,
    "kl_max": 19.850561141967773,
    "top1_pct": 63.65063788027477,
    "top5_pct": 78.32188420019627,
    "top10_pct": 82.6594700686948,
    "text_ppl_q": 80.446533203125,
    "text_ppl_ref": 77.45440673828125,
    "text_ppl_ratio": 1.038630723953247,
    "text_nll_delta_mean": 0.037903208285570145,
    "text_nll_delta_sem": 0.024527187599360113,
    "text_positions_compared_pct": 100.0,
    "text_ppl_unbiased": true,
    "text_top1_acc_q_pct": 44.94602551521099,
    "text_top1_acc_ref_pct": 45.37782139352306,
    "call_positions": 596,
    "call_nll_q": 1.874580979347229,
    "call_nll_ref": 1.8268033266067505,
    "call_nll_delta": 0.047777753323316574,
    "call_nll_delta_sem": 0.09040192085424628,
    "call_top1_acc_q_pct": 72.48322147651007,
    "call_top1_acc_ref_pct": 74.16107382550335,
    "call_p_correct_q_geomean": 0.15341924130916595,
    "call_p_correct_ref_geomean": 0.1609271764755249
   },
   "long_coding": {
    "sequences": 2,
    "positions": 9390,
    "median_kl": 0.035911377519369125,
    "mean_kl": 0.7038131952285767,
    "p90": 2.17490816116333,
    "p95": 4.21785306930542,
    "p99": 9.04932689666748,
    "kl_max": 19.187047958374023,
    "top1_pct": 74.49414270500533,
    "top5_pct": 86.74121405750799,
    "top10_pct": 89.55271565495208,
    "text_ppl_q": 37.17418670654297,
    "text_ppl_ref": 37.08015441894531,
    "text_ppl_ratio": 1.0025359392166138,
    "text_nll_delta_mean": 0.0025327790062874556,
    "text_nll_delta_sem": 0.01903810936491189,
    "text_positions_compared_pct": 100.0,
    "text_ppl_unbiased": true,
    "text_top1_acc_q_pct": 51.9275825346113,
    "text_top1_acc_ref_pct": 52.51331203407881
   },
   "long_longctx": {
    "sequences": 2,
    "positions": 10990,
    "median_kl": 0.0587487667798996,
    "mean_kl": 0.5435295701026917,
    "p90": 1.2706959247589111,
    "p95": 2.847358226776123,
    "p99": 8.863947868347168,
    "kl_max": 23.567869186401367,
    "top1_pct": 74.00363967242947,
    "top5_pct": 89.7179253867152,
    "top10_pct": 92.72065514103731,
    "text_ppl_q": 50.387821197509766,
    "text_ppl_ref": 48.85396957397461,
    "text_ppl_ratio": 1.0313963890075684,
    "text_nll_delta_mean": 0.03091362863779068,
    "text_nll_delta_sem": 0.015269082721638993,
    "text_positions_compared_pct": 100.0,
    "text_ppl_unbiased": true,
    "text_top1_acc_q_pct": 41.51046405823476,
    "text_top1_acc_ref_pct": 42.029117379435846
   },
   "science": {
    "sequences": 2,
    "positions": 3895,
    "median_kl": 0.09950811415910721,
    "mean_kl": 0.8198201060295105,
    "p90": 2.446251153945923,
    "p95": 4.460198879241943,
    "p99": 9.16201114654541,
    "kl_max": 26.808528900146484,
    "top1_pct": 68.54942233632862,
    "top5_pct": 85.57124518613607,
    "top10_pct": 88.98587933247754,
    "text_ppl_q": 54.17967224121094,
    "text_ppl_ref": 47.66631317138672,
    "text_ppl_ratio": 1.1366446018218994,
    "text_nll_delta_mean": 0.1280806064605713,
    "text_nll_delta_sem": 0.029425310867253786,
    "text_positions_compared_pct": 100.0,
    "text_ppl_unbiased": true,
    "text_top1_acc_q_pct": 40.821566110397946,
    "text_top1_acc_ref_pct": 41.54043645699615
   }
  }
 },
 "tool_conversations_native_format": {
  "sequences": 96,
  "positions": 65719,
  "median_kl": 0.29158198833465576,
  "mean_kl": 0.9305510520935059,
  "p90": 2.485236644744873,
  "p95": 3.8753490447998047,
  "p99": 9.514266967773438,
  "kl_max": 27.091535568237305,
  "top1_pct": 55.055615575404374,
  "top5_pct": 78.43698169479147,
  "top10_pct": 84.22982699067241,
  "text_ppl_q": 1861.92822265625,
  "text_ppl_ref": 1744.8138427734375,
  "text_ppl_ratio": 1.067121148109436,
  "text_nll_delta_mean": 0.06496450304985046,
  "text_nll_delta_sem": 0.0065325011678668814,
  "text_positions_compared_pct": 100.0,
  "text_ppl_unbiased": true,
  "text_top1_acc_q_pct": 18.725178411113987,
  "text_top1_acc_ref_pct": 18.764740790334606,
  "call_positions": 8544,
  "call_nll_q": 5.430123805999756,
  "call_nll_ref": 5.388685703277588,
  "call_nll_delta": 0.041438471525907516,
  "call_nll_delta_sem": 0.018797064849230553,
  "call_top1_acc_q_pct": 41.725187265917604,
  "call_top1_acc_ref_pct": 42.017790262172284,
  "call_p_correct_q_geomean": 0.004382553044706583,
  "call_p_correct_ref_geomean": 0.0045679728500545025,
  "decisions": {
   "call": {
    "n": 112,
    "argmax_agree_pct": 91.96428571428571,
    "call_answer_flips": 9,
    "reference_starts_tool_call": 100,
    "model_starts_tool_call": 97,
    "median_p_toolcall_ref": 0.8585168123245239,
    "median_p_toolcall_q": 0.8496556878089905,
    "min_p_toolcall_q": 0.053277384489774704,
    "max_p_toolcall_q": 0.9938733577728271
   },
   "answer": {
    "n": 96,
    "argmax_agree_pct": 91.66666666666667,
    "call_answer_flips": 0,
    "reference_starts_tool_call": 1,
    "model_starts_tool_call": 1,
    "median_p_toolcall_ref": 0.0023557787062600255,
    "median_p_toolcall_q": 0.002312922733835876,
    "min_p_toolcall_q": 0.00013880895858164877,
    "max_p_toolcall_q": 0.3773735463619232
   }
  }
 }
}