| { |
| "calibration": { |
| "raw": { |
| "n": 1120, |
| "accuracy": 0.9071428571428571, |
| "nll": 0.41561266005147984, |
| "brier": 0.15439869504298345, |
| "ece": 0.06529658877666611, |
| "confident_error_rate": 0.048214285714285716, |
| "score_mae": 0.3755916111744299 |
| }, |
| "calibrated": { |
| "n": 1120, |
| "accuracy": 0.9071428571428571, |
| "nll": 0.24696319292379612, |
| "brier": 0.1302047670837569, |
| "ece": 0.011292779251836818, |
| "confident_error_rate": 0.010714285714285714, |
| "score_mae": 0.4186460857642795 |
| }, |
| "temperature": 2.82842712474619, |
| "by_source": { |
| "agnews": { |
| "n": 40, |
| "accuracy": 0.875, |
| "nll": 0.42431218908163526, |
| "brier": 0.2124107838045201, |
| "ece": 0.10071692229045397, |
| "confident_error_rate": 0.05 |
| }, |
| "agnews_yn": { |
| "n": 80, |
| "accuracy": 0.95, |
| "nll": 0.17612961540906633, |
| "brier": 0.08799404961702698, |
| "ece": 0.04380159196878353, |
| "confident_error_rate": 0.0125 |
| }, |
| "amazon": { |
| "n": 40, |
| "accuracy": 0.525, |
| "nll": 1.0860239159124931, |
| "brier": 0.64950342264714, |
| "ece": 0.19886266952857856, |
| "confident_error_rate": 0.05, |
| "score_mae": 0.5730869934206698 |
| }, |
| "banking77": { |
| "n": 40, |
| "accuracy": 0.7, |
| "nll": 0.9027141743195617, |
| "brier": 0.35934266091172196, |
| "ece": 0.1505055259902214, |
| "confident_error_rate": 0.0 |
| }, |
| "boolq": { |
| "n": 40, |
| "accuracy": 0.825, |
| "nll": 0.43203726272070053, |
| "brier": 0.2525550579654571, |
| "ece": 0.1236843930865563, |
| "confident_error_rate": 0.05 |
| }, |
| "dbpedia14": { |
| "n": 40, |
| "accuracy": 1.0, |
| "nll": 0.03320092462083566, |
| "brier": 0.007730069688019724, |
| "ece": 0.028915295970673674, |
| "confident_error_rate": 0.0 |
| }, |
| "imdb": { |
| "n": 40, |
| "accuracy": 0.95, |
| "nll": 0.1253111832164516, |
| "brier": 0.07017642297254414, |
| "ece": 0.05097512371140157, |
| "confident_error_rate": 0.0 |
| }, |
| "mnli": { |
| "n": 40, |
| "accuracy": 0.775, |
| "nll": 0.5642081614377643, |
| "brier": 0.31750739242186327, |
| "ece": 0.13470223416022017, |
| "confident_error_rate": 0.05 |
| }, |
| "sst5": { |
| "n": 40, |
| "accuracy": 0.575, |
| "nll": 0.9575553270599549, |
| "brier": 0.5275216694379973, |
| "ece": 0.20643132818182183, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.5891576445775591 |
| }, |
| "synthetic_approval_en": { |
| "n": 13, |
| "accuracy": 1.0, |
| "nll": 0.00030560799850466096, |
| "brier": 5.905838244888711e-07, |
| "ece": 0.00030546021556154823, |
| "confident_error_rate": 0.0 |
| }, |
| "synthetic_approval_zh": { |
| "n": 13, |
| "accuracy": 1.0, |
| "nll": 0.0020052385561802446, |
| "brier": 2.8123618467501365e-05, |
| "ece": 0.001998170998211757, |
| "confident_error_rate": 0.0 |
| }, |
| "synthetic_deadline_en": { |
| "n": 12, |
| "accuracy": 1.0, |
| "nll": 0.10283984624697795, |
| "brier": 0.03984589702686843, |
| "ece": 0.08750310019891688, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.08970968160249897 |
| }, |
| "synthetic_deadline_zh": { |
| "n": 12, |
| "accuracy": 1.0, |
| "nll": 0.21881028826991109, |
| "brier": 0.10438565880424289, |
| "ece": 0.17256679879495238, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.14747273501359034 |
| }, |
| "synthetic_membership_en": { |
| "n": 12, |
| "accuracy": 1.0, |
| "nll": 0.0036049772544839306, |
| "brier": 8.753823193823712e-05, |
| "ece": 0.003581882695322669, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.003259818638849648 |
| }, |
| "synthetic_membership_zh": { |
| "n": 12, |
| "accuracy": 0.9166666666666666, |
| "nll": 0.4811591276047158, |
| "brier": 0.16561993873737488, |
| "ece": 0.08165048419394716, |
| "confident_error_rate": 0.08333333333333333, |
| "score_mae": 0.08426967783280753 |
| }, |
| "synthetic_refund_en": { |
| "n": 13, |
| "accuracy": 1.0, |
| "nll": 0.0012299626519328703, |
| "brier": 4.9117060931387845e-06, |
| "ece": 0.0012287328943566544, |
| "confident_error_rate": 0.0 |
| }, |
| "synthetic_refund_zh": { |
| "n": 13, |
| "accuracy": 1.0, |
| "nll": 0.02272363248292462, |
| "brier": 0.009325216105233203, |
| "ece": 0.019921873383823608, |
| "confident_error_rate": 0.0 |
| }, |
| "synthetic_snake_observable_teacher": { |
| "n": 500, |
| "accuracy": 0.996, |
| "nll": 0.01952316967795501, |
| "brier": 0.007312097952411837, |
| "ece": 0.015592937436342616, |
| "confident_error_rate": 0.0 |
| }, |
| "trec": { |
| "n": 40, |
| "accuracy": 0.825, |
| "nll": 0.37977351285618693, |
| "brier": 0.22044583921846525, |
| "ece": 0.1472219287313714, |
| "confident_error_rate": 0.025 |
| }, |
| "yelp": { |
| "n": 40, |
| "accuracy": 0.6, |
| "nll": 1.0141922301868742, |
| "brier": 0.5740260294211242, |
| "ece": 0.16420861364978293, |
| "confident_error_rate": 0.025, |
| "score_mae": 0.4986553482854209 |
| }, |
| "yelp_yn": { |
| "n": 40, |
| "accuracy": 0.925, |
| "nll": 0.14888145329933447, |
| "brier": 0.0911014727225844, |
| "ece": 0.04055253767844966, |
| "confident_error_rate": 0.0 |
| } |
| }, |
| "by_language": { |
| "en": { |
| "n": 1070, |
| "accuracy": 0.9037383177570093, |
| "nll": 0.2503529605239699, |
| "brier": 0.1331473631279441, |
| "ece": 0.013145780495476095, |
| "confident_error_rate": 0.010280373831775701, |
| "score_mae": 0.4691085656544596 |
| }, |
| "zh": { |
| "n": 50, |
| "accuracy": 0.98, |
| "nll": 0.1744221662800777, |
| "brier": 0.06723321173815046, |
| "ece": 0.05640985311627228, |
| "confident_error_rate": 0.02, |
| "score_mae": 0.11587120642319892 |
| } |
| }, |
| "by_qtype": { |
| "choice": { |
| "n": 726, |
| "accuracy": 0.9517906336088154, |
| "nll": 0.14044074977524487, |
| "brier": 0.06660315716614232, |
| "ece": 0.01212724658610386, |
| "confident_error_rate": 0.006887052341597796 |
| }, |
| "noul": { |
| "n": 226, |
| "accuracy": 0.9292035398230089, |
| "nll": 0.188721070526274, |
| "brier": 0.10492979547492401, |
| "ece": 0.02175879329766155, |
| "confident_error_rate": 0.01327433628318584 |
| }, |
| "score": { |
| "n": 168, |
| "accuracy": 0.6845238095238095, |
| "nll": 0.7856418440409879, |
| "brier": 0.4390554360348545, |
| "ece": 0.06740864067949157, |
| "confident_error_rate": 0.023809523809523808, |
| "score_mae": 0.4186460857642795 |
| } |
| }, |
| "prediction_precision": [ |
| "cuda:bfloat16:fp32_pointer" |
| ] |
| }, |
| "development": { |
| "raw": { |
| "n": 1380, |
| "accuracy": 0.8775362318840579, |
| "nll": 0.6664165871795136, |
| "brier": 0.20319295941172028, |
| "ece": 0.08675410618204699, |
| "confident_error_rate": 0.06811594202898551, |
| "score_mae": 0.4164554340277584 |
| }, |
| "calibrated": { |
| "n": 1380, |
| "accuracy": 0.8775362318840579, |
| "nll": 0.35527357086144407, |
| "brier": 0.17843796833448278, |
| "ece": 0.02931368494845367, |
| "confident_error_rate": 0.02391304347826087, |
| "score_mae": 0.4661770030349038 |
| }, |
| "temperature": 2.82842712474619, |
| "by_source": { |
| "agnews": { |
| "n": 60, |
| "accuracy": 0.85, |
| "nll": 0.48608902204224386, |
| "brier": 0.245769421897556, |
| "ece": 0.09525198364374235, |
| "confident_error_rate": 0.05 |
| }, |
| "agnews_yn": { |
| "n": 120, |
| "accuracy": 0.9166666666666666, |
| "nll": 0.23583966067185608, |
| "brier": 0.13021957477372714, |
| "ece": 0.048408689477263414, |
| "confident_error_rate": 0.025 |
| }, |
| "amazon": { |
| "n": 60, |
| "accuracy": 0.5166666666666667, |
| "nll": 1.06260039607987, |
| "brier": 0.5713134054914024, |
| "ece": 0.17560362149541306, |
| "confident_error_rate": 0.016666666666666666, |
| "score_mae": 0.6313692442176524 |
| }, |
| "banking77": { |
| "n": 60, |
| "accuracy": 0.7833333333333333, |
| "nll": 1.0275805577679649, |
| "brier": 0.3974346424601793, |
| "ece": 0.20365650257509377, |
| "confident_error_rate": 0.06666666666666667 |
| }, |
| "boolq": { |
| "n": 60, |
| "accuracy": 0.75, |
| "nll": 0.6448481489529573, |
| "brier": 0.3859323161313, |
| "ece": 0.12717312580718626, |
| "confident_error_rate": 0.08333333333333333 |
| }, |
| "dbpedia14": { |
| "n": 60, |
| "accuracy": 0.95, |
| "nll": 0.26884904831059203, |
| "brier": 0.08193384550316109, |
| "ece": 0.03403225609777914, |
| "confident_error_rate": 0.016666666666666666 |
| }, |
| "imdb": { |
| "n": 60, |
| "accuracy": 0.85, |
| "nll": 0.3576647304685024, |
| "brier": 0.20057051393723982, |
| "ece": 0.07045008300683726, |
| "confident_error_rate": 0.03333333333333333 |
| }, |
| "mnli": { |
| "n": 60, |
| "accuracy": 0.75, |
| "nll": 0.6862650872191253, |
| "brier": 0.37262579450429956, |
| "ece": 0.13897722294328826, |
| "confident_error_rate": 0.05 |
| }, |
| "sst5": { |
| "n": 60, |
| "accuracy": 0.5, |
| "nll": 1.058029735089506, |
| "brier": 0.5865356187980687, |
| "ece": 0.13290239322998643, |
| "confident_error_rate": 0.016666666666666666, |
| "score_mae": 0.5817275709485884 |
| }, |
| "synthetic_approval_en": { |
| "n": 13, |
| "accuracy": 1.0, |
| "nll": 0.00013326233616468077, |
| "brier": 8.015656526086572e-08, |
| "ece": 0.00013324229165623258, |
| "confident_error_rate": 0.0 |
| }, |
| "synthetic_approval_zh": { |
| "n": 13, |
| "accuracy": 1.0, |
| "nll": 0.0014438875166684917, |
| "brier": 1.8695595964628948e-05, |
| "ece": 0.001439189286445819, |
| "confident_error_rate": 0.0 |
| }, |
| "synthetic_deadline_en": { |
| "n": 12, |
| "accuracy": 1.0, |
| "nll": 0.0806837765833715, |
| "brier": 0.02073876048782872, |
| "ece": 0.07365987710687534, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.06450716585168666 |
| }, |
| "synthetic_deadline_zh": { |
| "n": 12, |
| "accuracy": 1.0, |
| "nll": 0.2904704911597443, |
| "brier": 0.1244676647263207, |
| "ece": 0.23892868874962897, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.2594397335856935 |
| }, |
| "synthetic_membership_en": { |
| "n": 12, |
| "accuracy": 1.0, |
| "nll": 0.0007200739092177655, |
| "brier": 2.3758779283160902e-06, |
| "ece": 0.0007192933290399717, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.0002506762089642081 |
| }, |
| "synthetic_membership_zh": { |
| "n": 12, |
| "accuracy": 1.0, |
| "nll": 0.0002813137061592351, |
| "brier": 3.6555200693006093e-07, |
| "ece": 0.00028119365871936974, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.00012271551971073288 |
| }, |
| "synthetic_refund_en": { |
| "n": 13, |
| "accuracy": 1.0, |
| "nll": 0.0008021440397073382, |
| "brier": 2.797422859989472e-06, |
| "ece": 0.0008014436644004919, |
| "confident_error_rate": 0.0 |
| }, |
| "synthetic_refund_zh": { |
| "n": 13, |
| "accuracy": 1.0, |
| "nll": 0.0010484815579191785, |
| "brier": 3.84849638290293e-06, |
| "ece": 0.0010475179061284479, |
| "confident_error_rate": 0.0 |
| }, |
| "synthetic_snake_observable_teacher": { |
| "n": 500, |
| "accuracy": 0.984, |
| "nll": 0.06450042285864938, |
| "brier": 0.025878707558094703, |
| "ece": 0.006173020010911379, |
| "confident_error_rate": 0.008 |
| }, |
| "trec": { |
| "n": 60, |
| "accuracy": 0.95, |
| "nll": 0.2357539227501207, |
| "brier": 0.10389425338822736, |
| "ece": 0.05850789034904416, |
| "confident_error_rate": 0.05 |
| }, |
| "yelp": { |
| "n": 60, |
| "accuracy": 0.6666666666666666, |
| "nll": 1.024155086477965, |
| "brier": 0.5195945926255235, |
| "ece": 0.1848595542294458, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.49351173813318266 |
| }, |
| "yelp_yn": { |
| "n": 60, |
| "accuracy": 0.9166666666666666, |
| "nll": 0.23509973373594384, |
| "brier": 0.13332647973353443, |
| "ece": 0.04864025400999865, |
| "confident_error_rate": 0.05 |
| } |
| }, |
| "by_language": { |
| "en": { |
| "n": 1330, |
| "accuracy": 0.8729323308270677, |
| "nll": 0.36598203408453556, |
| "brier": 0.18402292245492163, |
| "ece": 0.03121802413433171, |
| "confident_error_rate": 0.02481203007518797, |
| "score_mae": 0.5057529769739862 |
| }, |
| "zh": { |
| "n": 50, |
| "accuracy": 1.0, |
| "nll": 0.07042844912720966, |
| "brier": 0.029878188730808993, |
| "ece": 0.05805691564807308, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.1297812245527021 |
| } |
| }, |
| "by_qtype": { |
| "choice": { |
| "n": 826, |
| "accuracy": 0.9382566585956417, |
| "nll": 0.23552417997919411, |
| "brier": 0.10295287570101166, |
| "ece": 0.0281465339266411, |
| "confident_error_rate": 0.021791767554479417 |
| }, |
| "noul": { |
| "n": 326, |
| "accuracy": 0.8803680981595092, |
| "nll": 0.31466740553017203, |
| "brier": 0.18041777287706098, |
| "ece": 0.02823807759497307, |
| "confident_error_rate": 0.03987730061349693 |
| }, |
| "score": { |
| "n": 228, |
| "accuracy": 0.6535087719298246, |
| "nll": 0.8471621970313261, |
| "brier": 0.4490751184852136, |
| "ece": 0.06873028430130265, |
| "confident_error_rate": 0.008771929824561403, |
| "score_mae": 0.4661770030349038 |
| } |
| }, |
| "prediction_precision": [ |
| "cuda:bfloat16:fp32_pointer" |
| ] |
| }, |
| "evaluation_precision": { |
| "device": "cuda:0", |
| "weight_dtype": "float32", |
| "autocast_dtype": "bfloat16", |
| "pointer_dtype": "float32", |
| "arithmetic": "bf16_autocast" |
| }, |
| "calibration_fit_partition": "calibration", |
| "training_seconds": 1484.693956375122, |
| "optimizer_steps": 2238, |
| "forward_tokens": 9992028, |
| "scope": "Development results; public task families also occur in training. Chinese data are programmatic rules, not a broad Chinese benchmark." |
| } |
|
|
