| { |
| "raw": { |
| "n": 1380, |
| "accuracy": 0.7847826086956522, |
| "nll": 0.581156894274155, |
| "brier": 0.302482117542112, |
| "ece": 0.04275912569513245, |
| "confident_error_rate": 0.025362318840579712, |
| "score_mae": 0.4490936795609348 |
| }, |
| "calibrated": { |
| "n": 1380, |
| "accuracy": 0.7847826086956522, |
| "nll": 0.581156894274155, |
| "brier": 0.302482117542112, |
| "ece": 0.04275912569513245, |
| "confident_error_rate": 0.025362318840579712, |
| "score_mae": 0.4490936795609348 |
| }, |
| "temperature": 1.0, |
| "by_source": { |
| "agnews": { |
| "n": 60, |
| "accuracy": 0.85, |
| "nll": 0.6513144311741309, |
| "brier": 0.251907969145848, |
| "ece": 0.11842021801854002, |
| "confident_error_rate": 0.08333333333333333 |
| }, |
| "agnews_yn": { |
| "n": 120, |
| "accuracy": 0.9083333333333333, |
| "nll": 0.2551103684956211, |
| "brier": 0.14104220820492167, |
| "ece": 0.06372701494139532, |
| "confident_error_rate": 0.041666666666666664 |
| }, |
| "amazon": { |
| "n": 60, |
| "accuracy": 0.5, |
| "nll": 1.0774350213199255, |
| "brier": 0.5897144391160517, |
| "ece": 0.16833261355523615, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.6220959822294679 |
| }, |
| "banking77": { |
| "n": 60, |
| "accuracy": 0.7666666666666667, |
| "nll": 1.0541223604959076, |
| "brier": 0.37672671737088687, |
| "ece": 0.11317603709370838, |
| "confident_error_rate": 0.08333333333333333 |
| }, |
| "boolq": { |
| "n": 60, |
| "accuracy": 0.7, |
| "nll": 0.6696082752191643, |
| "brier": 0.4106637746455205, |
| "ece": 0.14089230686793108, |
| "confident_error_rate": 0.06666666666666667 |
| }, |
| "dbpedia14": { |
| "n": 60, |
| "accuracy": 0.9666666666666667, |
| "nll": 0.344855452983493, |
| "brier": 0.07056367579054522, |
| "ece": 0.038452588673439585, |
| "confident_error_rate": 0.03333333333333333 |
| }, |
| "imdb": { |
| "n": 60, |
| "accuracy": 0.85, |
| "nll": 0.5053863840848449, |
| "brier": 0.23495492962602346, |
| "ece": 0.11279579994702389, |
| "confident_error_rate": 0.06666666666666667 |
| }, |
| "mnli": { |
| "n": 60, |
| "accuracy": 0.8, |
| "nll": 0.7294453055896385, |
| "brier": 0.34649226553636786, |
| "ece": 0.15010490332446808, |
| "confident_error_rate": 0.05 |
| }, |
| "sst5": { |
| "n": 60, |
| "accuracy": 0.5166666666666667, |
| "nll": 1.0743940704017836, |
| "brier": 0.6115219538954813, |
| "ece": 0.17046597533413535, |
| "confident_error_rate": 0.016666666666666666, |
| "score_mae": 0.5835985922635848 |
| }, |
| "synthetic_approval_en": { |
| "n": 13, |
| "accuracy": 1.0, |
| "nll": 5.794195919754623e-06, |
| "brier": 1.374480022034159e-10, |
| "ece": 5.794161557548527e-06, |
| "confident_error_rate": 0.0 |
| }, |
| "synthetic_approval_zh": { |
| "n": 13, |
| "accuracy": 1.0, |
| "nll": 1.9408386281863444e-05, |
| "brier": 2.1491880932216216e-09, |
| "ece": 1.940784896004022e-05, |
| "confident_error_rate": 0.0 |
| }, |
| "synthetic_deadline_en": { |
| "n": 12, |
| "accuracy": 0.9166666666666666, |
| "nll": 0.11081959133781827, |
| "brier": 0.06204603633292526, |
| "ece": 0.08506965862457389, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.07901112666262512 |
| }, |
| "synthetic_deadline_zh": { |
| "n": 12, |
| "accuracy": 1.0, |
| "nll": 0.12181195209883083, |
| "brier": 0.05315409669261643, |
| "ece": 0.10159962877628, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.09981863623239674 |
| }, |
| "synthetic_membership_en": { |
| "n": 12, |
| "accuracy": 1.0, |
| "nll": 0.002240487862263786, |
| "brier": 0.0001068127336093784, |
| "ece": 0.0022132804887543545, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.002180111100623556 |
| }, |
| "synthetic_membership_zh": { |
| "n": 12, |
| "accuracy": 1.0, |
| "nll": 4.428395804630527e-05, |
| "brier": 1.6740347101226407e-08, |
| "ece": 4.427923544458867e-05, |
| "confident_error_rate": 0.0, |
| "score_mae": 3.434614207920017e-05 |
| }, |
| "synthetic_refund_en": { |
| "n": 13, |
| "accuracy": 1.0, |
| "nll": 5.0393487257410835e-05, |
| "brier": 1.0734725052356504e-08, |
| "ece": 5.039080336088908e-05, |
| "confident_error_rate": 0.0 |
| }, |
| "synthetic_refund_zh": { |
| "n": 13, |
| "accuracy": 1.0, |
| "nll": 5.624815425520311e-05, |
| "brier": 1.4202719808174667e-08, |
| "ece": 5.624460322828373e-05, |
| "confident_error_rate": 0.0 |
| }, |
| "synthetic_snake_observable_teacher": { |
| "n": 500, |
| "accuracy": 0.736, |
| "nll": 0.6332765788074997, |
| "brier": 0.3648938935907012, |
| "ece": 0.08253579614310942, |
| "confident_error_rate": 0.002 |
| }, |
| "trec": { |
| "n": 60, |
| "accuracy": 0.9666666666666667, |
| "nll": 0.1906178114249527, |
| "brier": 0.0665774868912819, |
| "ece": 0.035852720115976094, |
| "confident_error_rate": 0.03333333333333333 |
| }, |
| "yelp": { |
| "n": 60, |
| "accuracy": 0.6166666666666667, |
| "nll": 0.9644246569622162, |
| "brier": 0.5148714304971759, |
| "ece": 0.1522045943935158, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.4646525638109543 |
| }, |
| "yelp_yn": { |
| "n": 60, |
| "accuracy": 0.9166666666666666, |
| "nll": 0.27046740896257127, |
| "brier": 0.13716579955592267, |
| "ece": 0.0783473425274968, |
| "confident_error_rate": 0.05 |
| } |
| }, |
| "by_language": { |
| "en": { |
| "n": 1330, |
| "accuracy": 0.7766917293233083, |
| "nll": 0.6019047035568604, |
| "brier": 0.31337403957469484, |
| "ece": 0.04491990452563714, |
| "confident_error_rate": 0.02631578947368421, |
| "score_mae": 0.496054525251958 |
| }, |
| "zh": { |
| "n": 50, |
| "accuracy": 1.0, |
| "nll": 0.029265167354190148, |
| "brier": 0.012756991475407299, |
| "ece": 0.02441420756038306, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.04992649118723797 |
| } |
| }, |
| "by_qtype": { |
| "choice": { |
| "n": 826, |
| "accuracy": 0.7929782082324455, |
| "nll": 0.5991040420549707, |
| "brier": 0.3016743749505722, |
| "ece": 0.05670830022410128, |
| "confident_error_rate": 0.021791767554479417 |
| }, |
| "noul": { |
| "n": 326, |
| "accuracy": 0.8680981595092024, |
| "nll": 0.35994587318039584, |
| "brier": 0.19598814582339072, |
| "ece": 0.07256615132756117, |
| "confident_error_rate": 0.049079754601226995 |
| }, |
| "score": { |
| "n": 228, |
| "accuracy": 0.6359649122807017, |
| "nll": 0.8324307925619255, |
| "brier": 0.4576761094759496, |
| "ece": 0.10432282288474241, |
| "confident_error_rate": 0.0043859649122807015, |
| "score_mae": 0.4490936795609348 |
| } |
| }, |
| "prediction_precision": [ |
| "cuda:bfloat16:fp32_pointer" |
| ] |
| } |
|
|
