{
  "raw": {
    "n": 1380,
    "accuracy": 0.7847826086956522,
    "nll": 0.581156894274155,
    "brier": 0.302482117542112,
    "ece": 0.04275912569513245,
    "confident_error_rate": 0.025362318840579712,
    "score_mae": 0.4490936795609348
  },
  "calibrated": {
    "n": 1380,
    "accuracy": 0.7847826086956522,
    "nll": 0.581156894274155,
    "brier": 0.302482117542112,
    "ece": 0.04275912569513245,
    "confident_error_rate": 0.025362318840579712,
    "score_mae": 0.4490936795609348
  },
  "temperature": 1.0,
  "by_source": {
    "agnews": {
      "n": 60,
      "accuracy": 0.85,
      "nll": 0.6513144311741309,
      "brier": 0.251907969145848,
      "ece": 0.11842021801854002,
      "confident_error_rate": 0.08333333333333333
    },
    "agnews_yn": {
      "n": 120,
      "accuracy": 0.9083333333333333,
      "nll": 0.2551103684956211,
      "brier": 0.14104220820492167,
      "ece": 0.06372701494139532,
      "confident_error_rate": 0.041666666666666664
    },
    "amazon": {
      "n": 60,
      "accuracy": 0.5,
      "nll": 1.0774350213199255,
      "brier": 0.5897144391160517,
      "ece": 0.16833261355523615,
      "confident_error_rate": 0.0,
      "score_mae": 0.6220959822294679
    },
    "banking77": {
      "n": 60,
      "accuracy": 0.7666666666666667,
      "nll": 1.0541223604959076,
      "brier": 0.37672671737088687,
      "ece": 0.11317603709370838,
      "confident_error_rate": 0.08333333333333333
    },
    "boolq": {
      "n": 60,
      "accuracy": 0.7,
      "nll": 0.6696082752191643,
      "brier": 0.4106637746455205,
      "ece": 0.14089230686793108,
      "confident_error_rate": 0.06666666666666667
    },
    "dbpedia14": {
      "n": 60,
      "accuracy": 0.9666666666666667,
      "nll": 0.344855452983493,
      "brier": 0.07056367579054522,
      "ece": 0.038452588673439585,
      "confident_error_rate": 0.03333333333333333
    },
    "imdb": {
      "n": 60,
      "accuracy": 0.85,
      "nll": 0.5053863840848449,
      "brier": 0.23495492962602346,
      "ece": 0.11279579994702389,
      "confident_error_rate": 0.06666666666666667
    },
    "mnli": {
      "n": 60,
      "accuracy": 0.8,
      "nll": 0.7294453055896385,
      "brier": 0.34649226553636786,
      "ece": 0.15010490332446808,
      "confident_error_rate": 0.05
    },
    "sst5": {
      "n": 60,
      "accuracy": 0.5166666666666667,
      "nll": 1.0743940704017836,
      "brier": 0.6115219538954813,
      "ece": 0.17046597533413535,
      "confident_error_rate": 0.016666666666666666,
      "score_mae": 0.5835985922635848
    },
    "synthetic_approval_en": {
      "n": 13,
      "accuracy": 1.0,
      "nll": 5.794195919754623e-06,
      "brier": 1.374480022034159e-10,
      "ece": 5.794161557548527e-06,
      "confident_error_rate": 0.0
    },
    "synthetic_approval_zh": {
      "n": 13,
      "accuracy": 1.0,
      "nll": 1.9408386281863444e-05,
      "brier": 2.1491880932216216e-09,
      "ece": 1.940784896004022e-05,
      "confident_error_rate": 0.0
    },
    "synthetic_deadline_en": {
      "n": 12,
      "accuracy": 0.9166666666666666,
      "nll": 0.11081959133781827,
      "brier": 0.06204603633292526,
      "ece": 0.08506965862457389,
      "confident_error_rate": 0.0,
      "score_mae": 0.07901112666262512
    },
    "synthetic_deadline_zh": {
      "n": 12,
      "accuracy": 1.0,
      "nll": 0.12181195209883083,
      "brier": 0.05315409669261643,
      "ece": 0.10159962877628,
      "confident_error_rate": 0.0,
      "score_mae": 0.09981863623239674
    },
    "synthetic_membership_en": {
      "n": 12,
      "accuracy": 1.0,
      "nll": 0.002240487862263786,
      "brier": 0.0001068127336093784,
      "ece": 0.0022132804887543545,
      "confident_error_rate": 0.0,
      "score_mae": 0.002180111100623556
    },
    "synthetic_membership_zh": {
      "n": 12,
      "accuracy": 1.0,
      "nll": 4.428395804630527e-05,
      "brier": 1.6740347101226407e-08,
      "ece": 4.427923544458867e-05,
      "confident_error_rate": 0.0,
      "score_mae": 3.434614207920017e-05
    },
    "synthetic_refund_en": {
      "n": 13,
      "accuracy": 1.0,
      "nll": 5.0393487257410835e-05,
      "brier": 1.0734725052356504e-08,
      "ece": 5.039080336088908e-05,
      "confident_error_rate": 0.0
    },
    "synthetic_refund_zh": {
      "n": 13,
      "accuracy": 1.0,
      "nll": 5.624815425520311e-05,
      "brier": 1.4202719808174667e-08,
      "ece": 5.624460322828373e-05,
      "confident_error_rate": 0.0
    },
    "synthetic_snake_observable_teacher": {
      "n": 500,
      "accuracy": 0.736,
      "nll": 0.6332765788074997,
      "brier": 0.3648938935907012,
      "ece": 0.08253579614310942,
      "confident_error_rate": 0.002
    },
    "trec": {
      "n": 60,
      "accuracy": 0.9666666666666667,
      "nll": 0.1906178114249527,
      "brier": 0.0665774868912819,
      "ece": 0.035852720115976094,
      "confident_error_rate": 0.03333333333333333
    },
    "yelp": {
      "n": 60,
      "accuracy": 0.6166666666666667,
      "nll": 0.9644246569622162,
      "brier": 0.5148714304971759,
      "ece": 0.1522045943935158,
      "confident_error_rate": 0.0,
      "score_mae": 0.4646525638109543
    },
    "yelp_yn": {
      "n": 60,
      "accuracy": 0.9166666666666666,
      "nll": 0.27046740896257127,
      "brier": 0.13716579955592267,
      "ece": 0.0783473425274968,
      "confident_error_rate": 0.05
    }
  },
  "by_language": {
    "en": {
      "n": 1330,
      "accuracy": 0.7766917293233083,
      "nll": 0.6019047035568604,
      "brier": 0.31337403957469484,
      "ece": 0.04491990452563714,
      "confident_error_rate": 0.02631578947368421,
      "score_mae": 0.496054525251958
    },
    "zh": {
      "n": 50,
      "accuracy": 1.0,
      "nll": 0.029265167354190148,
      "brier": 0.012756991475407299,
      "ece": 0.02441420756038306,
      "confident_error_rate": 0.0,
      "score_mae": 0.04992649118723797
    }
  },
  "by_qtype": {
    "choice": {
      "n": 826,
      "accuracy": 0.7929782082324455,
      "nll": 0.5991040420549707,
      "brier": 0.3016743749505722,
      "ece": 0.05670830022410128,
      "confident_error_rate": 0.021791767554479417
    },
    "noul": {
      "n": 326,
      "accuracy": 0.8680981595092024,
      "nll": 0.35994587318039584,
      "brier": 0.19598814582339072,
      "ece": 0.07256615132756117,
      "confident_error_rate": 0.049079754601226995
    },
    "score": {
      "n": 228,
      "accuracy": 0.6359649122807017,
      "nll": 0.8324307925619255,
      "brier": 0.4576761094759496,
      "ece": 0.10432282288474241,
      "confident_error_rate": 0.0043859649122807015,
      "score_mae": 0.4490936795609348
    }
  },
  "prediction_precision": [
    "cuda:bfloat16:fp32_pointer"
  ]
}