{
  "temperature": 1.7458220720291138,
  "selected_step": 0,
  "status": "complete",
  "claim_scope": "Public decision benchmark; no claim of Jev-level intelligence or general calibration",
  "test": {
    "trained": {
      "overall": {
        "n": 2042,
        "accuracy": 0.9289911851126347,
        "nll": 0.2548728303419246,
        "brier_multiclass_sum": 0.11724661735348839,
        "ece_top_label_10_equal_width_bins": 0.042709464978984944,
        "reliability_bins": [
          {
            "count": 0,
            "confidence_sum": 0.0,
            "correct_sum": 0.0
          },
          {
            "count": 0,
            "confidence_sum": 0.0,
            "correct_sum": 0.0
          },
          {
            "count": 0,
            "confidence_sum": 0.0,
            "correct_sum": 0.0
          },
          {
            "count": 1,
            "confidence_sum": 0.3873113691806793,
            "correct_sum": 0.0
          },
          {
            "count": 6,
            "confidence_sum": 2.761519640684128,
            "correct_sum": 1.0
          },
          {
            "count": 25,
            "confidence_sum": 13.707988917827606,
            "correct_sum": 13.0
          },
          {
            "count": 37,
            "confidence_sum": 23.83959299325943,
            "correct_sum": 26.0
          },
          {
            "count": 44,
            "confidence_sum": 33.35772943496704,
            "correct_sum": 29.0
          },
          {
            "count": 71,
            "confidence_sum": 60.919248819351196,
            "correct_sum": 48.0
          },
          {
            "count": 1858,
            "confidence_sum": 1844.918522298336,
            "correct_sum": 1780.0
          }
        ],
        "accuracy_vs_coverage": {
          "0.25": {
            "n": 511,
            "accuracy": 1.0,
            "min_confidence": 0.9999915361404419
          },
          "0.5": {
            "n": 1021,
            "accuracy": 0.9921645445641528,
            "min_confidence": 0.9986754059791565
          },
          "0.75": {
            "n": 1532,
            "accuracy": 0.9843342036553525,
            "min_confidence": 0.9908576607704163
          },
          "1.0": {
            "n": 2042,
            "accuracy": 0.9289911851126347,
            "min_confidence": 0.3873113691806793
          }
        }
      },
      "per_family": {
        "banking": {
          "n": 512,
          "accuracy": 0.978515625,
          "nll": 0.12226520271792657,
          "brier_multiclass_sum": 0.037994640677064956,
          "ece_top_label_10_equal_width_bins": 0.016155527671799064,
          "reliability_bins": [
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 3,
              "confidence_sum": 1.705095112323761,
              "correct_sum": 2.0
            },
            {
              "count": 1,
              "confidence_sum": 0.6073724627494812,
              "correct_sum": 0.0
            },
            {
              "count": 7,
              "confidence_sum": 5.329456865787506,
              "correct_sum": 5.0
            },
            {
              "count": 5,
              "confidence_sum": 4.3502872586250305,
              "correct_sum": 5.0
            },
            {
              "count": 496,
              "confidence_sum": 495.3901832103729,
              "correct_sum": 489.0
            }
          ],
          "accuracy_vs_coverage": {
            "0.25": {
              "n": 128,
              "accuracy": 1.0,
              "min_confidence": 1.0
            },
            "0.5": {
              "n": 256,
              "accuracy": 1.0,
              "min_confidence": 0.9999991655349731
            },
            "0.75": {
              "n": 384,
              "accuracy": 0.9973958333333334,
              "min_confidence": 0.9999103546142578
            },
            "1.0": {
              "n": 512,
              "accuracy": 0.978515625,
              "min_confidence": 0.5397089123725891
            }
          }
        },
        "boolq": {
          "n": 506,
          "accuracy": 0.8952569169960475,
          "nll": 0.30082020614212623,
          "brier_multiclass_sum": 0.1645830420203524,
          "ece_top_label_10_equal_width_bins": 0.061629810352099273,
          "reliability_bins": [
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 8,
              "confidence_sum": 4.3537238240242,
              "correct_sum": 4.0
            },
            {
              "count": 18,
              "confidence_sum": 11.685294926166534,
              "correct_sum": 10.0
            },
            {
              "count": 12,
              "confidence_sum": 9.267768621444702,
              "correct_sum": 7.0
            },
            {
              "count": 29,
              "confidence_sum": 24.727146327495575,
              "correct_sum": 19.0
            },
            {
              "count": 439,
              "confidence_sum": 434.1507503390312,
              "correct_sum": 413.0
            }
          ],
          "accuracy_vs_coverage": {
            "0.25": {
              "n": 127,
              "accuracy": 0.9921259842519685,
              "min_confidence": 0.9984140396118164
            },
            "0.5": {
              "n": 253,
              "accuracy": 0.9960474308300395,
              "min_confidence": 0.995837926864624
            },
            "0.75": {
              "n": 380,
              "accuracy": 0.9631578947368421,
              "min_confidence": 0.9789161086082458
            },
            "1.0": {
              "n": 506,
              "accuracy": 0.8952569169960475,
              "min_confidence": 0.5028058886528015
            }
          }
        },
        "arc": {
          "n": 512,
          "accuracy": 0.94140625,
          "nll": 0.23545075006863606,
          "brier_multiclass_sum": 0.09582074176202018,
          "ece_top_label_10_equal_width_bins": 0.03452872653724626,
          "reliability_bins": [
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 1,
              "confidence_sum": 0.3873113691806793,
              "correct_sum": 0.0
            },
            {
              "count": 5,
              "confidence_sum": 2.285579204559326,
              "correct_sum": 1.0
            },
            {
              "count": 7,
              "confidence_sum": 3.7234585285186768,
              "correct_sum": 3.0
            },
            {
              "count": 10,
              "confidence_sum": 6.485258996486664,
              "correct_sum": 9.0
            },
            {
              "count": 15,
              "confidence_sum": 11.392396628856659,
              "correct_sum": 12.0
            },
            {
              "count": 17,
              "confidence_sum": 14.561253845691681,
              "correct_sum": 12.0
            },
            {
              "count": 457,
              "confidence_sum": 454.59876066446304,
              "correct_sum": 445.0
            }
          ],
          "accuracy_vs_coverage": {
            "0.25": {
              "n": 128,
              "accuracy": 1.0,
              "min_confidence": 0.9999977350234985
            },
            "0.5": {
              "n": 256,
              "accuracy": 1.0,
              "min_confidence": 0.9999246597290039
            },
            "0.75": {
              "n": 384,
              "accuracy": 0.9895833333333334,
              "min_confidence": 0.9971269965171814
            },
            "1.0": {
              "n": 512,
              "accuracy": 0.94140625,
              "min_confidence": 0.3873113691806793
            }
          }
        },
        "snli": {
          "n": 512,
          "accuracy": 0.900390625,
          "nll": 0.3614936082491682,
          "brier_multiclass_sum": 0.1711427686810808,
          "ece_top_label_10_equal_width_bins": 0.07405480305897072,
          "reliability_bins": [
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 1,
              "confidence_sum": 0.47594043612480164,
              "correct_sum": 0.0
            },
            {
              "count": 7,
              "confidence_sum": 3.925711452960968,
              "correct_sum": 4.0
            },
            {
              "count": 8,
              "confidence_sum": 5.0616666078567505,
              "correct_sum": 7.0
            },
            {
              "count": 10,
              "confidence_sum": 7.368107318878174,
              "correct_sum": 5.0
            },
            {
              "count": 20,
              "confidence_sum": 17.28056138753891,
              "correct_sum": 12.0
            },
            {
              "count": 466,
              "confidence_sum": 460.77882808446884,
              "correct_sum": 433.0
            }
          ],
          "accuracy_vs_coverage": {
            "0.25": {
              "n": 128,
              "accuracy": 0.96875,
              "min_confidence": 0.9978107810020447
            },
            "0.5": {
              "n": 256,
              "accuracy": 0.9765625,
              "min_confidence": 0.9943430423736572
            },
            "0.75": {
              "n": 384,
              "accuracy": 0.9609375,
              "min_confidence": 0.9827606678009033
            },
            "1.0": {
              "n": 512,
              "accuracy": 0.900390625,
              "min_confidence": 0.47594043612480164
            }
          }
        }
      }
    },
    "calibrated": {
      "overall": {
        "n": 2042,
        "accuracy": 0.9289911851126347,
        "nll": 0.2050675208059285,
        "brier_multiclass_sum": 0.10981914968288821,
        "ece_top_label_10_equal_width_bins": 0.010821329873058868,
        "reliability_bins": [
          {
            "count": 0,
            "confidence_sum": 0.0,
            "correct_sum": 0.0
          },
          {
            "count": 0,
            "confidence_sum": 0.0,
            "correct_sum": 0.0
          },
          {
            "count": 0,
            "confidence_sum": 0.0,
            "correct_sum": 0.0
          },
          {
            "count": 4,
            "confidence_sum": 1.5274722874164581,
            "correct_sum": 1.0
          },
          {
            "count": 11,
            "confidence_sum": 4.950159549713135,
            "correct_sum": 4.0
          },
          {
            "count": 58,
            "confidence_sum": 32.086635649204254,
            "correct_sum": 40.0
          },
          {
            "count": 58,
            "confidence_sum": 37.935491383075714,
            "correct_sum": 34.0
          },
          {
            "count": 101,
            "confidence_sum": 76.33427423238754,
            "correct_sum": 69.0
          },
          {
            "count": 158,
            "confidence_sum": 135.905029296875,
            "correct_sum": 136.0
          },
          {
            "count": 1652,
            "confidence_sum": 1614.3414230942726,
            "correct_sum": 1613.0
          }
        ],
        "accuracy_vs_coverage": {
          "0.25": {
            "n": 511,
            "accuracy": 1.0,
            "min_confidence": 0.9984586238861084
          },
          "0.5": {
            "n": 1021,
            "accuracy": 0.9921645445641528,
            "min_confidence": 0.976774275302887
          },
          "0.75": {
            "n": 1532,
            "accuracy": 0.9830287206266318,
            "min_confidence": 0.9275727272033691
          },
          "1.0": {
            "n": 2042,
            "accuracy": 0.9289911851126347,
            "min_confidence": 0.36327579617500305
          }
        }
      },
      "per_family": {
        "banking": {
          "n": 512,
          "accuracy": 0.978515625,
          "nll": 0.08680507836434942,
          "brier_multiclass_sum": 0.03854156218229658,
          "ece_top_label_10_equal_width_bins": 0.010919157532043755,
          "reliability_bins": [
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 6,
              "confidence_sum": 3.359699547290802,
              "correct_sum": 4.0
            },
            {
              "count": 5,
              "confidence_sum": 3.226902723312378,
              "correct_sum": 3.0
            },
            {
              "count": 6,
              "confidence_sum": 4.473788321018219,
              "correct_sum": 6.0
            },
            {
              "count": 9,
              "confidence_sum": 7.847302138805389,
              "correct_sum": 8.0
            },
            {
              "count": 486,
              "confidence_sum": 483.04449594020844,
              "correct_sum": 480.0
            }
          ],
          "accuracy_vs_coverage": {
            "0.25": {
              "n": 128,
              "accuracy": 1.0,
              "min_confidence": 0.9999681711196899
            },
            "0.5": {
              "n": 256,
              "accuracy": 1.0,
              "min_confidence": 0.9996205568313599
            },
            "0.75": {
              "n": 384,
              "accuracy": 0.9973958333333334,
              "min_confidence": 0.9949487447738647
            },
            "1.0": {
              "n": 512,
              "accuracy": 0.978515625,
              "min_confidence": 0.5222792029380798
            }
          }
        },
        "boolq": {
          "n": 506,
          "accuracy": 0.8952569169960475,
          "nll": 0.24933799422114145,
          "brier_multiclass_sum": 0.15022579183164184,
          "ece_top_label_10_equal_width_bins": 0.016275467844348652,
          "reliability_bins": [
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 21,
              "confidence_sum": 11.7305428981781,
              "correct_sum": 10.0
            },
            {
              "count": 22,
              "confidence_sum": 14.534841537475586,
              "correct_sum": 15.0
            },
            {
              "count": 34,
              "confidence_sum": 25.722497761249542,
              "correct_sum": 22.0
            },
            {
              "count": 49,
              "confidence_sum": 41.86055135726929,
              "correct_sum": 40.0
            },
            {
              "count": 380,
              "confidence_sum": 365.5433637499809,
              "correct_sum": 366.0
            }
          ],
          "accuracy_vs_coverage": {
            "0.25": {
              "n": 127,
              "accuracy": 0.9921259842519685,
              "min_confidence": 0.9756761789321899
            },
            "0.5": {
              "n": 253,
              "accuracy": 0.9960474308300395,
              "min_confidence": 0.9584143757820129
            },
            "0.75": {
              "n": 380,
              "accuracy": 0.9631578947368421,
              "min_confidence": 0.9001014232635498
            },
            "1.0": {
              "n": 506,
              "accuracy": 0.8952569169960475,
              "min_confidence": 0.5016071796417236
            }
          }
        },
        "arc": {
          "n": 512,
          "accuracy": 0.94140625,
          "nll": 0.19397132420263175,
          "brier_multiclass_sum": 0.09487676147069625,
          "ece_top_label_10_equal_width_bins": 0.02615507983136922,
          "reliability_bins": [
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 4,
              "confidence_sum": 1.5274722874164581,
              "correct_sum": 1.0
            },
            {
              "count": 9,
              "confidence_sum": 4.047605782747269,
              "correct_sum": 4.0
            },
            {
              "count": 15,
              "confidence_sum": 8.381983816623688,
              "correct_sum": 14.0
            },
            {
              "count": 17,
              "confidence_sum": 11.08071506023407,
              "correct_sum": 9.0
            },
            {
              "count": 26,
              "confidence_sum": 19.58449637889862,
              "correct_sum": 22.0
            },
            {
              "count": 27,
              "confidence_sum": 22.931695699691772,
              "correct_sum": 24.0
            },
            {
              "count": 414,
              "confidence_sum": 409.6337836384773,
              "correct_sum": 408.0
            }
          ],
          "accuracy_vs_coverage": {
            "0.25": {
              "n": 128,
              "accuracy": 1.0,
              "min_confidence": 0.9992086291313171
            },
            "0.5": {
              "n": 256,
              "accuracy": 1.0,
              "min_confidence": 0.9945464134216309
            },
            "0.75": {
              "n": 384,
              "accuracy": 0.9895833333333334,
              "min_confidence": 0.9568338990211487
            },
            "1.0": {
              "n": 512,
              "accuracy": 0.94140625,
              "min_confidence": 0.36327579617500305
            }
          }
        },
        "snli": {
          "n": 512,
          "accuracy": 0.900390625,
          "nll": 0.29067448104592586,
          "brier_multiclass_sum": 0.15610599858459887,
          "ece_top_label_10_equal_width_bins": 0.03427618817659095,
          "reliability_bins": [
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 2,
              "confidence_sum": 0.9025537669658661,
              "correct_sum": 0.0
            },
            {
              "count": 16,
              "confidence_sum": 8.614409387111664,
              "correct_sum": 12.0
            },
            {
              "count": 14,
              "confidence_sum": 9.09303206205368,
              "correct_sum": 7.0
            },
            {
              "count": 35,
              "confidence_sum": 26.55349177122116,
              "correct_sum": 19.0
            },
            {
              "count": 73,
              "confidence_sum": 63.26548010110855,
              "correct_sum": 64.0
            },
            {
              "count": 372,
              "confidence_sum": 356.1197797656059,
              "correct_sum": 359.0
            }
          ],
          "accuracy_vs_coverage": {
            "0.25": {
              "n": 128,
              "accuracy": 0.9765625,
              "min_confidence": 0.9674575924873352
            },
            "0.5": {
              "n": 256,
              "accuracy": 0.9765625,
              "min_confidence": 0.9432480931282043
            },
            "0.75": {
              "n": 384,
              "accuracy": 0.9583333333333334,
              "min_confidence": 0.8896382451057434
            },
            "1.0": {
              "n": 512,
              "accuracy": 0.900390625,
              "min_confidence": 0.4136597514152527
            }
          }
        }
      }
    },
    "base": {
      "overall": {
        "n": 2042,
        "accuracy": 0.8447600391772772,
        "nll": 1.6827827308918881,
        "brier_multiclass_sum": 0.2927494974423544,
        "ece_top_label_10_equal_width_bins": 0.13822684455279854,
        "reliability_bins": [
          {
            "count": 0,
            "confidence_sum": 0.0,
            "correct_sum": 0.0
          },
          {
            "count": 0,
            "confidence_sum": 0.0,
            "correct_sum": 0.0
          },
          {
            "count": 0,
            "confidence_sum": 0.0,
            "correct_sum": 0.0
          },
          {
            "count": 2,
            "confidence_sum": 0.732365071773529,
            "correct_sum": 0.0
          },
          {
            "count": 4,
            "confidence_sum": 1.8684652149677277,
            "correct_sum": 2.0
          },
          {
            "count": 25,
            "confidence_sum": 13.689240455627441,
            "correct_sum": 17.0
          },
          {
            "count": 18,
            "confidence_sum": 11.746700882911682,
            "correct_sum": 11.0
          },
          {
            "count": 36,
            "confidence_sum": 27.12858122587204,
            "correct_sum": 20.0
          },
          {
            "count": 46,
            "confidence_sum": 39.98839032649994,
            "correct_sum": 26.0
          },
          {
            "count": 1911,
            "confidence_sum": 1905.2208847403526,
            "correct_sum": 1649.0
          }
        ],
        "accuracy_vs_coverage": {
          "0.25": {
            "n": 511,
            "accuracy": 0.9354207436399217,
            "min_confidence": 1.0
          },
          "0.5": {
            "n": 1021,
            "accuracy": 0.9422135161606269,
            "min_confidence": 1.0
          },
          "0.75": {
            "n": 1532,
            "accuracy": 0.9007832898172323,
            "min_confidence": 0.9998598098754883
          },
          "1.0": {
            "n": 2042,
            "accuracy": 0.8447600391772772,
            "min_confidence": 0.3561801314353943
          }
        }
      },
      "per_family": {
        "banking": {
          "n": 512,
          "accuracy": 0.8828125,
          "nll": 0.8035370189185151,
          "brier_multiclass_sum": 0.2032958888533993,
          "ece_top_label_10_equal_width_bins": 0.08182763156946748,
          "reliability_bins": [
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 2,
              "confidence_sum": 0.732365071773529,
              "correct_sum": 0.0
            },
            {
              "count": 2,
              "confidence_sum": 0.9544804692268372,
              "correct_sum": 1.0
            },
            {
              "count": 9,
              "confidence_sum": 4.947599828243256,
              "correct_sum": 5.0
            },
            {
              "count": 8,
              "confidence_sum": 5.206531882286072,
              "correct_sum": 4.0
            },
            {
              "count": 16,
              "confidence_sum": 11.929070949554443,
              "correct_sum": 9.0
            },
            {
              "count": 19,
              "confidence_sum": 16.535522401332855,
              "correct_sum": 15.0
            },
            {
              "count": 456,
              "confidence_sum": 453.39433735609055,
              "correct_sum": 418.0
            }
          ],
          "accuracy_vs_coverage": {
            "0.25": {
              "n": 128,
              "accuracy": 0.9921875,
              "min_confidence": 1.0
            },
            "0.5": {
              "n": 256,
              "accuracy": 0.9765625,
              "min_confidence": 0.9999994039535522
            },
            "0.75": {
              "n": 384,
              "accuracy": 0.9401041666666666,
              "min_confidence": 0.9933443069458008
            },
            "1.0": {
              "n": 512,
              "accuracy": 0.8828125,
              "min_confidence": 0.3561801314353943
            }
          }
        },
        "boolq": {
          "n": 506,
          "accuracy": 0.8557312252964426,
          "nll": 2.0259620797809275,
          "brier_multiclass_sum": 0.2843932332075561,
          "ece_top_label_10_equal_width_bins": 0.14410379540778903,
          "reliability_bins": [
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 4,
              "confidence_sum": 2.20137357711792,
              "correct_sum": 4.0
            },
            {
              "count": 1,
              "confidence_sum": 0.645257830619812,
              "correct_sum": 1.0
            },
            {
              "count": 3,
              "confidence_sum": 2.2964596152305603,
              "correct_sum": 1.0
            },
            {
              "count": 6,
              "confidence_sum": 5.2520251870155334,
              "correct_sum": 2.0
            },
            {
              "count": 492,
              "confidence_sum": 491.2146670818329,
              "correct_sum": 425.0
            }
          ],
          "accuracy_vs_coverage": {
            "0.25": {
              "n": 127,
              "accuracy": 0.8976377952755905,
              "min_confidence": 1.0
            },
            "0.5": {
              "n": 253,
              "accuracy": 0.9169960474308301,
              "min_confidence": 1.0
            },
            "0.75": {
              "n": 380,
              "accuracy": 0.9289473684210526,
              "min_confidence": 0.9999997615814209
            },
            "1.0": {
              "n": 506,
              "accuracy": 0.8557312252964426,
              "min_confidence": 0.5300353169441223
            }
          }
        },
        "arc": {
          "n": 512,
          "accuracy": 0.9296875,
          "nll": 0.8372381083637264,
          "brier_multiclass_sum": 0.13284523221990113,
          "ece_top_label_10_equal_width_bins": 0.06219080294249579,
          "reliability_bins": [
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 2,
              "confidence_sum": 0.9139847457408905,
              "correct_sum": 1.0
            },
            {
              "count": 2,
              "confidence_sum": 1.0453534722328186,
              "correct_sum": 1.0
            },
            {
              "count": 5,
              "confidence_sum": 3.2189850211143494,
              "correct_sum": 3.0
            },
            {
              "count": 6,
              "confidence_sum": 4.524070084095001,
              "correct_sum": 6.0
            },
            {
              "count": 8,
              "confidence_sum": 6.894674122333527,
              "correct_sum": 5.0
            },
            {
              "count": 489,
              "confidence_sum": 488.12073332071304,
              "correct_sum": 460.0
            }
          ],
          "accuracy_vs_coverage": {
            "0.25": {
              "n": 128,
              "accuracy": 0.984375,
              "min_confidence": 1.0
            },
            "0.5": {
              "n": 256,
              "accuracy": 0.98828125,
              "min_confidence": 1.0
            },
            "0.75": {
              "n": 384,
              "accuracy": 0.9817708333333334,
              "min_confidence": 0.9999997615814209
            },
            "1.0": {
              "n": 512,
              "accuracy": 0.9296875,
              "min_confidence": 0.4202010929584503
            }
          }
        },
        "snli": {
          "n": 512,
          "accuracy": 0.7109375,
          "nll": 3.0684153494991766,
          "brier_multiclass_sum": 0.5503657105170595,
          "ece_top_label_10_equal_width_bins": 0.2734481571242213,
          "reliability_bins": [
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 10,
              "confidence_sum": 5.494913578033447,
              "correct_sum": 7.0
            },
            {
              "count": 4,
              "confidence_sum": 2.675926148891449,
              "correct_sum": 3.0
            },
            {
              "count": 11,
              "confidence_sum": 8.378980576992035,
              "correct_sum": 4.0
            },
            {
              "count": 13,
              "confidence_sum": 11.306168615818024,
              "correct_sum": 4.0
            },
            {
              "count": 474,
              "confidence_sum": 472.49114698171616,
              "correct_sum": 346.0
            }
          ],
          "accuracy_vs_coverage": {
            "0.25": {
              "n": 128,
              "accuracy": 0.8125,
              "min_confidence": 1.0
            },
            "0.5": {
              "n": 256,
              "accuracy": 0.80078125,
              "min_confidence": 0.9999966621398926
            },
            "0.75": {
              "n": 384,
              "accuracy": 0.7630208333333334,
              "min_confidence": 0.9995673298835754
            },
            "1.0": {
              "n": 512,
              "accuracy": 0.7109375,
              "min_confidence": 0.5036829113960266
            }
          }
        }
      }
    },
    "base_calibrated": {
      "overall": {
        "n": 2042,
        "accuracy": 0.8447600391772772,
        "nll": 0.452739927518432,
        "brier_multiclass_sum": 0.25478263453519945,
        "ece_top_label_10_equal_width_bins": 0.06263403555088248,
        "reliability_bins": [
          {
            "count": 0,
            "confidence_sum": 0.0,
            "correct_sum": 0.0
          },
          {
            "count": 0,
            "confidence_sum": 0.0,
            "correct_sum": 0.0
          },
          {
            "count": 8,
            "confidence_sum": 2.2937141954898834,
            "correct_sum": 3.0
          },
          {
            "count": 62,
            "confidence_sum": 21.780550003051758,
            "correct_sum": 42.0
          },
          {
            "count": 87,
            "confidence_sum": 39.44270572066307,
            "correct_sum": 70.0
          },
          {
            "count": 147,
            "confidence_sum": 80.74226522445679,
            "correct_sum": 103.0
          },
          {
            "count": 150,
            "confidence_sum": 97.46173959970474,
            "correct_sum": 93.0
          },
          {
            "count": 199,
            "confidence_sum": 150.20955330133438,
            "correct_sum": 151.0
          },
          {
            "count": 311,
            "confidence_sum": 265.94291496276855,
            "correct_sum": 250.0
          },
          {
            "count": 1078,
            "confidence_sum": 1045.9628344774246,
            "correct_sum": 1013.0
          }
        ],
        "accuracy_vs_coverage": {
          "0.25": {
            "n": 511,
            "accuracy": 0.9843444227005871,
            "min_confidence": 0.9829810857772827
          },
          "0.5": {
            "n": 1021,
            "accuracy": 0.9480901077375122,
            "min_confidence": 0.9118173122406006
          },
          "0.75": {
            "n": 1532,
            "accuracy": 0.8955613577023499,
            "min_confidence": 0.7337803840637207
          },
          "1.0": {
            "n": 2042,
            "accuracy": 0.8447600391772772,
            "min_confidence": 0.26615577936172485
          }
        }
      },
      "per_family": {
        "banking": {
          "n": 512,
          "accuracy": 0.8828125,
          "nll": 0.4803847811426749,
          "brier_multiclass_sum": 0.24231384330718306,
          "ece_top_label_10_equal_width_bins": 0.15125263947993517,
          "reliability_bins": [
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 6,
              "confidence_sum": 1.7256377339363098,
              "correct_sum": 2.0
            },
            {
              "count": 47,
              "confidence_sum": 16.507239133119583,
              "correct_sum": 34.0
            },
            {
              "count": 61,
              "confidence_sum": 27.371423810720444,
              "correct_sum": 50.0
            },
            {
              "count": 56,
              "confidence_sum": 30.601767003536224,
              "correct_sum": 47.0
            },
            {
              "count": 47,
              "confidence_sum": 30.510030925273895,
              "correct_sum": 34.0
            },
            {
              "count": 38,
              "confidence_sum": 28.63151115179062,
              "correct_sum": 35.0
            },
            {
              "count": 68,
              "confidence_sum": 58.034259259700775,
              "correct_sum": 66.0
            },
            {
              "count": 189,
              "confidence_sum": 181.17677956819534,
              "correct_sum": 184.0
            }
          ],
          "accuracy_vs_coverage": {
            "0.25": {
              "n": 128,
              "accuracy": 0.984375,
              "min_confidence": 0.9501156210899353
            },
            "0.5": {
              "n": 256,
              "accuracy": 0.9765625,
              "min_confidence": 0.8002985715866089
            },
            "0.75": {
              "n": 384,
              "accuracy": 0.9322916666666666,
              "min_confidence": 0.5225431323051453
            },
            "1.0": {
              "n": 512,
              "accuracy": 0.8828125,
              "min_confidence": 0.26615577936172485
            }
          }
        },
        "boolq": {
          "n": 506,
          "accuracy": 0.8557312252964426,
          "nll": 0.3844228962146085,
          "brier_multiclass_sum": 0.22706346727506466,
          "ece_top_label_10_equal_width_bins": 0.07297720126954935,
          "reliability_bins": [
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 18,
              "confidence_sum": 9.970384955406189,
              "correct_sum": 12.0
            },
            {
              "count": 29,
              "confidence_sum": 18.950466096401215,
              "correct_sum": 18.0
            },
            {
              "count": 29,
              "confidence_sum": 21.715792536735535,
              "correct_sum": 16.0
            },
            {
              "count": 50,
              "confidence_sum": 42.18575972318649,
              "correct_sum": 34.0
            },
            {
              "count": 380,
              "confidence_sum": 373.0448304414749,
              "correct_sum": 353.0
            }
          ],
          "accuracy_vs_coverage": {
            "0.25": {
              "n": 127,
              "accuracy": 0.9921259842519685,
              "min_confidence": 0.9963729381561279
            },
            "0.5": {
              "n": 253,
              "accuracy": 0.9802371541501976,
              "min_confidence": 0.9850993752479553
            },
            "0.75": {
              "n": 380,
              "accuracy": 0.9289473684210526,
              "min_confidence": 0.9002280831336975
            },
            "1.0": {
              "n": 506,
              "accuracy": 0.8557312252964426,
              "min_confidence": 0.5043465495109558
            }
          }
        },
        "arc": {
          "n": 512,
          "accuracy": 0.9296875,
          "nll": 0.2752359951973631,
          "brier_multiclass_sum": 0.13593068181824372,
          "ece_top_label_10_equal_width_bins": 0.05345189612125978,
          "reliability_bins": [
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 2,
              "confidence_sum": 0.5680764615535736,
              "correct_sum": 1.0
            },
            {
              "count": 15,
              "confidence_sum": 5.273310869932175,
              "correct_sum": 8.0
            },
            {
              "count": 19,
              "confidence_sum": 8.764264017343521,
              "correct_sum": 16.0
            },
            {
              "count": 25,
              "confidence_sum": 13.797979295253754,
              "correct_sum": 21.0
            },
            {
              "count": 20,
              "confidence_sum": 13.02436488866806,
              "correct_sum": 13.0
            },
            {
              "count": 28,
              "confidence_sum": 20.95827430486679,
              "correct_sum": 27.0
            },
            {
              "count": 52,
              "confidence_sum": 44.90535968542099,
              "correct_sum": 44.0
            },
            {
              "count": 351,
              "confidence_sum": 343.20044881105423,
              "correct_sum": 346.0
            }
          ],
          "accuracy_vs_coverage": {
            "0.25": {
              "n": 128,
              "accuracy": 1.0,
              "min_confidence": 0.9898765683174133
            },
            "0.5": {
              "n": 256,
              "accuracy": 0.99609375,
              "min_confidence": 0.9743005633354187
            },
            "0.75": {
              "n": 384,
              "accuracy": 0.9791666666666666,
              "min_confidence": 0.8585602045059204
            },
            "1.0": {
              "n": 512,
              "accuracy": 0.9296875,
              "min_confidence": 0.27811235189437866
            }
          }
        },
        "snli": {
          "n": 512,
          "accuracy": 0.7109375,
          "nll": 0.6701154473084898,
          "brier_multiclass_sum": 0.4134977117489767,
          "ece_top_label_10_equal_width_bins": 0.0982505488791503,
          "reliability_bins": [
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 7,
              "confidence_sum": 3.307017892599106,
              "correct_sum": 4.0
            },
            {
              "count": 48,
              "confidence_sum": 26.37213397026062,
              "correct_sum": 23.0
            },
            {
              "count": 54,
              "confidence_sum": 34.97687768936157,
              "correct_sum": 28.0
            },
            {
              "count": 104,
              "confidence_sum": 78.90397530794144,
              "correct_sum": 73.0
            },
            {
              "count": 141,
              "confidence_sum": 120.8175362944603,
              "correct_sum": 106.0
            },
            {
              "count": 158,
              "confidence_sum": 148.54077565670013,
              "correct_sum": 130.0
            }
          ],
          "accuracy_vs_coverage": {
            "0.25": {
              "n": 128,
              "accuracy": 0.859375,
              "min_confidence": 0.9148098826408386
            },
            "0.5": {
              "n": 256,
              "accuracy": 0.80859375,
              "min_confidence": 0.8419564962387085
            },
            "0.75": {
              "n": 384,
              "accuracy": 0.7708333333333334,
              "min_confidence": 0.7315099835395813
            },
            "1.0": {
              "n": 512,
              "accuracy": 0.7109375,
              "min_confidence": 0.43296143412590027
            }
          }
        }
      }
    },
    "calibrated_difference_95pct": {
      "method": "400 stratified source-group bootstrap resamples; decision-weighted tuned minus base",
      "point_delta": {
        "accuracy": 0.0842311459353575,
        "nll": -0.2476724067125035,
        "brier": -0.1449634848523113
      },
      "accuracy": [
        0.06854799216454456,
        0.098922624877571
      ],
      "nll": [
        -0.27686958949075574,
        -0.21881351599842205
      ],
      "brier": [
        -0.16374186238337493,
        -0.1257739683435538
      ]
    }
  },
  "holdout": {
    "trained": {
      "overall": {
        "n": 768,
        "accuracy": 0.7291666666666666,
        "nll": 0.9046048978141895,
        "brier_multiclass_sum": 0.41865507801212026,
        "ece_top_label_10_equal_width_bins": 0.1554523635810862,
        "reliability_bins": [
          {
            "count": 0,
            "confidence_sum": 0.0,
            "correct_sum": 0.0
          },
          {
            "count": 0,
            "confidence_sum": 0.0,
            "correct_sum": 0.0
          },
          {
            "count": 0,
            "confidence_sum": 0.0,
            "correct_sum": 0.0
          },
          {
            "count": 3,
            "confidence_sum": 1.1163514852523804,
            "correct_sum": 0.0
          },
          {
            "count": 13,
            "confidence_sum": 6.152260005474091,
            "correct_sum": 3.0
          },
          {
            "count": 54,
            "confidence_sum": 29.515426993370056,
            "correct_sum": 27.0
          },
          {
            "count": 50,
            "confidence_sum": 32.41912567615509,
            "correct_sum": 27.0
          },
          {
            "count": 65,
            "confidence_sum": 48.71525001525879,
            "correct_sum": 33.0
          },
          {
            "count": 84,
            "confidence_sum": 71.8504301905632,
            "correct_sum": 44.0
          },
          {
            "count": 499,
            "confidence_sum": 489.6185708642006,
            "correct_sum": 426.0
          }
        ],
        "accuracy_vs_coverage": {
          "0.25": {
            "n": 192,
            "accuracy": 0.9375,
            "min_confidence": 0.9978193044662476
          },
          "0.5": {
            "n": 384,
            "accuracy": 0.8828125,
            "min_confidence": 0.9679536819458008
          },
          "0.75": {
            "n": 576,
            "accuracy": 0.8107638888888888,
            "min_confidence": 0.8080979585647583
          },
          "1.0": {
            "n": 768,
            "accuracy": 0.7291666666666666,
            "min_confidence": 0.3503410816192627
          }
        }
      },
      "per_family": {
        "social": {
          "n": 768,
          "accuracy": 0.7291666666666666,
          "nll": 0.9046048978141895,
          "brier_multiclass_sum": 0.41865507801212026,
          "ece_top_label_10_equal_width_bins": 0.1554523635810862,
          "reliability_bins": [
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 3,
              "confidence_sum": 1.1163514852523804,
              "correct_sum": 0.0
            },
            {
              "count": 13,
              "confidence_sum": 6.152260005474091,
              "correct_sum": 3.0
            },
            {
              "count": 54,
              "confidence_sum": 29.515426993370056,
              "correct_sum": 27.0
            },
            {
              "count": 50,
              "confidence_sum": 32.41912567615509,
              "correct_sum": 27.0
            },
            {
              "count": 65,
              "confidence_sum": 48.71525001525879,
              "correct_sum": 33.0
            },
            {
              "count": 84,
              "confidence_sum": 71.8504301905632,
              "correct_sum": 44.0
            },
            {
              "count": 499,
              "confidence_sum": 489.6185708642006,
              "correct_sum": 426.0
            }
          ],
          "accuracy_vs_coverage": {
            "0.25": {
              "n": 192,
              "accuracy": 0.9375,
              "min_confidence": 0.9978193044662476
            },
            "0.5": {
              "n": 384,
              "accuracy": 0.8828125,
              "min_confidence": 0.9679536819458008
            },
            "0.75": {
              "n": 576,
              "accuracy": 0.8107638888888888,
              "min_confidence": 0.8080979585647583
            },
            "1.0": {
              "n": 768,
              "accuracy": 0.7291666666666666,
              "min_confidence": 0.3503410816192627
            }
          }
        }
      }
    },
    "calibrated": {
      "overall": {
        "n": 768,
        "accuracy": 0.7291666666666666,
        "nll": 0.6782619158996491,
        "brier_multiclass_sum": 0.37973740706466513,
        "ece_top_label_10_equal_width_bins": 0.08299602890231957,
        "reliability_bins": [
          {
            "count": 0,
            "confidence_sum": 0.0,
            "correct_sum": 0.0
          },
          {
            "count": 0,
            "confidence_sum": 0.0,
            "correct_sum": 0.0
          },
          {
            "count": 0,
            "confidence_sum": 0.0,
            "correct_sum": 0.0
          },
          {
            "count": 4,
            "confidence_sum": 1.4600901305675507,
            "correct_sum": 0.0
          },
          {
            "count": 43,
            "confidence_sum": 19.645778954029083,
            "correct_sum": 15.0
          },
          {
            "count": 84,
            "confidence_sum": 46.113935708999634,
            "correct_sum": 48.0
          },
          {
            "count": 84,
            "confidence_sum": 54.28614550828934,
            "correct_sum": 39.0
          },
          {
            "count": 97,
            "confidence_sum": 72.91012477874756,
            "correct_sum": 59.0
          },
          {
            "count": 130,
            "confidence_sum": 110.77482378482819,
            "correct_sum": 107.0
          },
          {
            "count": 326,
            "confidence_sum": 314.77792274951935,
            "correct_sum": 292.0
          }
        ],
        "accuracy_vs_coverage": {
          "0.25": {
            "n": 192,
            "accuracy": 0.9375,
            "min_confidence": 0.9662192463874817
          },
          "0.5": {
            "n": 384,
            "accuracy": 0.8854166666666666,
            "min_confidence": 0.8613662123680115
          },
          "0.75": {
            "n": 576,
            "accuracy": 0.8072916666666666,
            "min_confidence": 0.670463502407074
          },
          "1.0": {
            "n": 768,
            "accuracy": 0.7291666666666666,
            "min_confidence": 0.3430308401584625
          }
        }
      },
      "per_family": {
        "social": {
          "n": 768,
          "accuracy": 0.7291666666666666,
          "nll": 0.6782619158996491,
          "brier_multiclass_sum": 0.37973740706466513,
          "ece_top_label_10_equal_width_bins": 0.08299602890231957,
          "reliability_bins": [
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 4,
              "confidence_sum": 1.4600901305675507,
              "correct_sum": 0.0
            },
            {
              "count": 43,
              "confidence_sum": 19.645778954029083,
              "correct_sum": 15.0
            },
            {
              "count": 84,
              "confidence_sum": 46.113935708999634,
              "correct_sum": 48.0
            },
            {
              "count": 84,
              "confidence_sum": 54.28614550828934,
              "correct_sum": 39.0
            },
            {
              "count": 97,
              "confidence_sum": 72.91012477874756,
              "correct_sum": 59.0
            },
            {
              "count": 130,
              "confidence_sum": 110.77482378482819,
              "correct_sum": 107.0
            },
            {
              "count": 326,
              "confidence_sum": 314.77792274951935,
              "correct_sum": 292.0
            }
          ],
          "accuracy_vs_coverage": {
            "0.25": {
              "n": 192,
              "accuracy": 0.9375,
              "min_confidence": 0.9662192463874817
            },
            "0.5": {
              "n": 384,
              "accuracy": 0.8854166666666666,
              "min_confidence": 0.8613662123680115
            },
            "0.75": {
              "n": 576,
              "accuracy": 0.8072916666666666,
              "min_confidence": 0.670463502407074
            },
            "1.0": {
              "n": 768,
              "accuracy": 0.7291666666666666,
              "min_confidence": 0.3430308401584625
            }
          }
        }
      }
    },
    "base": {
      "overall": {
        "n": 768,
        "accuracy": 0.703125,
        "nll": 2.087191693346451,
        "brier_multiclass_sum": 0.5129237150352639,
        "ece_top_label_10_equal_width_bins": 0.22570987732615322,
        "reliability_bins": [
          {
            "count": 0,
            "confidence_sum": 0.0,
            "correct_sum": 0.0
          },
          {
            "count": 0,
            "confidence_sum": 0.0,
            "correct_sum": 0.0
          },
          {
            "count": 0,
            "confidence_sum": 0.0,
            "correct_sum": 0.0
          },
          {
            "count": 1,
            "confidence_sum": 0.3542192578315735,
            "correct_sum": 1.0
          },
          {
            "count": 18,
            "confidence_sum": 8.336367458105087,
            "correct_sum": 10.0
          },
          {
            "count": 39,
            "confidence_sum": 21.718455612659454,
            "correct_sum": 17.0
          },
          {
            "count": 22,
            "confidence_sum": 14.276574194431305,
            "correct_sum": 13.0
          },
          {
            "count": 33,
            "confidence_sum": 24.695184469223022,
            "correct_sum": 15.0
          },
          {
            "count": 62,
            "confidence_sum": 52.76100742816925,
            "correct_sum": 33.0
          },
          {
            "count": 593,
            "confidence_sum": 586.5845507979393,
            "correct_sum": 451.0
          }
        ],
        "accuracy_vs_coverage": {
          "0.25": {
            "n": 192,
            "accuracy": 0.90625,
            "min_confidence": 0.9999998807907104
          },
          "0.5": {
            "n": 384,
            "accuracy": 0.8255208333333334,
            "min_confidence": 0.9991338849067688
          },
          "0.75": {
            "n": 576,
            "accuracy": 0.7690972222222222,
            "min_confidence": 0.9154785871505737
          },
          "1.0": {
            "n": 768,
            "accuracy": 0.703125,
            "min_confidence": 0.3542192578315735
          }
        }
      },
      "per_family": {
        "social": {
          "n": 768,
          "accuracy": 0.703125,
          "nll": 2.087191693346451,
          "brier_multiclass_sum": 0.5129237150352639,
          "ece_top_label_10_equal_width_bins": 0.22570987732615322,
          "reliability_bins": [
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 1,
              "confidence_sum": 0.3542192578315735,
              "correct_sum": 1.0
            },
            {
              "count": 18,
              "confidence_sum": 8.336367458105087,
              "correct_sum": 10.0
            },
            {
              "count": 39,
              "confidence_sum": 21.718455612659454,
              "correct_sum": 17.0
            },
            {
              "count": 22,
              "confidence_sum": 14.276574194431305,
              "correct_sum": 13.0
            },
            {
              "count": 33,
              "confidence_sum": 24.695184469223022,
              "correct_sum": 15.0
            },
            {
              "count": 62,
              "confidence_sum": 52.76100742816925,
              "correct_sum": 33.0
            },
            {
              "count": 593,
              "confidence_sum": 586.5845507979393,
              "correct_sum": 451.0
            }
          ],
          "accuracy_vs_coverage": {
            "0.25": {
              "n": 192,
              "accuracy": 0.90625,
              "min_confidence": 0.9999998807907104
            },
            "0.5": {
              "n": 384,
              "accuracy": 0.8255208333333334,
              "min_confidence": 0.9991338849067688
            },
            "0.75": {
              "n": 576,
              "accuracy": 0.7690972222222222,
              "min_confidence": 0.9154785871505737
            },
            "1.0": {
              "n": 768,
              "accuracy": 0.703125,
              "min_confidence": 0.3542192578315735
            }
          }
        }
      }
    },
    "base_calibrated": {
      "overall": {
        "n": 768,
        "accuracy": 0.703125,
        "nll": 0.7425018713104995,
        "brier_multiclass_sum": 0.4306530777581018,
        "ece_top_label_10_equal_width_bins": 0.08665639813989401,
        "reliability_bins": [
          {
            "count": 0,
            "confidence_sum": 0.0,
            "correct_sum": 0.0
          },
          {
            "count": 0,
            "confidence_sum": 0.0,
            "correct_sum": 0.0
          },
          {
            "count": 0,
            "confidence_sum": 0.0,
            "correct_sum": 0.0
          },
          {
            "count": 69,
            "confidence_sum": 25.828479051589966,
            "correct_sum": 33.0
          },
          {
            "count": 165,
            "confidence_sum": 74.53762590885162,
            "correct_sum": 90.0
          },
          {
            "count": 107,
            "confidence_sum": 58.80419135093689,
            "correct_sum": 72.0
          },
          {
            "count": 95,
            "confidence_sum": 61.89751034975052,
            "correct_sum": 73.0
          },
          {
            "count": 86,
            "confidence_sum": 64.67725455760956,
            "correct_sum": 60.0
          },
          {
            "count": 86,
            "confidence_sum": 73.19006419181824,
            "correct_sum": 66.0
          },
          {
            "count": 160,
            "confidence_sum": 153.7526016831398,
            "correct_sum": 146.0
          }
        ],
        "accuracy_vs_coverage": {
          "0.25": {
            "n": 192,
            "accuracy": 0.9010416666666666,
            "min_confidence": 0.8640128970146179
          },
          "0.5": {
            "n": 384,
            "accuracy": 0.8098958333333334,
            "min_confidence": 0.6482440233230591
          },
          "0.75": {
            "n": 576,
            "accuracy": 0.7638888888888888,
            "min_confidence": 0.4738004803657532
          },
          "1.0": {
            "n": 768,
            "accuracy": 0.703125,
            "min_confidence": 0.33634451031684875
          }
        }
      },
      "per_family": {
        "social": {
          "n": 768,
          "accuracy": 0.703125,
          "nll": 0.7425018713104995,
          "brier_multiclass_sum": 0.4306530777581018,
          "ece_top_label_10_equal_width_bins": 0.08665639813989401,
          "reliability_bins": [
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 0,
              "confidence_sum": 0.0,
              "correct_sum": 0.0
            },
            {
              "count": 69,
              "confidence_sum": 25.828479051589966,
              "correct_sum": 33.0
            },
            {
              "count": 165,
              "confidence_sum": 74.53762590885162,
              "correct_sum": 90.0
            },
            {
              "count": 107,
              "confidence_sum": 58.80419135093689,
              "correct_sum": 72.0
            },
            {
              "count": 95,
              "confidence_sum": 61.89751034975052,
              "correct_sum": 73.0
            },
            {
              "count": 86,
              "confidence_sum": 64.67725455760956,
              "correct_sum": 60.0
            },
            {
              "count": 86,
              "confidence_sum": 73.19006419181824,
              "correct_sum": 66.0
            },
            {
              "count": 160,
              "confidence_sum": 153.7526016831398,
              "correct_sum": 146.0
            }
          ],
          "accuracy_vs_coverage": {
            "0.25": {
              "n": 192,
              "accuracy": 0.9010416666666666,
              "min_confidence": 0.8640128970146179
            },
            "0.5": {
              "n": 384,
              "accuracy": 0.8098958333333334,
              "min_confidence": 0.6482440233230591
            },
            "0.75": {
              "n": 576,
              "accuracy": 0.7638888888888888,
              "min_confidence": 0.4738004803657532
            },
            "1.0": {
              "n": 768,
              "accuracy": 0.703125,
              "min_confidence": 0.33634451031684875
            }
          }
        }
      }
    },
    "calibrated_difference_95pct": {
      "method": "400 stratified source-group bootstrap resamples; decision-weighted tuned minus base",
      "point_delta": {
        "accuracy": 0.026041666666666668,
        "nll": -0.0642399554108503,
        "brier": -0.050915670693436714
      },
      "accuracy": [
        0.0013020833333333333,
        0.05341796874999997
      ],
      "nll": [
        -0.11439402483851543,
        -0.016014919936165502
      ],
      "brier": [
        -0.0754793339156752,
        -0.02637446983897743
      ]
    }
  },
  "base_temperature": 6.918309211730957,
  "model_sha256": "e270e3da905604d97bf5a8f380ea308133403d1c4790a5c012cb1c12e9b6f348",
  "peak_cuda_allocated_bytes": 16356398592,
  "peak_cuda_reserved_bytes": 16393437184,
  "final_mem_available_bytes": 94531235840
}