{
  "objective": -0.5498728060426441,
  "paired_flip": {
    "pairs": 80,
    "flip_rate": 0.8375,
    "both_correct_rate": 0.825,
    "invariant_pairs": 32,
    "invariance_rate": 1.0,
    "invariant_both_correct_rate": 0.84375
  },
  "unknowable": null,
  "clean": {
    "n": 1264,
    "nll": 0.6468405219068211,
    "acc": 0.8267405063291139,
    "ece": 0.10212917934363491,
    "brier": 0.2739376394391063,
    "mean_conf": 0.9287298947829927,
    "confident_error_rate": 0.07041139240506329,
    "coverage_at_0_9": 0.7705696202531646,
    "accuracy_at_0_9": 0.9086242299794661,
    "coverage_at_5pct_error": 0.5893987341772152,
    "coverage_at_1pct_error": 0.3188291139240506,
    "aurc": 0.05376999371647826,
    "error_rate_at_0_9": 0.09137577002053388,
    "confidence_bias": 0.10198938845387884,
    "top_bins": {
      "0.9": {
        "n": 974,
        "errors": 89,
        "error_rate": 0.09137577002053388
      },
      "0.95": {
        "n": 896,
        "errors": 67,
        "error_rate": 0.07477678571428571
      },
      "0.99": {
        "n": 756,
        "errors": 39,
        "error_rate": 0.051587301587301584
      }
    },
    "selective": {
      "0.5": {
        "coverage": 0.5,
        "accuracy": 0.9667721518987342,
        "confidence_cutoff": 0.9975252105885826
      },
      "0.8": {
        "coverage": 0.8006329113924051,
        "accuracy": 0.8972332015810277,
        "confidence_cutoff": 0.873414767882448
      }
    },
    "score_mae": 0.5003761251952392,
    "ranked_probability_score": 0.09176394701019726
  },
  "tasks": {
    "agnews": {
      "n": 80,
      "nll": 0.9449896733484138,
      "acc": 0.825,
      "ece": 0.14358549310113145,
      "brier": 0.2990934424590571,
      "mean_conf": 0.9685854931011313,
      "confident_error_rate": 0.1125,
      "coverage_at_0_9": 0.9,
      "accuracy_at_0_9": 0.875,
      "coverage_at_5pct_error": 0.6375,
      "coverage_at_1pct_error": 0.35,
      "aurc": 0.04725108167663764,
      "error_rate_at_0_9": 0.125,
      "confidence_bias": 0.14358549310113133,
      "top_bins": {
        "0.9": {
          "n": 72,
          "errors": 9,
          "error_rate": 0.125
        },
        "0.95": {
          "n": 70,
          "errors": 8,
          "error_rate": 0.11428571428571428
        },
        "0.99": {
          "n": 64,
          "errors": 6,
          "error_rate": 0.09375
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.975,
          "confidence_cutoff": 0.9998947981988004
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.90625,
          "confidence_cutoff": 0.9914323508021363
        }
      }
    },
    "agnews_yn": {
      "n": 160,
      "nll": 0.2518408232714972,
      "acc": 0.93125,
      "ece": 0.045003669094672225,
      "brier": 0.11756034520073606,
      "mean_conf": 0.9756982049318779,
      "confident_error_rate": 0.04375,
      "coverage_at_0_9": 0.93125,
      "accuracy_at_0_9": 0.9530201342281879,
      "coverage_at_5pct_error": 0.9375,
      "coverage_at_1pct_error": 0.7125,
      "aurc": 0.011018667226869875,
      "error_rate_at_0_9": 0.04697986577181208,
      "confidence_bias": 0.044448204931877844,
      "top_bins": {
        "0.9": {
          "n": 149,
          "errors": 7,
          "error_rate": 0.04697986577181208
        },
        "0.95": {
          "n": 144,
          "errors": 6,
          "error_rate": 0.041666666666666664
        },
        "0.99": {
          "n": 136,
          "errors": 4,
          "error_rate": 0.029411764705882353
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9999110556394879
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.96875,
          "confidence_cutoff": 0.9936313528188293
        }
      }
    },
    "amazon": {
      "n": 80,
      "nll": 1.324250311821012,
      "acc": 0.575,
      "ece": 0.22860842944044593,
      "brier": 0.6067224696237944,
      "mean_conf": 0.7718615023887951,
      "confident_error_rate": 0.0625,
      "coverage_at_0_9": 0.3125,
      "accuracy_at_0_9": 0.8,
      "coverage_at_5pct_error": 0.1125,
      "coverage_at_1pct_error": 0.1125,
      "aurc": 0.24800718168430919,
      "error_rate_at_0_9": 0.2,
      "confidence_bias": 0.19686150238879518,
      "top_bins": {
        "0.9": {
          "n": 25,
          "errors": 5,
          "error_rate": 0.2
        },
        "0.95": {
          "n": 11,
          "errors": 1,
          "error_rate": 0.09090909090909091
        },
        "0.99": {
          "n": 1,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.725,
          "confidence_cutoff": 0.8343865905169211
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.65625,
          "confidence_cutoff": 0.5845343129479065
        }
      },
      "score_mae": 0.5405116262397505,
      "ranked_probability_score": 0.09803298161037352
    },
    "banking77": {
      "n": 80,
      "nll": 0.410162378726587,
      "acc": 0.8625,
      "ece": 0.0684380316004429,
      "brier": 0.19369609625764847,
      "mean_conf": 0.9032688636575348,
      "confident_error_rate": 0.0125,
      "coverage_at_0_9": 0.725,
      "accuracy_at_0_9": 0.9827586206896551,
      "coverage_at_5pct_error": 0.8,
      "coverage_at_1pct_error": 0.6125,
      "aurc": 0.023725003641932918,
      "error_rate_at_0_9": 0.017241379310344827,
      "confidence_bias": 0.04076886365753474,
      "top_bins": {
        "0.9": {
          "n": 58,
          "errors": 1,
          "error_rate": 0.017241379310344827
        },
        "0.95": {
          "n": 56,
          "errors": 1,
          "error_rate": 0.017857142857142856
        },
        "0.99": {
          "n": 48,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.999096616183809
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.953125,
          "confidence_cutoff": 0.7555751960506732
        }
      }
    },
    "boolq": {
      "n": 80,
      "nll": 1.040614879266006,
      "acc": 0.725,
      "ece": 0.21244497975326623,
      "brier": 0.40454697808100326,
      "mean_conf": 0.9133623135557816,
      "confident_error_rate": 0.15,
      "coverage_at_0_9": 0.725,
      "accuracy_at_0_9": 0.7931034482758621,
      "coverage_at_5pct_error": 0.0125,
      "coverage_at_1pct_error": 0.0125,
      "aurc": 0.21416214725688515,
      "error_rate_at_0_9": 0.20689655172413793,
      "confidence_bias": 0.1883623135557816,
      "top_bins": {
        "0.9": {
          "n": 58,
          "errors": 12,
          "error_rate": 0.20689655172413793
        },
        "0.95": {
          "n": 58,
          "errors": 12,
          "error_rate": 0.20689655172413793
        },
        "0.99": {
          "n": 41,
          "errors": 8,
          "error_rate": 0.1951219512195122
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.8,
          "confidence_cutoff": 0.9907489151398479
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.8125,
          "confidence_cutoff": 0.8405418616291048
        }
      }
    },
    "composition_atom": {
      "n": 16,
      "nll": 0.002680492285466909,
      "acc": 1.0,
      "ece": 0.0026731550276416494,
      "brier": 2.9196443861008606e-05,
      "mean_conf": 0.9973268449723584,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 1.0,
      "accuracy_at_0_9": 1.0,
      "coverage_at_5pct_error": 1.0,
      "coverage_at_1pct_error": 1.0,
      "aurc": 0.0,
      "error_rate_at_0_9": 0.0,
      "confidence_bias": -0.0026731550276416494,
      "top_bins": {
        "0.9": {
          "n": 16,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 16,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 15,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5625,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9986770752579941
        },
        "0.8": {
          "coverage": 0.875,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9955206251132698
        }
      }
    },
    "composition_conditional": {
      "n": 16,
      "nll": 1.4381850941357321,
      "acc": 0.5625,
      "ece": 0.3481338460231991,
      "brier": 0.661713684210192,
      "mean_conf": 0.9106338460231991,
      "confident_error_rate": 0.25,
      "coverage_at_0_9": 0.8125,
      "accuracy_at_0_9": 0.6923076923076923,
      "coverage_at_5pct_error": 0.0,
      "coverage_at_1pct_error": 0.0,
      "aurc": 0.5198508089133089,
      "error_rate_at_0_9": 0.3076923076923077,
      "confidence_bias": 0.3481338460231991,
      "top_bins": {
        "0.9": {
          "n": 13,
          "errors": 4,
          "error_rate": 0.3076923076923077
        },
        "0.95": {
          "n": 10,
          "errors": 4,
          "error_rate": 0.4
        },
        "0.99": {
          "n": 4,
          "errors": 3,
          "error_rate": 0.75
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.5,
          "confidence_cutoff": 0.9569063568806717
        },
        "0.8": {
          "coverage": 0.8125,
          "accuracy": 0.6923076923076923,
          "confidence_cutoff": 0.9097507460452505
        }
      }
    },
    "composition_conjunction": {
      "n": 16,
      "nll": 0.38746653003635695,
      "acc": 0.9375,
      "ece": 0.09463049067355556,
      "brier": 0.1389206579242385,
      "mean_conf": 0.9576638062031289,
      "confident_error_rate": 0.0625,
      "coverage_at_0_9": 0.8125,
      "accuracy_at_0_9": 0.9230769230769231,
      "coverage_at_5pct_error": 0.4375,
      "coverage_at_1pct_error": 0.4375,
      "aurc": 0.048470886752136746,
      "error_rate_at_0_9": 0.07692307692307693,
      "confidence_bias": 0.020163806203128853,
      "top_bins": {
        "0.9": {
          "n": 13,
          "errors": 1,
          "error_rate": 0.07692307692307693
        },
        "0.95": {
          "n": 13,
          "errors": 1,
          "error_rate": 0.07692307692307693
        },
        "0.99": {
          "n": 10,
          "errors": 1,
          "error_rate": 0.1
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.875,
          "confidence_cutoff": 0.9957344139983478
        },
        "0.8": {
          "coverage": 0.8125,
          "accuracy": 0.9230769230769231,
          "confidence_cutoff": 0.9648820305598085
        }
      }
    },
    "composition_disjunction": {
      "n": 16,
      "nll": 0.55008765340706,
      "acc": 0.8125,
      "ece": 0.21554479717440728,
      "brier": 0.34604733899181345,
      "mean_conf": 0.9221003397570159,
      "confident_error_rate": 0.1875,
      "coverage_at_0_9": 0.75,
      "accuracy_at_0_9": 0.75,
      "coverage_at_5pct_error": 0.5625,
      "coverage_at_1pct_error": 0.5625,
      "aurc": 0.08949409965034966,
      "error_rate_at_0_9": 0.25,
      "confidence_bias": 0.10960033975701589,
      "top_bins": {
        "0.9": {
          "n": 12,
          "errors": 3,
          "error_rate": 0.25
        },
        "0.95": {
          "n": 9,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 5,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9634917450211327
        },
        "0.8": {
          "coverage": 0.8125,
          "accuracy": 0.7692307692307693,
          "confidence_cutoff": 0.854980123862328
        }
      }
    },
    "composition_exception": {
      "n": 16,
      "nll": 0.23657229145907507,
      "acc": 0.9375,
      "ece": 0.1103620522080225,
      "brier": 0.138121436655377,
      "mean_conf": 0.8600103369113501,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.5,
      "accuracy_at_0_9": 1.0,
      "coverage_at_5pct_error": 0.6875,
      "coverage_at_1pct_error": 0.6875,
      "aurc": 0.02225560897435897,
      "error_rate_at_0_9": 0.0,
      "confidence_bias": -0.07748966308864991,
      "top_bins": {
        "0.9": {
          "n": 8,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 8,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 4,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9511738554937388
        },
        "0.8": {
          "coverage": 0.8125,
          "accuracy": 0.9230769230769231,
          "confidence_cutoff": 0.7046734304628799
        }
      }
    },
    "composition_negation": {
      "n": 16,
      "nll": 0.47424362442543067,
      "acc": 0.8125,
      "ece": 0.22960102424196827,
      "brier": 0.3146887466200047,
      "mean_conf": 0.8903118087091675,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.6875,
      "accuracy_at_0_9": 1.0,
      "coverage_at_5pct_error": 0.6875,
      "coverage_at_1pct_error": 0.6875,
      "aurc": 0.056212797619047614,
      "error_rate_at_0_9": 0.0,
      "confidence_bias": 0.07781180870916748,
      "top_bins": {
        "0.9": {
          "n": 11,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 2,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9099642173940572
        },
        "0.8": {
          "coverage": 0.875,
          "accuracy": 0.7857142857142857,
          "confidence_cutoff": 0.8671133647428955
        }
      }
    },
    "composition_nested_and": {
      "n": 16,
      "nll": 0.37663416694448315,
      "acc": 0.8125,
      "ece": 0.18809247118975278,
      "brier": 0.2688867469106061,
      "mean_conf": 0.9087914724823688,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.6875,
      "accuracy_at_0_9": 1.0,
      "coverage_at_5pct_error": 0.75,
      "coverage_at_1pct_error": 0.75,
      "aurc": 0.04207589285714286,
      "error_rate_at_0_9": 0.0,
      "confidence_bias": 0.0962914724823688,
      "top_bins": {
        "0.9": {
          "n": 11,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 8,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 3,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9511395799123449
        },
        "0.8": {
          "coverage": 0.875,
          "accuracy": 0.8571428571428571,
          "confidence_cutoff": 0.8206092592983059
        }
      }
    },
    "composition_nested_or": {
      "n": 16,
      "nll": 0.041209903480834115,
      "acc": 1.0,
      "ece": 0.037854312750542234,
      "brier": 0.011501302622745553,
      "mean_conf": 0.9621456872494576,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.875,
      "accuracy_at_0_9": 1.0,
      "coverage_at_5pct_error": 1.0,
      "coverage_at_1pct_error": 1.0,
      "aurc": 0.0,
      "error_rate_at_0_9": 0.0,
      "confidence_bias": -0.03785431275054241,
      "top_bins": {
        "0.9": {
          "n": 14,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 14,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 5,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9847120257545044
        },
        "0.8": {
          "coverage": 0.875,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9772779654967141
        }
      }
    },
    "contrastive_age_eligibility": {
      "n": 24,
      "nll": 0.00026585468442352447,
      "acc": 1.0,
      "ece": 0.0002657952242964834,
      "brier": 2.3776395341825677e-07,
      "mean_conf": 0.9997342047757035,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 1.0,
      "accuracy_at_0_9": 1.0,
      "coverage_at_5pct_error": 1.0,
      "coverage_at_1pct_error": 1.0,
      "aurc": 0.0,
      "error_rate_at_0_9": 0.0,
      "confidence_bias": -0.0002657952242964834,
      "top_bins": {
        "0.9": {
          "n": 24,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 24,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 24,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9998354995609443
        },
        "0.8": {
          "coverage": 0.8333333333333334,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9996112670831452
        }
      }
    },
    "contrastive_quantity_limit": {
      "n": 24,
      "nll": 0.04906113593244044,
      "acc": 0.9583333333333334,
      "ece": 0.030251939053325728,
      "brier": 0.038146919063443786,
      "mean_conf": 0.9843247335311552,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.9583333333333334,
      "accuracy_at_0_9": 1.0,
      "coverage_at_5pct_error": 1.0,
      "coverage_at_1pct_error": 0.9583333333333334,
      "aurc": 0.001736111111111111,
      "error_rate_at_0_9": 0.0,
      "confidence_bias": 0.025991400197821868,
      "top_bins": {
        "0.9": {
          "n": 23,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 23,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 22,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9999704039492336
        },
        "0.8": {
          "coverage": 0.8333333333333334,
          "accuracy": 1.0,
          "confidence_cutoff": 0.999174817467293
        }
      }
    },
    "contrastive_return_window": {
      "n": 24,
      "nll": 0.5543185090251112,
      "acc": 0.7916666666666666,
      "ece": 0.20556245807513226,
      "brier": 0.3715826172317896,
      "mean_conf": 0.8721899660042904,
      "confident_error_rate": 0.041666666666666664,
      "coverage_at_0_9": 0.5,
      "accuracy_at_0_9": 0.9166666666666666,
      "coverage_at_5pct_error": 0.4583333333333333,
      "coverage_at_1pct_error": 0.4583333333333333,
      "aurc": 0.09400846084114356,
      "error_rate_at_0_9": 0.08333333333333333,
      "confidence_bias": 0.08052329933762381,
      "top_bins": {
        "0.9": {
          "n": 12,
          "errors": 1,
          "error_rate": 0.08333333333333333
        },
        "0.95": {
          "n": 7,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 1,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.9166666666666666,
          "confidence_cutoff": 0.9066088863209552
        },
        "0.8": {
          "coverage": 0.8333333333333334,
          "accuracy": 0.75,
          "confidence_cutoff": 0.8221897360356192
        }
      }
    },
    "contrastive_spend_threshold": {
      "n": 24,
      "nll": 0.0007032392546397221,
      "acc": 1.0,
      "ece": 0.0006999080058708307,
      "brier": 1.3168254762536335e-05,
      "mean_conf": 0.9993000919941292,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 1.0,
      "accuracy_at_0_9": 1.0,
      "coverage_at_5pct_error": 1.0,
      "coverage_at_1pct_error": 1.0,
      "aurc": 0.0,
      "error_rate_at_0_9": 0.0,
      "confidence_bias": -0.0006999080058708307,
      "top_bins": {
        "0.9": {
          "n": 24,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 24,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 23,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9999813585467731
        },
        "0.8": {
          "coverage": 0.8333333333333334,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9993636765668262
        }
      }
    },
    "dbpedia14": {
      "n": 80,
      "nll": 0.41883315113801806,
      "acc": 0.9625,
      "ece": 0.03696562770122125,
      "brier": 0.07300392552693573,
      "mean_conf": 0.9994656277012213,
      "confident_error_rate": 0.0375,
      "coverage_at_0_9": 1.0,
      "accuracy_at_0_9": 0.9625,
      "coverage_at_5pct_error": 1.0,
      "coverage_at_1pct_error": 0.45,
      "aurc": 0.011179882217377419,
      "error_rate_at_0_9": 0.0375,
      "confidence_bias": 0.03696562770122125,
      "top_bins": {
        "0.9": {
          "n": 80,
          "errors": 3,
          "error_rate": 0.0375
        },
        "0.95": {
          "n": 80,
          "errors": 3,
          "error_rate": 0.0375
        },
        "0.99": {
          "n": 79,
          "errors": 2,
          "error_rate": 0.02531645569620253
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.975,
          "confidence_cutoff": 0.9999999220926464
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.984375,
          "confidence_cutoff": 0.9999996558857667
        }
      }
    },
    "imdb": {
      "n": 80,
      "nll": 0.5725454492156331,
      "acc": 0.8875,
      "ece": 0.11017186135253637,
      "brier": 0.1995242493892689,
      "mean_conf": 0.9789566679550672,
      "confident_error_rate": 0.075,
      "coverage_at_0_9": 0.9375,
      "accuracy_at_0_9": 0.92,
      "coverage_at_5pct_error": 0.8375,
      "coverage_at_1pct_error": 0.3,
      "aurc": 0.030055917121328518,
      "error_rate_at_0_9": 0.08,
      "confidence_bias": 0.09145666795506724,
      "top_bins": {
        "0.9": {
          "n": 75,
          "errors": 6,
          "error_rate": 0.08
        },
        "0.95": {
          "n": 73,
          "errors": 5,
          "error_rate": 0.0684931506849315
        },
        "0.99": {
          "n": 66,
          "errors": 3,
          "error_rate": 0.045454545454545456
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.975,
          "confidence_cutoff": 0.9999765124722082
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.953125,
          "confidence_cutoff": 0.9968554678837199
        }
      }
    },
    "mnli": {
      "n": 80,
      "nll": 1.0771061846724677,
      "acc": 0.8,
      "ece": 0.1708574878787405,
      "brier": 0.3546161858206295,
      "mean_conf": 0.963465925978932,
      "confident_error_rate": 0.15,
      "coverage_at_0_9": 0.9,
      "accuracy_at_0_9": 0.8333333333333334,
      "coverage_at_5pct_error": 0.275,
      "coverage_at_1pct_error": 0.2375,
      "aurc": 0.09906389886271771,
      "error_rate_at_0_9": 0.16666666666666666,
      "confidence_bias": 0.16346592597893195,
      "top_bins": {
        "0.9": {
          "n": 72,
          "errors": 12,
          "error_rate": 0.16666666666666666
        },
        "0.95": {
          "n": 67,
          "errors": 11,
          "error_rate": 0.16417910447761194
        },
        "0.99": {
          "n": 57,
          "errors": 6,
          "error_rate": 0.10526315789473684
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.875,
          "confidence_cutoff": 0.997514074186558
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.84375,
          "confidence_cutoff": 0.9711133720916449
        }
      }
    },
    "sst5": {
      "n": 80,
      "nll": 1.4963841088063536,
      "acc": 0.575,
      "ece": 0.2486466914113493,
      "brier": 0.6950572098874017,
      "mean_conf": 0.8051073569042574,
      "confident_error_rate": 0.1375,
      "coverage_at_0_9": 0.2875,
      "accuracy_at_0_9": 0.5217391304347826,
      "coverage_at_5pct_error": 0.0,
      "coverage_at_1pct_error": 0.0,
      "aurc": 0.4449588964070667,
      "error_rate_at_0_9": 0.4782608695652174,
      "confidence_bias": 0.23010735690425743,
      "top_bins": {
        "0.9": {
          "n": 23,
          "errors": 11,
          "error_rate": 0.4782608695652174
        },
        "0.95": {
          "n": 11,
          "errors": 5,
          "error_rate": 0.45454545454545453
        },
        "0.99": {
          "n": 1,
          "errors": 1,
          "error_rate": 1.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.575,
          "confidence_cutoff": 0.8445807633004492
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.578125,
          "confidence_cutoff": 0.6831346394936841
        }
      },
      "score_mae": 0.5577337834172155,
      "ranked_probability_score": 0.10503153292252385
    },
    "trec": {
      "n": 80,
      "nll": 0.19702283241356697,
      "acc": 0.9625,
      "ece": 0.03659841518218783,
      "brier": 0.06035787060668799,
      "mean_conf": 0.9875699073206501,
      "confident_error_rate": 0.025,
      "coverage_at_0_9": 0.9625,
      "accuracy_at_0_9": 0.974025974025974,
      "coverage_at_5pct_error": 1.0,
      "coverage_at_1pct_error": 0.8375,
      "aurc": 0.003666199961819887,
      "error_rate_at_0_9": 0.025974025974025976,
      "confidence_bias": 0.025069907320650064,
      "top_bins": {
        "0.9": {
          "n": 77,
          "errors": 2,
          "error_rate": 0.025974025974025976
        },
        "0.95": {
          "n": 77,
          "errors": 2,
          "error_rate": 0.025974025974025976
        },
        "0.99": {
          "n": 72,
          "errors": 1,
          "error_rate": 0.013888888888888888
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9999961538099368
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9999727440088424
        }
      }
    },
    "yelp": {
      "n": 80,
      "nll": 1.003998910170863,
      "acc": 0.6375,
      "ece": 0.19853519009111997,
      "brier": 0.5397156607261331,
      "mean_conf": 0.8311354772904022,
      "confident_error_rate": 0.1,
      "coverage_at_0_9": 0.425,
      "accuracy_at_0_9": 0.7647058823529411,
      "coverage_at_5pct_error": 0.1125,
      "coverage_at_1pct_error": 0.1125,
      "aurc": 0.24795414540453392,
      "error_rate_at_0_9": 0.23529411764705882,
      "confidence_bias": 0.1936354772904022,
      "top_bins": {
        "0.9": {
          "n": 34,
          "errors": 8,
          "error_rate": 0.23529411764705882
        },
        "0.95": {
          "n": 23,
          "errors": 4,
          "error_rate": 0.17391304347826086
        },
        "0.99": {
          "n": 11,
          "errors": 1,
          "error_rate": 0.09090909090909091
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.7,
          "confidence_cutoff": 0.8800023203578181
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.671875,
          "confidence_cutoff": 0.7128260868757033
        }
      },
      "score_mae": 0.40288296592875195,
      "ranked_probability_score": 0.07222732649769444
    },
    "yelp_yn": {
      "n": 80,
      "nll": 0.34777014710198545,
      "acc": 0.8875,
      "ece": 0.08654197085390745,
      "brier": 0.1678552195878949,
      "mean_conf": 0.9612952645005635,
      "confident_error_rate": 0.05,
      "coverage_at_0_9": 0.875,
      "accuracy_at_0_9": 0.9428571428571428,
      "coverage_at_5pct_error": 0.825,
      "coverage_at_1pct_error": 0.6125,
      "aurc": 0.019420242079986067,
      "error_rate_at_0_9": 0.05714285714285714,
      "confidence_bias": 0.0737952645005635,
      "top_bins": {
        "0.9": {
          "n": 70,
          "errors": 4,
          "error_rate": 0.05714285714285714
        },
        "0.95": {
          "n": 68,
          "errors": 4,
          "error_rate": 0.058823529411764705
        },
        "0.99": {
          "n": 64,
          "errors": 3,
          "error_rate": 0.046875
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9997654584142818
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.953125,
          "confidence_cutoff": 0.9932514503698145
        }
      }
    }
  },
  "variants": {
    "clean": {
      "n": 1264,
      "nll": 0.6468405219068211,
      "acc": 0.8267405063291139,
      "ece": 0.10212917934363491,
      "brier": 0.2739376394391063,
      "mean_conf": 0.9287298947829927,
      "confident_error_rate": 0.07041139240506329,
      "coverage_at_0_9": 0.7705696202531646,
      "accuracy_at_0_9": 0.9086242299794661,
      "coverage_at_5pct_error": 0.5893987341772152,
      "coverage_at_1pct_error": 0.3188291139240506,
      "aurc": 0.05376999371647826,
      "error_rate_at_0_9": 0.09137577002053388,
      "confidence_bias": 0.10198938845387884,
      "top_bins": {
        "0.9": {
          "n": 974,
          "errors": 89,
          "error_rate": 0.09137577002053388
        },
        "0.95": {
          "n": 896,
          "errors": 67,
          "error_rate": 0.07477678571428571
        },
        "0.99": {
          "n": 756,
          "errors": 39,
          "error_rate": 0.051587301587301584
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.9667721518987342,
          "confidence_cutoff": 0.9975252105885826
        },
        "0.8": {
          "coverage": 0.8006329113924051,
          "accuracy": 0.8972332015810277,
          "confidence_cutoff": 0.873414767882448
        }
      },
      "score_mae": 0.5003761251952392,
      "ranked_probability_score": 0.09176394701019726
    },
    "none_absent": {
      "n": 60,
      "nll": 0.5126701213798888,
      "acc": 0.85,
      "ece": 0.11186969281671219,
      "brier": 0.23126610360275227,
      "mean_conf": 0.9544530228032457,
      "confident_error_rate": 0.06666666666666667,
      "coverage_at_0_9": 0.8333333333333334,
      "accuracy_at_0_9": 0.92,
      "coverage_at_5pct_error": 0.6833333333333333,
      "coverage_at_1pct_error": 0.13333333333333333,
      "aurc": 0.062348342206284574,
      "error_rate_at_0_9": 0.08,
      "confidence_bias": 0.10445302280324575,
      "top_bins": {
        "0.9": {
          "n": 50,
          "errors": 4,
          "error_rate": 0.08
        },
        "0.95": {
          "n": 44,
          "errors": 4,
          "error_rate": 0.09090909090909091
        },
        "0.99": {
          "n": 32,
          "errors": 2,
          "error_rate": 0.0625
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.9666666666666667,
          "confidence_cutoff": 0.9953637749863697
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.9166666666666666,
          "confidence_cutoff": 0.9330138103573286
        }
      }
    },
    "none_present": {
      "n": 60,
      "nll": 0.8013395380431254,
      "acc": 0.85,
      "ece": 0.10917610793098725,
      "brier": 0.2497676286873287,
      "mean_conf": 0.9505158370888986,
      "confident_error_rate": 0.05,
      "coverage_at_0_9": 0.8166666666666667,
      "accuracy_at_0_9": 0.9387755102040817,
      "coverage_at_5pct_error": 0.6833333333333333,
      "coverage_at_1pct_error": 0.016666666666666666,
      "aurc": 0.08622271676065942,
      "error_rate_at_0_9": 0.061224489795918366,
      "confidence_bias": 0.10051583708889866,
      "top_bins": {
        "0.9": {
          "n": 49,
          "errors": 3,
          "error_rate": 0.061224489795918366
        },
        "0.95": {
          "n": 45,
          "errors": 3,
          "error_rate": 0.06666666666666667
        },
        "0.99": {
          "n": 27,
          "errors": 1,
          "error_rate": 0.037037037037037035
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.9666666666666667,
          "confidence_cutoff": 0.9839017877805669
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.9375,
          "confidence_cutoff": 0.9317478417971637
        }
      }
    },
    "permuted": {
      "n": 84,
      "nll": 0.49225169833838855,
      "acc": 0.9166666666666666,
      "ece": 0.06548951350387361,
      "brier": 0.14989312655973092,
      "mean_conf": 0.9785580799292399,
      "confident_error_rate": 0.05952380952380952,
      "coverage_at_0_9": 0.9404761904761905,
      "accuracy_at_0_9": 0.9367088607594937,
      "coverage_at_5pct_error": 0.8690476190476191,
      "coverage_at_1pct_error": 0.09523809523809523,
      "aurc": 0.03793864529834329,
      "error_rate_at_0_9": 0.06329113924050633,
      "confidence_bias": 0.0618914132625733,
      "top_bins": {
        "0.9": {
          "n": 79,
          "errors": 5,
          "error_rate": 0.06329113924050633
        },
        "0.95": {
          "n": 78,
          "errors": 5,
          "error_rate": 0.0641025641025641
        },
        "0.99": {
          "n": 69,
          "errors": 2,
          "error_rate": 0.028985507246376812
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.9761904761904762,
          "confidence_cutoff": 0.9998085742747008
        },
        "0.8": {
          "coverage": 0.8095238095238095,
          "accuracy": 0.9705882352941176,
          "confidence_cutoff": 0.99288884061492
        }
      }
    }
  },
  "heldout_tasks": {},
  "permutation": {
    "n": 60,
    "mean_max_delta": 0.02845344004311335,
    "flip_rate": 0.05
  },
  "temperature": 2.2191389441356897,
  "calibrated_clean": {
    "n": 1264,
    "nll": 0.45474875213453486,
    "acc": 0.8267405063291139,
    "ece": 0.025974188543495685,
    "brier": 0.24875114618128633,
    "mean_conf": 0.8419813520899577,
    "confident_error_rate": 0.028481012658227847,
    "coverage_at_0_9": 0.5656645569620253,
    "accuracy_at_0_9": 0.9496503496503497,
    "coverage_at_5pct_error": 0.5561708860759493,
    "coverage_at_1pct_error": 0.2879746835443038,
    "aurc": 0.05333286472745321,
    "error_rate_at_0_9": 0.05034965034965035,
    "confidence_bias": 0.015240845760843835,
    "top_bins": {
      "0.9": {
        "n": 715,
        "errors": 36,
        "error_rate": 0.05034965034965035
      },
      "0.95": {
        "n": 573,
        "errors": 17,
        "error_rate": 0.029668411867364748
      },
      "0.99": {
        "n": 299,
        "errors": 2,
        "error_rate": 0.006688963210702341
      }
    },
    "selective": {
      "0.5": {
        "coverage": 0.5,
        "accuracy": 0.9667721518987342,
        "confidence_cutoff": 0.9305159828813725
      },
      "0.8": {
        "coverage": 0.8014240506329114,
        "accuracy": 0.8983218163869694,
        "confidence_cutoff": 0.6679287392165254
      }
    },
    "score_mae": 0.5469473813325069,
    "ranked_probability_score": 0.08419851205840653
  },
  "metric_policy": {
    "version": 2,
    "selective_ties": "whole_confidence_groups",
    "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee",
    "aurc": "right-step integral over whole confidence groups",
    "confident_error_rate": "high-confidence errors divided by all questions",
    "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9",
    "nll": "exact from logits when recorded; otherwise from floored probabilities",
    "nll_floor": 1e-09,
    "renormalize_returned_probabilities": true,
    "raw_sums_outside_1e_5": 0,
    "returned_zeros": 0
  },
  "coverage": {
    "requested_records": 1204,
    "requested_questions": 1468,
    "evaluated_records": 1204,
    "evaluated_questions": 1468,
    "rejected_records": 0,
    "truncated_records": 0
  },
  "latency_ms": {
    "median": 41.359284999998636,
    "p95": 43.40382945015335
  },
  "calibration": {
    "inference_temperature": 1.0,
    "additional_temperature": 2.2191389441356897,
    "logits_recorded": true
  },
  "transfer": {
    "objective": -0.8325745919267149,
    "paired_flip": {
      "pairs": 64,
      "flip_rate": 0.46875,
      "both_correct_rate": 0.421875,
      "invariant_pairs": 24,
      "invariance_rate": 1.0,
      "invariant_both_correct_rate": 0.5833333333333334
    },
    "unknowable": null,
    "clean": {
      "n": 656,
      "nll": 0.8473673030417037,
      "acc": 0.6478658536585366,
      "ece": 0.15246554096939913,
      "brier": 0.48125328370231507,
      "mean_conf": 0.8003313946279355,
      "confident_error_rate": 0.056402439024390245,
      "coverage_at_0_9": 0.39634146341463417,
      "accuracy_at_0_9": 0.8576923076923076,
      "coverage_at_5pct_error": 0.16920731707317074,
      "coverage_at_1pct_error": 0.07469512195121951,
      "aurc": 0.17511028693209713,
      "error_rate_at_0_9": 0.1423076923076923,
      "confidence_bias": 0.1524655409693989,
      "top_bins": {
        "0.9": {
          "n": 260,
          "errors": 37,
          "error_rate": 0.1423076923076923
        },
        "0.95": {
          "n": 208,
          "errors": 21,
          "error_rate": 0.10096153846153846
        },
        "0.99": {
          "n": 106,
          "errors": 5,
          "error_rate": 0.04716981132075472
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.8140243902439024,
          "confidence_cutoff": 0.855563355320146
        },
        "0.8": {
          "coverage": 0.8003048780487805,
          "accuracy": 0.7123809523809523,
          "confidence_cutoff": 0.6205200915575361
        }
      },
      "score_mae": 0.67300036097947,
      "ranked_probability_score": 0.2294783296027286
    },
    "tasks": {
      "composition_held_and_or": {
        "n": 32,
        "nll": 0.9766401605534833,
        "acc": 0.65625,
        "ece": 0.2759884336884012,
        "brier": 0.5522134595300838,
        "mean_conf": 0.9322384336884012,
        "confident_error_rate": 0.21875,
        "coverage_at_0_9": 0.75,
        "accuracy_at_0_9": 0.7083333333333334,
        "coverage_at_5pct_error": 0.15625,
        "coverage_at_1pct_error": 0.15625,
        "aurc": 0.22049902211489347,
        "error_rate_at_0_9": 0.2916666666666667,
        "confidence_bias": 0.2759884336884012,
        "top_bins": {
          "0.9": {
            "n": 24,
            "errors": 7,
            "error_rate": 0.2916666666666667
          },
          "0.95": {
            "n": 20,
            "errors": 4,
            "error_rate": 0.2
          },
          "0.99": {
            "n": 5,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.53125,
            "accuracy": 0.8235294117647058,
            "confidence_cutoff": 0.9758496433679673
          },
          "0.8": {
            "coverage": 0.84375,
            "accuracy": 0.7407407407407407,
            "confidence_cutoff": 0.8654291033744812
          }
        }
      },
      "composition_held_conditional": {
        "n": 32,
        "nll": 0.6266604441471431,
        "acc": 0.65625,
        "ece": 0.14122441205349498,
        "brier": 0.44580578473896104,
        "mean_conf": 0.7488511904208259,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.09375,
        "accuracy_at_0_9": 1.0,
        "coverage_at_5pct_error": 0.125,
        "coverage_at_1pct_error": 0.125,
        "aurc": 0.23706863318202734,
        "error_rate_at_0_9": 0.0,
        "confidence_bias": 0.09260119042082593,
        "top_bins": {
          "0.9": {
            "n": 3,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.625,
            "confidence_cutoff": 0.7114255044289084
          },
          "0.8": {
            "coverage": 0.8125,
            "accuracy": 0.6538461538461539,
            "confidence_cutoff": 0.6640934944152832
          }
        }
      },
      "composition_held_or_not": {
        "n": 32,
        "nll": 0.8918989730655735,
        "acc": 0.5625,
        "ece": 0.27646008506339326,
        "brier": 0.6158885950256443,
        "mean_conf": 0.8339621685489521,
        "confident_error_rate": 0.15625,
        "coverage_at_0_9": 0.3125,
        "accuracy_at_0_9": 0.5,
        "coverage_at_5pct_error": 0.125,
        "coverage_at_1pct_error": 0.125,
        "aurc": 0.33870655616802325,
        "error_rate_at_0_9": 0.5,
        "confidence_bias": 0.2714621685489521,
        "top_bins": {
          "0.9": {
            "n": 10,
            "errors": 5,
            "error_rate": 0.5
          },
          "0.95": {
            "n": 5,
            "errors": 1,
            "error_rate": 0.2
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.6875,
            "confidence_cutoff": 0.8887970248878667
          },
          "0.8": {
            "coverage": 0.8125,
            "accuracy": 0.5769230769230769,
            "confidence_cutoff": 0.7215237451507893
          }
        }
      },
      "contrastive_authorization": {
        "n": 40,
        "nll": 0.18874489020814633,
        "acc": 0.95,
        "ece": 0.07035061559644126,
        "brier": 0.11583471559515715,
        "mean_conf": 0.8868357194684137,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.7,
        "accuracy_at_0_9": 1.0,
        "coverage_at_5pct_error": 1.0,
        "coverage_at_1pct_error": 0.75,
        "aurc": 0.01259009378789602,
        "error_rate_at_0_9": 0.0,
        "confidence_bias": -0.06316428053158629,
        "top_bins": {
          "0.9": {
            "n": 28,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 27,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 2,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9661230192971932
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.96875,
            "confidence_cutoff": 0.7516713886456854
          }
        }
      },
      "contrastive_deadline": {
        "n": 40,
        "nll": 1.2368328689761974,
        "acc": 0.35,
        "ece": 0.31302734346005,
        "brier": 0.7620299272400921,
        "mean_conf": 0.66302734346005,
        "confident_error_rate": 0.075,
        "coverage_at_0_9": 0.15,
        "accuracy_at_0_9": 0.5,
        "coverage_at_5pct_error": 0.05,
        "coverage_at_1pct_error": 0.05,
        "aurc": 0.518712885866291,
        "error_rate_at_0_9": 0.5,
        "confidence_bias": 0.31302734346005,
        "top_bins": {
          "0.9": {
            "n": 6,
            "errors": 3,
            "error_rate": 0.5
          },
          "0.95": {
            "n": 3,
            "errors": 1,
            "error_rate": 0.3333333333333333
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.45,
            "confidence_cutoff": 0.5926517844200134
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.40625,
            "confidence_cutoff": 0.5105663614059442
          }
        },
        "score_mae": 0.67300036097947,
        "ranked_probability_score": 0.2294783296027286
      },
      "emotion": {
        "n": 80,
        "nll": 1.6431966355834482,
        "acc": 0.5,
        "ece": 0.25791654554851956,
        "brier": 0.6869115290571809,
        "mean_conf": 0.7415177656608363,
        "confident_error_rate": 0.05,
        "coverage_at_0_9": 0.2875,
        "accuracy_at_0_9": 0.8260869565217391,
        "coverage_at_5pct_error": 0.0,
        "coverage_at_1pct_error": 0.0,
        "aurc": 0.3812803499509923,
        "error_rate_at_0_9": 0.17391304347826086,
        "confidence_bias": 0.24151776566083627,
        "top_bins": {
          "0.9": {
            "n": 23,
            "errors": 4,
            "error_rate": 0.17391304347826086
          },
          "0.95": {
            "n": 18,
            "errors": 4,
            "error_rate": 0.2222222222222222
          },
          "0.99": {
            "n": 9,
            "errors": 3,
            "error_rate": 0.3333333333333333
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.625,
            "confidence_cutoff": 0.7548409340511426
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.53125,
            "confidence_cutoff": 0.5661950644158379
          }
        }
      },
      "mmlu": {
        "n": 80,
        "nll": 1.361566206264453,
        "acc": 0.425,
        "ece": 0.2153956746505119,
        "brier": 0.7279459890735432,
        "mean_conf": 0.6363134116980336,
        "confident_error_rate": 0.0375,
        "coverage_at_0_9": 0.2125,
        "accuracy_at_0_9": 0.8235294117647058,
        "coverage_at_5pct_error": 0.1125,
        "coverage_at_1pct_error": 0.1125,
        "aurc": 0.3994876536778187,
        "error_rate_at_0_9": 0.17647058823529413,
        "confidence_bias": 0.21131341169803358,
        "top_bins": {
          "0.9": {
            "n": 17,
            "errors": 3,
            "error_rate": 0.17647058823529413
          },
          "0.95": {
            "n": 16,
            "errors": 2,
            "error_rate": 0.125
          },
          "0.99": {
            "n": 8,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.525,
            "confidence_cutoff": 0.6025702694004552
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.4375,
            "confidence_cutoff": 0.4188710575959402
          }
        }
      },
      "paws": {
        "n": 80,
        "nll": 0.8366071802737661,
        "acc": 0.6,
        "ece": 0.2162675509324284,
        "brier": 0.5800350137964118,
        "mean_conf": 0.7838076093454985,
        "confident_error_rate": 0.0375,
        "coverage_at_0_9": 0.125,
        "accuracy_at_0_9": 0.7,
        "coverage_at_5pct_error": 0.0375,
        "coverage_at_1pct_error": 0.0375,
        "aurc": 0.4046128749440664,
        "error_rate_at_0_9": 0.3,
        "confidence_bias": 0.18380760934549856,
        "top_bins": {
          "0.9": {
            "n": 10,
            "errors": 3,
            "error_rate": 0.3
          },
          "0.95": {
            "n": 4,
            "errors": 1,
            "error_rate": 0.25
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.625,
            "confidence_cutoff": 0.8137024752078965
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.578125,
            "confidence_cutoff": 0.6951168775558472
          }
        }
      },
      "qnli": {
        "n": 80,
        "nll": 0.38899775186075874,
        "acc": 0.8125,
        "ece": 0.07408691641504837,
        "brier": 0.25666992327751414,
        "mean_conf": 0.8845413569974496,
        "confident_error_rate": 0.0375,
        "coverage_at_0_9": 0.5875,
        "accuracy_at_0_9": 0.9361702127659575,
        "coverage_at_5pct_error": 0.575,
        "coverage_at_1pct_error": 0.4125,
        "aurc": 0.0549858681940542,
        "error_rate_at_0_9": 0.06382978723404255,
        "confidence_bias": 0.07204135699744962,
        "top_bins": {
          "0.9": {
            "n": 47,
            "errors": 3,
            "error_rate": 0.06382978723404255
          },
          "0.95": {
            "n": 37,
            "errors": 1,
            "error_rate": 0.02702702702702703
          },
          "0.99": {
            "n": 22,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.975,
            "confidence_cutoff": 0.9311983834505684
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.859375,
            "confidence_cutoff": 0.7695535444055837
          }
        }
      },
      "sciq": {
        "n": 80,
        "nll": 0.2497897921800699,
        "acc": 0.9,
        "ece": 0.046077537347433306,
        "brier": 0.12783578395936696,
        "mean_conf": 0.9368845477958011,
        "confident_error_rate": 0.025,
        "coverage_at_0_9": 0.825,
        "accuracy_at_0_9": 0.9696969696969697,
        "coverage_at_5pct_error": 0.9375,
        "coverage_at_1pct_error": 0.6875,
        "aurc": 0.012234213706852841,
        "error_rate_at_0_9": 0.030303030303030304,
        "confidence_bias": 0.03688454779580108,
        "top_bins": {
          "0.9": {
            "n": 66,
            "errors": 2,
            "error_rate": 0.030303030303030304
          },
          "0.95": {
            "n": 62,
            "errors": 2,
            "error_rate": 0.03225806451612903
          },
          "0.99": {
            "n": 56,
            "errors": 1,
            "error_rate": 0.017857142857142856
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9997168000196087
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.96875,
            "confidence_cutoff": 0.9325019582333779
          }
        }
      },
      "tweet_offensive": {
        "n": 80,
        "nll": 0.7573856080808232,
        "acc": 0.675,
        "ece": 0.13613936065564303,
        "brier": 0.48238323005946604,
        "mean_conf": 0.798700495923949,
        "confident_error_rate": 0.0875,
        "coverage_at_0_9": 0.325,
        "accuracy_at_0_9": 0.7307692307692307,
        "coverage_at_5pct_error": 0.025,
        "coverage_at_1pct_error": 0.025,
        "aurc": 0.2724541868080348,
        "error_rate_at_0_9": 0.2692307692307692,
        "confidence_bias": 0.12370049592394894,
        "top_bins": {
          "0.9": {
            "n": 26,
            "errors": 7,
            "error_rate": 0.2692307692307692
          },
          "0.95": {
            "n": 16,
            "errors": 5,
            "error_rate": 0.3125
          },
          "0.99": {
            "n": 4,
            "errors": 1,
            "error_rate": 0.25
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.725,
            "confidence_cutoff": 0.8354779953438343
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.703125,
            "confidence_cutoff": 0.6685881217893398
          }
        }
      }
    },
    "variants": {
      "clean": {
        "n": 656,
        "nll": 0.8473673030417037,
        "acc": 0.6478658536585366,
        "ece": 0.15246554096939913,
        "brier": 0.48125328370231507,
        "mean_conf": 0.8003313946279355,
        "confident_error_rate": 0.056402439024390245,
        "coverage_at_0_9": 0.39634146341463417,
        "accuracy_at_0_9": 0.8576923076923076,
        "coverage_at_5pct_error": 0.16920731707317074,
        "coverage_at_1pct_error": 0.07469512195121951,
        "aurc": 0.17511028693209713,
        "error_rate_at_0_9": 0.1423076923076923,
        "confidence_bias": 0.1524655409693989,
        "top_bins": {
          "0.9": {
            "n": 260,
            "errors": 37,
            "error_rate": 0.1423076923076923
          },
          "0.95": {
            "n": 208,
            "errors": 21,
            "error_rate": 0.10096153846153846
          },
          "0.99": {
            "n": 106,
            "errors": 5,
            "error_rate": 0.04716981132075472
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.8140243902439024,
            "confidence_cutoff": 0.855563355320146
          },
          "0.8": {
            "coverage": 0.8003048780487805,
            "accuracy": 0.7123809523809523,
            "confidence_cutoff": 0.6205200915575361
          }
        },
        "score_mae": 0.67300036097947,
        "ranked_probability_score": 0.2294783296027286
      },
      "none_absent": {
        "n": 36,
        "nll": 0.2678495272223286,
        "acc": 0.8611111111111112,
        "ece": 0.07302698178690374,
        "brier": 0.15272327314057532,
        "mean_conf": 0.8192931556434507,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.5,
        "accuracy_at_0_9": 1.0,
        "coverage_at_5pct_error": 0.7777777777777778,
        "coverage_at_1pct_error": 0.75,
        "aurc": 0.021981759765605926,
        "error_rate_at_0_9": 0.0,
        "confidence_bias": -0.04181795546766043,
        "top_bins": {
          "0.9": {
            "n": 18,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 16,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 8,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9444444917414034
          },
          "0.8": {
            "coverage": 0.8055555555555556,
            "accuracy": 0.9310344827586207,
            "confidence_cutoff": 0.599643828866869
          }
        }
      },
      "none_present": {
        "n": 36,
        "nll": 1.3620711504760268,
        "acc": 0.4722222222222222,
        "ece": 0.3203540084212916,
        "brier": 0.7174598637678359,
        "mean_conf": 0.7801488569930156,
        "confident_error_rate": 0.08333333333333333,
        "coverage_at_0_9": 0.4722222222222222,
        "accuracy_at_0_9": 0.8235294117647058,
        "coverage_at_5pct_error": 0.19444444444444445,
        "coverage_at_1pct_error": 0.19444444444444445,
        "aurc": 0.27124233664404535,
        "error_rate_at_0_9": 0.17647058823529413,
        "confidence_bias": 0.3079266347707934,
        "top_bins": {
          "0.9": {
            "n": 17,
            "errors": 3,
            "error_rate": 0.17647058823529413
          },
          "0.95": {
            "n": 14,
            "errors": 2,
            "error_rate": 0.14285714285714285
          },
          "0.99": {
            "n": 4,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.7777777777777778,
            "confidence_cutoff": 0.8994667250242726
          },
          "0.8": {
            "coverage": 0.8055555555555556,
            "accuracy": 0.5172413793103449,
            "confidence_cutoff": 0.5635468668007361
          }
        }
      },
      "permuted": {
        "n": 36,
        "nll": 0.5781990033628658,
        "acc": 0.7222222222222222,
        "ece": 0.10345419697625599,
        "brier": 0.34842882696675187,
        "mean_conf": 0.7946043557149448,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.5,
        "accuracy_at_0_9": 1.0,
        "coverage_at_5pct_error": 0.5833333333333334,
        "coverage_at_1pct_error": 0.5555555555555556,
        "aurc": 0.0849340315371725,
        "error_rate_at_0_9": 0.0,
        "confidence_bias": 0.0723821334927226,
        "top_bins": {
          "0.9": {
            "n": 18,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 17,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 13,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9148048670779622
          },
          "0.8": {
            "coverage": 0.8055555555555556,
            "accuracy": 0.7931034482758621,
            "confidence_cutoff": 0.5414417295288749
          }
        }
      }
    },
    "heldout_tasks": {
      "composition_held_and_or": {
        "n": 32,
        "nll": 0.9766401605534833,
        "acc": 0.65625,
        "ece": 0.2759884336884012,
        "brier": 0.5522134595300838,
        "mean_conf": 0.9322384336884012,
        "confident_error_rate": 0.21875,
        "coverage_at_0_9": 0.75,
        "accuracy_at_0_9": 0.7083333333333334,
        "coverage_at_5pct_error": 0.15625,
        "coverage_at_1pct_error": 0.15625,
        "aurc": 0.22049902211489347,
        "error_rate_at_0_9": 0.2916666666666667,
        "confidence_bias": 0.2759884336884012,
        "top_bins": {
          "0.9": {
            "n": 24,
            "errors": 7,
            "error_rate": 0.2916666666666667
          },
          "0.95": {
            "n": 20,
            "errors": 4,
            "error_rate": 0.2
          },
          "0.99": {
            "n": 5,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.53125,
            "accuracy": 0.8235294117647058,
            "confidence_cutoff": 0.9758496433679673
          },
          "0.8": {
            "coverage": 0.84375,
            "accuracy": 0.7407407407407407,
            "confidence_cutoff": 0.8654291033744812
          }
        }
      },
      "composition_held_conditional": {
        "n": 32,
        "nll": 0.6266604441471431,
        "acc": 0.65625,
        "ece": 0.14122441205349498,
        "brier": 0.44580578473896104,
        "mean_conf": 0.7488511904208259,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.09375,
        "accuracy_at_0_9": 1.0,
        "coverage_at_5pct_error": 0.125,
        "coverage_at_1pct_error": 0.125,
        "aurc": 0.23706863318202734,
        "error_rate_at_0_9": 0.0,
        "confidence_bias": 0.09260119042082593,
        "top_bins": {
          "0.9": {
            "n": 3,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.625,
            "confidence_cutoff": 0.7114255044289084
          },
          "0.8": {
            "coverage": 0.8125,
            "accuracy": 0.6538461538461539,
            "confidence_cutoff": 0.6640934944152832
          }
        }
      },
      "composition_held_or_not": {
        "n": 32,
        "nll": 0.8918989730655735,
        "acc": 0.5625,
        "ece": 0.27646008506339326,
        "brier": 0.6158885950256443,
        "mean_conf": 0.8339621685489521,
        "confident_error_rate": 0.15625,
        "coverage_at_0_9": 0.3125,
        "accuracy_at_0_9": 0.5,
        "coverage_at_5pct_error": 0.125,
        "coverage_at_1pct_error": 0.125,
        "aurc": 0.33870655616802325,
        "error_rate_at_0_9": 0.5,
        "confidence_bias": 0.2714621685489521,
        "top_bins": {
          "0.9": {
            "n": 10,
            "errors": 5,
            "error_rate": 0.5
          },
          "0.95": {
            "n": 5,
            "errors": 1,
            "error_rate": 0.2
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.6875,
            "confidence_cutoff": 0.8887970248878667
          },
          "0.8": {
            "coverage": 0.8125,
            "accuracy": 0.5769230769230769,
            "confidence_cutoff": 0.7215237451507893
          }
        }
      },
      "contrastive_authorization": {
        "n": 40,
        "nll": 0.18874489020814633,
        "acc": 0.95,
        "ece": 0.07035061559644126,
        "brier": 0.11583471559515715,
        "mean_conf": 0.8868357194684137,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.7,
        "accuracy_at_0_9": 1.0,
        "coverage_at_5pct_error": 1.0,
        "coverage_at_1pct_error": 0.75,
        "aurc": 0.01259009378789602,
        "error_rate_at_0_9": 0.0,
        "confidence_bias": -0.06316428053158629,
        "top_bins": {
          "0.9": {
            "n": 28,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 27,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 2,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9661230192971932
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.96875,
            "confidence_cutoff": 0.7516713886456854
          }
        }
      },
      "contrastive_deadline": {
        "n": 40,
        "nll": 1.2368328689761974,
        "acc": 0.35,
        "ece": 0.31302734346005,
        "brier": 0.7620299272400921,
        "mean_conf": 0.66302734346005,
        "confident_error_rate": 0.075,
        "coverage_at_0_9": 0.15,
        "accuracy_at_0_9": 0.5,
        "coverage_at_5pct_error": 0.05,
        "coverage_at_1pct_error": 0.05,
        "aurc": 0.518712885866291,
        "error_rate_at_0_9": 0.5,
        "confidence_bias": 0.31302734346005,
        "top_bins": {
          "0.9": {
            "n": 6,
            "errors": 3,
            "error_rate": 0.5
          },
          "0.95": {
            "n": 3,
            "errors": 1,
            "error_rate": 0.3333333333333333
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.45,
            "confidence_cutoff": 0.5926517844200134
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.40625,
            "confidence_cutoff": 0.5105663614059442
          }
        },
        "score_mae": 0.67300036097947,
        "ranked_probability_score": 0.2294783296027286
      },
      "emotion": {
        "n": 80,
        "nll": 1.6431966355834482,
        "acc": 0.5,
        "ece": 0.25791654554851956,
        "brier": 0.6869115290571809,
        "mean_conf": 0.7415177656608363,
        "confident_error_rate": 0.05,
        "coverage_at_0_9": 0.2875,
        "accuracy_at_0_9": 0.8260869565217391,
        "coverage_at_5pct_error": 0.0,
        "coverage_at_1pct_error": 0.0,
        "aurc": 0.3812803499509923,
        "error_rate_at_0_9": 0.17391304347826086,
        "confidence_bias": 0.24151776566083627,
        "top_bins": {
          "0.9": {
            "n": 23,
            "errors": 4,
            "error_rate": 0.17391304347826086
          },
          "0.95": {
            "n": 18,
            "errors": 4,
            "error_rate": 0.2222222222222222
          },
          "0.99": {
            "n": 9,
            "errors": 3,
            "error_rate": 0.3333333333333333
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.625,
            "confidence_cutoff": 0.7548409340511426
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.53125,
            "confidence_cutoff": 0.5661950644158379
          }
        }
      },
      "mmlu": {
        "n": 80,
        "nll": 1.361566206264453,
        "acc": 0.425,
        "ece": 0.2153956746505119,
        "brier": 0.7279459890735432,
        "mean_conf": 0.6363134116980336,
        "confident_error_rate": 0.0375,
        "coverage_at_0_9": 0.2125,
        "accuracy_at_0_9": 0.8235294117647058,
        "coverage_at_5pct_error": 0.1125,
        "coverage_at_1pct_error": 0.1125,
        "aurc": 0.3994876536778187,
        "error_rate_at_0_9": 0.17647058823529413,
        "confidence_bias": 0.21131341169803358,
        "top_bins": {
          "0.9": {
            "n": 17,
            "errors": 3,
            "error_rate": 0.17647058823529413
          },
          "0.95": {
            "n": 16,
            "errors": 2,
            "error_rate": 0.125
          },
          "0.99": {
            "n": 8,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.525,
            "confidence_cutoff": 0.6025702694004552
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.4375,
            "confidence_cutoff": 0.4188710575959402
          }
        }
      },
      "paws": {
        "n": 80,
        "nll": 0.8366071802737661,
        "acc": 0.6,
        "ece": 0.2162675509324284,
        "brier": 0.5800350137964118,
        "mean_conf": 0.7838076093454985,
        "confident_error_rate": 0.0375,
        "coverage_at_0_9": 0.125,
        "accuracy_at_0_9": 0.7,
        "coverage_at_5pct_error": 0.0375,
        "coverage_at_1pct_error": 0.0375,
        "aurc": 0.4046128749440664,
        "error_rate_at_0_9": 0.3,
        "confidence_bias": 0.18380760934549856,
        "top_bins": {
          "0.9": {
            "n": 10,
            "errors": 3,
            "error_rate": 0.3
          },
          "0.95": {
            "n": 4,
            "errors": 1,
            "error_rate": 0.25
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.625,
            "confidence_cutoff": 0.8137024752078965
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.578125,
            "confidence_cutoff": 0.6951168775558472
          }
        }
      },
      "qnli": {
        "n": 80,
        "nll": 0.38899775186075874,
        "acc": 0.8125,
        "ece": 0.07408691641504837,
        "brier": 0.25666992327751414,
        "mean_conf": 0.8845413569974496,
        "confident_error_rate": 0.0375,
        "coverage_at_0_9": 0.5875,
        "accuracy_at_0_9": 0.9361702127659575,
        "coverage_at_5pct_error": 0.575,
        "coverage_at_1pct_error": 0.4125,
        "aurc": 0.0549858681940542,
        "error_rate_at_0_9": 0.06382978723404255,
        "confidence_bias": 0.07204135699744962,
        "top_bins": {
          "0.9": {
            "n": 47,
            "errors": 3,
            "error_rate": 0.06382978723404255
          },
          "0.95": {
            "n": 37,
            "errors": 1,
            "error_rate": 0.02702702702702703
          },
          "0.99": {
            "n": 22,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.975,
            "confidence_cutoff": 0.9311983834505684
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.859375,
            "confidence_cutoff": 0.7695535444055837
          }
        }
      },
      "sciq": {
        "n": 80,
        "nll": 0.2497897921800699,
        "acc": 0.9,
        "ece": 0.046077537347433306,
        "brier": 0.12783578395936696,
        "mean_conf": 0.9368845477958011,
        "confident_error_rate": 0.025,
        "coverage_at_0_9": 0.825,
        "accuracy_at_0_9": 0.9696969696969697,
        "coverage_at_5pct_error": 0.9375,
        "coverage_at_1pct_error": 0.6875,
        "aurc": 0.012234213706852841,
        "error_rate_at_0_9": 0.030303030303030304,
        "confidence_bias": 0.03688454779580108,
        "top_bins": {
          "0.9": {
            "n": 66,
            "errors": 2,
            "error_rate": 0.030303030303030304
          },
          "0.95": {
            "n": 62,
            "errors": 2,
            "error_rate": 0.03225806451612903
          },
          "0.99": {
            "n": 56,
            "errors": 1,
            "error_rate": 0.017857142857142856
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9997168000196087
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.96875,
            "confidence_cutoff": 0.9325019582333779
          }
        }
      },
      "tweet_offensive": {
        "n": 80,
        "nll": 0.7573856080808232,
        "acc": 0.675,
        "ece": 0.13613936065564303,
        "brier": 0.48238323005946604,
        "mean_conf": 0.798700495923949,
        "confident_error_rate": 0.0875,
        "coverage_at_0_9": 0.325,
        "accuracy_at_0_9": 0.7307692307692307,
        "coverage_at_5pct_error": 0.025,
        "coverage_at_1pct_error": 0.025,
        "aurc": 0.2724541868080348,
        "error_rate_at_0_9": 0.2692307692307692,
        "confidence_bias": 0.12370049592394894,
        "top_bins": {
          "0.9": {
            "n": 26,
            "errors": 7,
            "error_rate": 0.2692307692307692
          },
          "0.95": {
            "n": 16,
            "errors": 5,
            "error_rate": 0.3125
          },
          "0.99": {
            "n": 4,
            "errors": 1,
            "error_rate": 0.25
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.725,
            "confidence_cutoff": 0.8354779953438343
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.703125,
            "confidence_cutoff": 0.6685881217893398
          }
        }
      }
    },
    "permutation": {
      "n": 36,
      "mean_max_delta": 0.07706799223592653,
      "flip_rate": 0.05555555555555555
    },
    "temperature": 2.2191389441356897,
    "calibrated_clean": {
      "n": 656,
      "nll": 0.7160535378887153,
      "acc": 0.6478658536585366,
      "ece": 0.04885377242921153,
      "brier": 0.43008777427102707,
      "mean_conf": 0.6676453244578785,
      "confident_error_rate": 0.003048780487804878,
      "coverage_at_0_9": 0.12347560975609756,
      "accuracy_at_0_9": 0.9753086419753086,
      "coverage_at_5pct_error": 0.14939024390243902,
      "coverage_at_1pct_error": 0.09298780487804878,
      "aurc": 0.170446517322486,
      "error_rate_at_0_9": 0.024691358024691357,
      "confidence_bias": 0.019779470799341947,
      "top_bins": {
        "0.9": {
          "n": 81,
          "errors": 2,
          "error_rate": 0.024691358024691357
        },
        "0.95": {
          "n": 52,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 13,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.8292682926829268,
          "confidence_cutoff": 0.671365229447973
        },
        "0.8": {
          "coverage": 0.8003048780487805,
          "accuracy": 0.7314285714285714,
          "confidence_cutoff": 0.5105853176548708
        }
      },
      "score_mae": 0.6974708226544527,
      "ranked_probability_score": 0.21183668750655005
    },
    "metric_policy": {
      "version": 2,
      "selective_ties": "whole_confidence_groups",
      "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee",
      "aurc": "right-step integral over whole confidence groups",
      "confident_error_rate": "high-confidence errors divided by all questions",
      "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9",
      "nll": "exact from logits when recorded; otherwise from floored probabilities",
      "nll_floor": 1e-09,
      "renormalize_returned_probabilities": true,
      "raw_sums_outside_1e_5": 0,
      "returned_zeros": 0
    },
    "coverage": {
      "requested_records": 764,
      "requested_questions": 764,
      "evaluated_records": 764,
      "evaluated_questions": 764,
      "rejected_records": 0,
      "truncated_records": 0
    },
    "latency_ms": {
      "median": 41.09924749991478,
      "p95": 41.60827904979669
    },
    "calibration": {
      "inference_temperature": 1.0,
      "additional_temperature": 2.2191389441356897,
      "logits_recorded": true
    },
    "suite_sha256": "31677c2256b406222e7d94ffdc0a02a70ce05746b9efe307876024c4e77291d1"
  },
  "provenance": {
    "config": {
      "epochs": 1,
      "seed": 1,
      "lr": 2e-05,
      "lora": 16,
      "accum": 2,
      "batch": 4,
      "perm_kl": 0.0,
      "perm_frac": 0.3,
      "ord_w": 0.0,
      "p_none": 0.1,
      "p_none_distract": 0.12,
      "p_distract": 0.15,
      "p_none_pair": 0.25,
      "synthetic_repeat": 1,
      "public_frac": 1.0,
      "head_lr": 0.0,
      "weight_decay": 0.01,
      "anchor_w": 0.0,
      "label_smoothing": 0.0,
      "brier_w": 0.0,
      "focal_gamma": 0.0,
      "dtype": "bf16",
      "checkpointing": 1,
      "replay": 6000,
      "max_state": 7552,
      "data": "evals/round15/joint/train.jsonl",
      "base": "Qwen/Qwen3.5-0.8B-Base",
      "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68",
      "init_from": "jaredpalmer/kev-0.8b"
    },
    "config_sha256": "6b30b3d48d805636aae8797c94bdce8ae8b7af866a06b6ac65bbc83ebb14d95a",
    "suite_sha256": "a8f50e481b7d90b97da049e0ff6a01cee2f1ed204aed61a8265af0edbb5514d2",
    "source_hashes": {
      "kev/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
      "kev/anchors.py": "6963eafdb276db5a6c94d939eae675c761555553b8d0f963ffd803448beaabbb",
      "kev/api.py": "7bffacfb762c626b8bc2f670f350295af5ccb0883dbe90239c7d2f8e5ef56582",
      "kev/autoresearch.py": "f7d7fabb9c0df7065bee3fec4aa4bee7028c5d1565847aefe2d74e3d7d40ed8a",
      "kev/benchmark.py": "10a892d76cb4c200e94d0a68ee74054aa1870d7f484c0530a531cacfa75032ed",
      "kev/calibrate.py": "c1eff4744fabd349e8abca86777a7aa0cbea44904c8223d3b61fca3d43741519",
      "kev/checkpoint.py": "f3edb4d159c0aa12d7006fc599fe6ebc3f2c288b3b69890c289f7dac451dd03d",
      "kev/compare.py": "3606dbf98bb305d66420158498cb837304d2edb8dab01cdaf2ec42964bef3435",
      "kev/composition.py": "f335ed17e18e0a544893db5e22b9a059e6ce1b2e14dbb863ac7d7bbf8f3e0536",
      "kev/contrastive.py": "cbb979aa5d40265ad0e64695f94b281d91751fa405811ddfa8212fede111edcf",
      "kev/cuda_graphs.py": "12f1953fea8f77c3c9dff29468202fbb37545145f93d3618194565a89949fd55",
      "kev/data.py": "e77a2fffeb8ee7e05b118893be8e38b8358aef97c3ca2e143cdce208201b92f9",
      "kev/device.py": "d1677fd98ec0979c7284546306e34e0d09298ca4042fc39eb2b32a74f4e975c4",
      "kev/evaluate.py": "999264f2837dcfbf2ec93601aa4745e701698ab850a43ad89324672a6604965c",
      "kev/experiment.py": "644adedf43fbd10566830fa6094e16dd001b8cad29d9b9da82f0e0f245ca06b1",
      "kev/jev.py": "acd4cc3f1844e438cc83a8d409c15ef78a5ef64d39ea10f583d75a6c7646b243",
      "kev/metrics.py": "dba8d90550999edcd642581fb4da0367bd4b39d08471815184fa732d11e137b9",
      "kev/mlx_model.py": "f582428796faf6bf962b772a69a6227ac3259ccbd3215c0c8b55ce28dbbd0909",
      "kev/model.py": "2634ffe7747d69473fb6596942c248e5df2586df3610bd1adb47e7e9acd99f96",
      "kev/plot.py": "0874bcff2885d8155a1cceade0de8a275a7163d3c3fd6aa0c294e8ccf5ff7e02",
      "kev/predictors.py": "4d02906c285d78bcbd91908f4fc86577ccc95c320d704817f6444aabbc4fcd4f",
      "kev/publish.py": "8abc9bcf4a01365697b05cf3c5ad0013462bd92d005f5616954e40ab1100c7aa",
      "kev/serve.py": "57d379bbcdeb3e4d4c3fdde5471e4dddeaf7710b8ed61ecad3466f3b67e6c610",
      "kev/study_v3.py": "7891150aa623e4479c7c789d4dc4662f186132b37b7bac1e1f072b9e08ee13e3",
      "kev/suite.py": "32b2882d1ea0fe5d03ba4d676a6180d580545f157e36fa715e7e89169ec99130",
      "kev/train.py": "68428ed43b3e362e52d4650dc5296ab88767061f7998a525715e1b306eeef474",
      "kev/transfer_v9.py": "0902409742151250a28af1fd8f42258b70fcc161f3deed3ee35fe3f473f3c763",
      "modal_app.py": "7c6bc859f7d3e7011539a8f5e9e90390288cc72b5d80f911c8841c068f4d1136",
      "pyproject.toml": "7c17fbe9efcadda3eb488b59adf1b6db5938cd33af4d3bd389ce4359b73fcc7d",
      "uv.lock": "a9922dbb89acdef78299fd2b4a8c3f7f0fa1b2bc08b55595b6926fa785a9c466"
    },
    "git_commit": "45923b7a3460b6d36358e2e143455902c1eb856b",
    "platform": "Linux-4.19.0-gvisor-x86_64-with-glibc2.36",
    "torch": "2.8.0+cu128",
    "device": "cuda",
    "gpu": "NVIDIA H200",
    "legacy_checkpoint": false,
    "measured_checkpoint": {
      "requested": "/runs/r15-08b/00-trial-0/checkpoint",
      "resolved": "/runs/r15-08b/00-trial-0/checkpoint",
      "head_sha256": "bc77488acd55d782a34ec875885c8647aa4afbb446fb1c4c3bb35fbc94ce865f",
      "adapter_sha256": "9b908623acb162118575f4e7a94524f9c139c335be4bfb74d6cfceca01e1885a",
      "inference_temperature": 1.0
    }
  },
  "mechanism_checks": {
    "n": 8,
    "packed_max_delta": 0.0,
    "sibling_max_delta": 0.0005142688751220703,
    "tolerance": 0.001,
    "passed": true
  },
  "gates": {
    "passed": false,
    "checks": {
      "complete_coverage": true,
      "isolation_and_packing": true,
      "transfer_complete": true,
      "heldout_pairs_at_least_70pct": false,
      "transfer_confident_errors_below_10pct": true
    },
    "policy": "Research screening only: 0.001 isolation; 5pp task regression; 70% heldout pair correctness; <=10% confident errors; no automatic release."
  },
  "calibration_fit": {
    "temperature": 2.2191389441356897,
    "aggregation": "micro",
    "objective": "raw-logit NLL",
    "split": "calibration",
    "rows_sha256": "685cd53bba6a49514b048094c342287c4ea78cbd8dc4b2338703b0f702cce0c4",
    "suite_sha256": "a8f50e481b7d90b97da049e0ff6a01cee2f1ed204aed61a8265af0edbb5514d2",
    "n": 1148
  },
  "wall_seconds": 3316.015814384,
  "promotable": false,
  "test_evaluated": false,
  "training_resources": {
    "wall_seconds": 3136.3209748268127,
    "records_seen": 30329,
    "requested_records": 22539,
    "truncated_records": 0,
    "rejected_records": 0,
    "optimizer_steps": 2818,
    "forward_tokens": 15531716,
    "peak_device_bytes": 23937857536,
    "device": "cuda",
    "dtype": "bf16",
    "batch": 4,
    "peak_rss_bytes": 10316873728
  }
}