{
  "objective": -0.3781113270188752,
  "paired_flip": {
    "pairs": 80,
    "flip_rate": 0.8875,
    "both_correct_rate": 0.8875,
    "invariant_pairs": 32,
    "invariance_rate": 1.0,
    "invariant_both_correct_rate": 0.90625
  },
  "clean": {
    "n": 1264,
    "nll": 0.47096940162233064,
    "acc": 0.8536392405063291,
    "ece": 0.0654366260188363,
    "brier": 0.21322991316558398,
    "mean_conf": 0.9178467644873951,
    "confident_error_rate": 0.041930379746835444,
    "coverage_at_0_9": 0.757120253164557,
    "accuracy_at_0_9": 0.9446185997910136,
    "confidence_bias": 0.064207523981066,
    "top_bins": {
      "0.9": {
        "n": 957,
        "errors": 53,
        "error_rate": 0.055381400208986416
      },
      "0.95": {
        "n": 911,
        "errors": 41,
        "error_rate": 0.04500548847420417
      },
      "0.99": {
        "n": 795,
        "errors": 27,
        "error_rate": 0.033962264150943396
      }
    },
    "selective": {
      "0.5": {
        "coverage": 0.5007911392405063,
        "accuracy": 0.9778830963665087,
        "confidence_cutoff": 0.9976429106296094
      },
      "0.8": {
        "coverage": 0.8006329113924051,
        "accuracy": 0.9268774703557312,
        "confidence_cutoff": 0.829332684596399
      }
    },
    "score_mae": 0.45394055553275015,
    "ranked_probability_score": 0.0717298259542393
  },
  "tasks": {
    "agnews": {
      "n": 80,
      "nll": 0.7764970089350782,
      "acc": 0.8375,
      "ece": 0.14810546152452678,
      "brier": 0.2815909368447759,
      "mean_conf": 0.9768289098497732,
      "confident_error_rate": 0.1125,
      "coverage_at_0_9": 0.9375,
      "accuracy_at_0_9": 0.88,
      "confidence_bias": 0.1393289098497732,
      "top_bins": {
        "0.9": {
          "n": 75,
          "errors": 9,
          "error_rate": 0.12
        },
        "0.95": {
          "n": 72,
          "errors": 8,
          "error_rate": 0.1111111111111111
        },
        "0.99": {
          "n": 63,
          "errors": 6,
          "error_rate": 0.09523809523809523
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.975,
          "confidence_cutoff": 0.9990865056765899
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.90625,
          "confidence_cutoff": 0.9894646850622684
        }
      }
    },
    "agnews_yn": {
      "n": 160,
      "nll": 0.24504913251170982,
      "acc": 0.925,
      "ece": 0.06460526620007352,
      "brier": 0.13290747891560833,
      "mean_conf": 0.9730792763846988,
      "confident_error_rate": 0.05,
      "coverage_at_0_9": 0.93125,
      "accuracy_at_0_9": 0.9463087248322147,
      "confidence_bias": 0.04807927638469878,
      "top_bins": {
        "0.9": {
          "n": 149,
          "errors": 8,
          "error_rate": 0.053691275167785234
        },
        "0.95": {
          "n": 140,
          "errors": 4,
          "error_rate": 0.02857142857142857
        },
        "0.99": {
          "n": 119,
          "errors": 2,
          "error_rate": 0.01680672268907563
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9978675951482209
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.9765625,
          "confidence_cutoff": 0.9823942798793209
        }
      }
    },
    "amazon": {
      "n": 80,
      "nll": 0.9351267400695216,
      "acc": 0.575,
      "ece": 0.11115798555393083,
      "brier": 0.49294492359563036,
      "mean_conf": 0.6772779379261196,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.125,
      "accuracy_at_0_9": 1.0,
      "confidence_bias": 0.10227793792611961,
      "top_bins": {
        "0.9": {
          "n": 10,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 8,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 2,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.8,
          "confidence_cutoff": 0.6379188511283084
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.671875,
          "confidence_cutoff": 0.502037328376669
        }
      },
      "score_mae": 0.5075561225779872,
      "ranked_probability_score": 0.080820187811701
    },
    "banking77": {
      "n": 80,
      "nll": 0.43079029036341554,
      "acc": 0.8875,
      "ece": 0.06732893165588627,
      "brier": 0.18477308183131425,
      "mean_conf": 0.9039776123393157,
      "confident_error_rate": 0.025,
      "coverage_at_0_9": 0.725,
      "accuracy_at_0_9": 0.9655172413793104,
      "confidence_bias": 0.016477612339315773,
      "top_bins": {
        "0.9": {
          "n": 58,
          "errors": 2,
          "error_rate": 0.034482758620689655
        },
        "0.95": {
          "n": 52,
          "errors": 2,
          "error_rate": 0.038461538461538464
        },
        "0.99": {
          "n": 45,
          "errors": 1,
          "error_rate": 0.022222222222222223
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.975,
          "confidence_cutoff": 0.9935029742060698
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.921875,
          "confidence_cutoff": 0.8365431118140658
        }
      }
    },
    "boolq": {
      "n": 80,
      "nll": 0.5692292766048166,
      "acc": 0.85,
      "ece": 0.12247463111478857,
      "brier": 0.2618067497766957,
      "mean_conf": 0.9492440640971932,
      "confident_error_rate": 0.075,
      "coverage_at_0_9": 0.825,
      "accuracy_at_0_9": 0.9090909090909091,
      "confidence_bias": 0.09924406409719322,
      "top_bins": {
        "0.9": {
          "n": 66,
          "errors": 6,
          "error_rate": 0.09090909090909091
        },
        "0.95": {
          "n": 64,
          "errors": 6,
          "error_rate": 0.09375
        },
        "0.99": {
          "n": 51,
          "errors": 3,
          "error_rate": 0.058823529411764705
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.95,
          "confidence_cutoff": 0.997317726504134
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.90625,
          "confidence_cutoff": 0.95449530011607
        }
      }
    },
    "composition_atom": {
      "n": 16,
      "nll": 0.0003952823957936077,
      "acc": 1.0,
      "ece": 0.0003950750452166307,
      "brier": 8.287164835155329e-07,
      "mean_conf": 0.9996049249547834,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 1.0,
      "accuracy_at_0_9": 1.0,
      "confidence_bias": -0.0003950750452166307,
      "top_bins": {
        "0.9": {
          "n": 16,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 16,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 16,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5625,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9998752945554933
        },
        "0.8": {
          "coverage": 0.8125,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9991505367027831
        }
      }
    },
    "composition_conditional": {
      "n": 16,
      "nll": 0.5019992150539019,
      "acc": 0.75,
      "ece": 0.1902398997443278,
      "brier": 0.3599381608463673,
      "mean_conf": 0.7634217561511185,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.3125,
      "accuracy_at_0_9": 1.0,
      "confidence_bias": 0.013421756151118514,
      "top_bins": {
        "0.9": {
          "n": 5,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 4,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.744914077070775
        },
        "0.8": {
          "coverage": 0.8125,
          "accuracy": 0.6923076923076923,
          "confidence_cutoff": 0.598910492691085
        }
      }
    },
    "composition_conjunction": {
      "n": 16,
      "nll": 0.0016869379224466628,
      "acc": 1.0,
      "ece": 0.0016845554564111076,
      "brier": 9.51034121939477e-06,
      "mean_conf": 0.9983154445435889,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 1.0,
      "accuracy_at_0_9": 1.0,
      "confidence_bias": -0.0016845554564111076,
      "top_bins": {
        "0.9": {
          "n": 16,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 16,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 16,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9993657412742123
        },
        "0.8": {
          "coverage": 0.875,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9967811070502508
        }
      }
    },
    "composition_disjunction": {
      "n": 16,
      "nll": 0.5432369947671923,
      "acc": 0.75,
      "ece": 0.20894149081016355,
      "brier": 0.38181577671679756,
      "mean_conf": 0.9589414908101636,
      "confident_error_rate": 0.125,
      "coverage_at_0_9": 0.875,
      "accuracy_at_0_9": 0.8571428571428571,
      "confidence_bias": 0.20894149081016355,
      "top_bins": {
        "0.9": {
          "n": 14,
          "errors": 2,
          "error_rate": 0.14285714285714285
        },
        "0.95": {
          "n": 12,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 9,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9934810838022192
        },
        "0.8": {
          "coverage": 0.875,
          "accuracy": 0.8571428571428571,
          "confidence_cutoff": 0.9103974050520304
        }
      }
    },
    "composition_exception": {
      "n": 16,
      "nll": 0.1551373177250381,
      "acc": 0.9375,
      "ece": 0.05574866742756779,
      "brier": 0.10455386244590524,
      "mean_conf": 0.9932486674275678,
      "confident_error_rate": 0.0625,
      "coverage_at_0_9": 1.0,
      "accuracy_at_0_9": 0.9375,
      "confidence_bias": 0.05574866742756779,
      "top_bins": {
        "0.9": {
          "n": 16,
          "errors": 1,
          "error_rate": 0.0625
        },
        "0.95": {
          "n": 15,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 15,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9990134402017437
        },
        "0.8": {
          "coverage": 0.8125,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9962711415415374
        }
      }
    },
    "composition_negation": {
      "n": 16,
      "nll": 0.003782036848642094,
      "acc": 1.0,
      "ece": 0.0037719108100304766,
      "brier": 4.034726206637131e-05,
      "mean_conf": 0.9962280891899695,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 1.0,
      "accuracy_at_0_9": 1.0,
      "confidence_bias": -0.0037719108100304766,
      "top_bins": {
        "0.9": {
          "n": 16,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 16,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 16,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5625,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9958158766414407
        },
        "0.8": {
          "coverage": 0.8125,
          "accuracy": 1.0,
          "confidence_cutoff": 0.994085511181873
        }
      }
    },
    "composition_nested_and": {
      "n": 16,
      "nll": 0.6928116765567718,
      "acc": 0.8125,
      "ece": 0.17331043627000298,
      "brier": 0.35599699730189693,
      "mean_conf": 0.985810436270003,
      "confident_error_rate": 0.1875,
      "coverage_at_0_9": 1.0,
      "accuracy_at_0_9": 0.8125,
      "confidence_bias": 0.17331043627000298,
      "top_bins": {
        "0.9": {
          "n": 16,
          "errors": 3,
          "error_rate": 0.1875
        },
        "0.95": {
          "n": 16,
          "errors": 3,
          "error_rate": 0.1875
        },
        "0.99": {
          "n": 9,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5625,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9903228993107223
        },
        "0.8": {
          "coverage": 0.8125,
          "accuracy": 0.9230769230769231,
          "confidence_cutoff": 0.9728845808970246
        }
      }
    },
    "composition_nested_or": {
      "n": 16,
      "nll": 0.016058082714040366,
      "acc": 1.0,
      "ece": 0.01574006900953684,
      "brier": 0.0012317899851369283,
      "mean_conf": 0.9842599309904632,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 1.0,
      "accuracy_at_0_9": 1.0,
      "confidence_bias": -0.01574006900953684,
      "top_bins": {
        "0.9": {
          "n": 16,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 15,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 11,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9949054004552627
        },
        "0.8": {
          "coverage": 0.875,
          "accuracy": 1.0,
          "confidence_cutoff": 0.95337582700002
        }
      }
    },
    "contrastive_age_eligibility": {
      "n": 24,
      "nll": 5.0117464260243464e-05,
      "acc": 1.0,
      "ece": 5.011085399608817e-05,
      "brier": 2.6433003191483944e-08,
      "mean_conf": 0.9999498891460039,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 1.0,
      "accuracy_at_0_9": 1.0,
      "confidence_bias": -5.011085399608817e-05,
      "top_bins": {
        "0.9": {
          "n": 24,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 24,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 24,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9999858161181404
        },
        "0.8": {
          "coverage": 0.8333333333333334,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9999397590096755
        }
      }
    },
    "contrastive_quantity_limit": {
      "n": 24,
      "nll": 0.009079523471059399,
      "acc": 1.0,
      "ece": 0.008472354885487591,
      "brier": 0.0021768652024033363,
      "mean_conf": 0.9915276451145125,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.9583333333333334,
      "accuracy_at_0_9": 1.0,
      "confidence_bias": -0.008472354885487476,
      "top_bins": {
        "0.9": {
          "n": 23,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 23,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 22,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.999989081677853
        },
        "0.8": {
          "coverage": 0.8333333333333334,
          "accuracy": 1.0,
          "confidence_cutoff": 0.99856468186255
        }
      }
    },
    "contrastive_return_window": {
      "n": 24,
      "nll": 0.48560927718172114,
      "acc": 0.875,
      "ece": 0.14552832909469873,
      "brier": 0.24864852616575747,
      "mean_conf": 0.948072710063192,
      "confident_error_rate": 0.125,
      "coverage_at_0_9": 0.8333333333333334,
      "accuracy_at_0_9": 0.85,
      "confidence_bias": 0.07307271006319205,
      "top_bins": {
        "0.9": {
          "n": 20,
          "errors": 3,
          "error_rate": 0.15
        },
        "0.95": {
          "n": 17,
          "errors": 2,
          "error_rate": 0.11764705882352941
        },
        "0.99": {
          "n": 10,
          "errors": 1,
          "error_rate": 0.1
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.9166666666666666,
          "confidence_cutoff": 0.9877289487620399
        },
        "0.8": {
          "coverage": 0.8333333333333334,
          "accuracy": 0.85,
          "confidence_cutoff": 0.9337131559678992
        }
      }
    },
    "contrastive_spend_threshold": {
      "n": 24,
      "nll": 1.653518426304661e-06,
      "acc": 1.0,
      "ece": 1.6535165444642175e-06,
      "brier": 5.729719218057532e-12,
      "mean_conf": 0.9999983464834555,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 1.0,
      "accuracy_at_0_9": 1.0,
      "confidence_bias": -1.6535165444642175e-06,
      "top_bins": {
        "0.9": {
          "n": 24,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 24,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 24,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.99999888450003
        },
        "0.8": {
          "coverage": 0.8333333333333334,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9999975114889905
        }
      }
    },
    "dbpedia14": {
      "n": 80,
      "nll": 0.352393166043517,
      "acc": 0.9625,
      "ece": 0.032725262113352305,
      "brier": 0.06411155116331552,
      "mean_conf": 0.9952252621133522,
      "confident_error_rate": 0.025,
      "coverage_at_0_9": 0.9875,
      "accuracy_at_0_9": 0.9746835443037974,
      "confidence_bias": 0.03272526211335214,
      "top_bins": {
        "0.9": {
          "n": 79,
          "errors": 2,
          "error_rate": 0.02531645569620253
        },
        "0.95": {
          "n": 78,
          "errors": 2,
          "error_rate": 0.02564102564102564
        },
        "0.99": {
          "n": 77,
          "errors": 2,
          "error_rate": 0.025974025974025976
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.975,
          "confidence_cutoff": 0.9999907673769224
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.984375,
          "confidence_cutoff": 0.9999730956432238
        }
      }
    },
    "imdb": {
      "n": 80,
      "nll": 0.28706193547875214,
      "acc": 0.925,
      "ece": 0.048198774610847796,
      "brier": 0.11596031960348152,
      "mean_conf": 0.9730559394793742,
      "confident_error_rate": 0.025,
      "coverage_at_0_9": 0.9125,
      "accuracy_at_0_9": 0.9726027397260274,
      "confidence_bias": 0.04805593947937414,
      "top_bins": {
        "0.9": {
          "n": 73,
          "errors": 2,
          "error_rate": 0.0273972602739726
        },
        "0.95": {
          "n": 72,
          "errors": 2,
          "error_rate": 0.027777777777777776
        },
        "0.99": {
          "n": 65,
          "errors": 2,
          "error_rate": 0.03076923076923077
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.975,
          "confidence_cutoff": 0.9995681686832004
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.96875,
          "confidence_cutoff": 0.9938746881478918
        }
      }
    },
    "mnli": {
      "n": 80,
      "nll": 0.888157101486706,
      "acc": 0.875,
      "ece": 0.11179936394023159,
      "brier": 0.23549742863127898,
      "mean_conf": 0.9842962604308741,
      "confident_error_rate": 0.1125,
      "coverage_at_0_9": 0.9375,
      "accuracy_at_0_9": 0.88,
      "confidence_bias": 0.10929626043087415,
      "top_bins": {
        "0.9": {
          "n": 75,
          "errors": 9,
          "error_rate": 0.12
        },
        "0.95": {
          "n": 74,
          "errors": 8,
          "error_rate": 0.10810810810810811
        },
        "0.99": {
          "n": 72,
          "errors": 7,
          "error_rate": 0.09722222222222222
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.925,
          "confidence_cutoff": 0.9999273470630932
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.9375,
          "confidence_cutoff": 0.9993270111828301
        }
      }
    },
    "sst5": {
      "n": 80,
      "nll": 0.9756452930629113,
      "acc": 0.575,
      "ece": 0.09928021239575943,
      "brier": 0.5462656054729043,
      "mean_conf": 0.6724271814545316,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.0125,
      "accuracy_at_0_9": 1.0,
      "confidence_bias": 0.09742718145453166,
      "top_bins": {
        "0.9": {
          "n": 1,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.99": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.65,
          "confidence_cutoff": 0.6593349139603858
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.640625,
          "confidence_cutoff": 0.5640514758708144
        }
      },
      "score_mae": 0.49195305628447744,
      "ranked_probability_score": 0.07798845711611709
    },
    "trec": {
      "n": 80,
      "nll": 0.29521071463972587,
      "acc": 0.9625,
      "ece": 0.04335316463941281,
      "brier": 0.0751019611146547,
      "mean_conf": 0.9896871670671918,
      "confident_error_rate": 0.0375,
      "coverage_at_0_9": 0.9625,
      "accuracy_at_0_9": 0.961038961038961,
      "confidence_bias": 0.02718716706719182,
      "top_bins": {
        "0.9": {
          "n": 77,
          "errors": 3,
          "error_rate": 0.03896103896103896
        },
        "0.95": {
          "n": 76,
          "errors": 2,
          "error_rate": 0.02631578947368421
        },
        "0.99": {
          "n": 74,
          "errors": 2,
          "error_rate": 0.02702702702702703
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9999414239640388
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.96875,
          "confidence_cutoff": 0.9994822242519307
        }
      }
    },
    "yelp": {
      "n": 80,
      "nll": 0.7097296189599043,
      "acc": 0.675,
      "ece": 0.11383632871560348,
      "brier": 0.4167826081797442,
      "mean_conf": 0.7656471258802438,
      "confident_error_rate": 0.0125,
      "coverage_at_0_9": 0.225,
      "accuracy_at_0_9": 0.9444444444444444,
      "confidence_bias": 0.0906471258802437,
      "top_bins": {
        "0.9": {
          "n": 18,
          "errors": 1,
          "error_rate": 0.05555555555555555
        },
        "0.95": {
          "n": 14,
          "errors": 1,
          "error_rate": 0.07142857142857142
        },
        "0.99": {
          "n": 2,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.825,
          "confidence_cutoff": 0.7783977437192959
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.75,
          "confidence_cutoff": 0.6039501956802334
        }
      },
      "score_mae": 0.36231248773578584,
      "ranked_probability_score": 0.056380832934899795
    },
    "yelp_yn": {
      "n": 80,
      "nll": 0.19993345467765128,
      "acc": 0.9,
      "ece": 0.05032214018379682,
      "brier": 0.11241742390597173,
      "mean_conf": 0.9503221401837969,
      "confident_error_rate": 0.025,
      "coverage_at_0_9": 0.875,
      "accuracy_at_0_9": 0.9714285714285714,
      "confidence_bias": 0.05032214018379688,
      "top_bins": {
        "0.9": {
          "n": 70,
          "errors": 2,
          "error_rate": 0.02857142857142857
        },
        "0.95": {
          "n": 63,
          "errors": 1,
          "error_rate": 0.015873015873015872
        },
        "0.99": {
          "n": 53,
          "errors": 1,
          "error_rate": 0.018867924528301886
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9976448653637842
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.984375,
          "confidence_cutoff": 0.9449425130872713
        }
      }
    }
  },
  "variants": {
    "clean": {
      "n": 1264,
      "nll": 0.47096940162233064,
      "acc": 0.8536392405063291,
      "ece": 0.0654366260188363,
      "brier": 0.21322991316558398,
      "mean_conf": 0.9178467644873951,
      "confident_error_rate": 0.041930379746835444,
      "coverage_at_0_9": 0.757120253164557,
      "accuracy_at_0_9": 0.9446185997910136,
      "confidence_bias": 0.064207523981066,
      "top_bins": {
        "0.9": {
          "n": 957,
          "errors": 53,
          "error_rate": 0.055381400208986416
        },
        "0.95": {
          "n": 911,
          "errors": 41,
          "error_rate": 0.04500548847420417
        },
        "0.99": {
          "n": 795,
          "errors": 27,
          "error_rate": 0.033962264150943396
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5007911392405063,
          "accuracy": 0.9778830963665087,
          "confidence_cutoff": 0.9976429106296094
        },
        "0.8": {
          "coverage": 0.8006329113924051,
          "accuracy": 0.9268774703557312,
          "confidence_cutoff": 0.829332684596399
        }
      },
      "score_mae": 0.45394055553275015,
      "ranked_probability_score": 0.0717298259542393
    },
    "none_absent": {
      "n": 60,
      "nll": 0.2677466069278561,
      "acc": 0.9,
      "ece": 0.04481354248148454,
      "brier": 0.12271424327012309,
      "mean_conf": 0.9448135424814846,
      "confident_error_rate": 0.016666666666666666,
      "coverage_at_0_9": 0.8333333333333334,
      "accuracy_at_0_9": 0.98,
      "confidence_bias": 0.044813542481484614,
      "top_bins": {
        "0.9": {
          "n": 50,
          "errors": 1,
          "error_rate": 0.02
        },
        "0.95": {
          "n": 43,
          "errors": 1,
          "error_rate": 0.023255813953488372
        },
        "0.99": {
          "n": 37,
          "errors": 1,
          "error_rate": 0.02702702702702703
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.9666666666666667,
          "confidence_cutoff": 0.9961424168910517
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.9791666666666666,
          "confidence_cutoff": 0.9062051924985772
        }
      }
    },
    "none_present": {
      "n": 60,
      "nll": 0.559854866685432,
      "acc": 0.8666666666666667,
      "ece": 0.05224881763254032,
      "brier": 0.19237936181427168,
      "mean_conf": 0.9124065831677801,
      "confident_error_rate": 0.03333333333333333,
      "coverage_at_0_9": 0.7,
      "accuracy_at_0_9": 0.9523809523809523,
      "confidence_bias": 0.04573991650111342,
      "top_bins": {
        "0.9": {
          "n": 42,
          "errors": 2,
          "error_rate": 0.047619047619047616
        },
        "0.95": {
          "n": 40,
          "errors": 1,
          "error_rate": 0.025
        },
        "0.99": {
          "n": 26,
          "errors": 1,
          "error_rate": 0.038461538461538464
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.9666666666666667,
          "confidence_cutoff": 0.984968995229399
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.9375,
          "confidence_cutoff": 0.846107297275265
        }
      }
    },
    "permuted": {
      "n": 84,
      "nll": 0.26296669876591966,
      "acc": 0.9642857142857143,
      "ece": 0.038646583486446284,
      "brier": 0.06057403759598415,
      "mean_conf": 0.9717462012238827,
      "confident_error_rate": 0.023809523809523808,
      "coverage_at_0_9": 0.9166666666666666,
      "accuracy_at_0_9": 0.974025974025974,
      "confidence_bias": 0.007460486938168365,
      "top_bins": {
        "0.9": {
          "n": 77,
          "errors": 2,
          "error_rate": 0.025974025974025976
        },
        "0.95": {
          "n": 74,
          "errors": 2,
          "error_rate": 0.02702702702702703
        },
        "0.99": {
          "n": 63,
          "errors": 1,
          "error_rate": 0.015873015873015872
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.9761904761904762,
          "confidence_cutoff": 0.9997112982180948
        },
        "0.8": {
          "coverage": 0.8095238095238095,
          "accuracy": 0.9852941176470589,
          "confidence_cutoff": 0.9815243603295957
        }
      }
    }
  },
  "heldout_tasks": {},
  "permutation": {
    "n": 60,
    "mean_max_delta": 0.02218218472923495,
    "flip_rate": 0.016666666666666666
  },
  "temperature": 1.6817928305074288,
  "calibrated_clean": {
    "n": 1264,
    "nll": 0.3928855341201546,
    "acc": 0.8536392405063291,
    "ece": 0.02783520772554953,
    "brier": 0.2051613865533046,
    "mean_conf": 0.8663882981180834,
    "confident_error_rate": 0.0245253164556962,
    "coverage_at_0_9": 0.6677215189873418,
    "accuracy_at_0_9": 0.9632701421800948,
    "confidence_bias": 0.012749057611754266,
    "top_bins": {
      "0.9": {
        "n": 844,
        "errors": 31,
        "error_rate": 0.03672985781990521
      },
      "0.95": {
        "n": 736,
        "errors": 24,
        "error_rate": 0.03260869565217391
      },
      "0.99": {
        "n": 419,
        "errors": 11,
        "error_rate": 0.026252983293556086
      }
    },
    "selective": {
      "0.5": {
        "coverage": 0.5,
        "accuracy": 0.9746835443037974,
        "confidence_cutoff": 0.970184381160819
      },
      "0.8": {
        "coverage": 0.8006329113924051,
        "accuracy": 0.9288537549407114,
        "confidence_cutoff": 0.6712393014030038
      }
    },
    "score_mae": 0.47852238769255256,
    "ranked_probability_score": 0.07118696124768613
  },
  "metric_policy": {
    "nll_floor": 1e-09,
    "renormalize_returned_probabilities": true,
    "raw_sums_outside_1e_5": 0,
    "returned_zeros": 0
  },
  "coverage": {
    "requested_records": 1204,
    "requested_questions": 1468,
    "evaluated_records": 1204,
    "evaluated_questions": 1468,
    "rejected_records": 0,
    "truncated_records": 0
  },
  "latency_ms": {
    "median": 51.816085000155,
    "p95": 173.47100315005264
  },
  "transfer": {
    "objective": -0.6958444571045241,
    "paired_flip": {
      "pairs": 64,
      "flip_rate": 0.734375,
      "both_correct_rate": 0.734375,
      "invariant_pairs": 24,
      "invariance_rate": 0.9583333333333334,
      "invariant_both_correct_rate": 0.8333333333333334
    },
    "clean": {
      "n": 656,
      "nll": 0.6976569692441479,
      "acc": 0.7896341463414634,
      "ece": 0.10175734176675508,
      "brier": 0.32825872907075376,
      "mean_conf": 0.8873174290671823,
      "confident_error_rate": 0.08231707317073171,
      "coverage_at_0_9": 0.6783536585365854,
      "accuracy_at_0_9": 0.8786516853932584,
      "confidence_bias": 0.0976832827257188,
      "top_bins": {
        "0.9": {
          "n": 445,
          "errors": 54,
          "error_rate": 0.12134831460674157
        },
        "0.95": {
          "n": 400,
          "errors": 39,
          "error_rate": 0.0975
        },
        "0.99": {
          "n": 262,
          "errors": 19,
          "error_rate": 0.07251908396946564
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.9146341463414634,
          "confidence_cutoff": 0.9790200062252951
        },
        "0.8": {
          "coverage": 0.8003048780487805,
          "accuracy": 0.8476190476190476,
          "confidence_cutoff": 0.75841224193573
        }
      },
      "score_mae": 0.45259144588683287,
      "ranked_probability_score": 0.20820131499001437
    },
    "tasks": {
      "composition_held_and_or": {
        "n": 32,
        "nll": 0.7456535624236502,
        "acc": 0.78125,
        "ece": 0.17958555475745153,
        "brier": 0.3359556523619844,
        "mean_conf": 0.9225585020071866,
        "confident_error_rate": 0.09375,
        "coverage_at_0_9": 0.78125,
        "accuracy_at_0_9": 0.88,
        "confidence_bias": 0.1413085020071866,
        "top_bins": {
          "0.9": {
            "n": 25,
            "errors": 3,
            "error_rate": 0.12
          },
          "0.95": {
            "n": 25,
            "errors": 3,
            "error_rate": 0.12
          },
          "0.99": {
            "n": 24,
            "errors": 3,
            "error_rate": 0.125
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.9375,
            "confidence_cutoff": 0.997011398040403
          },
          "0.8": {
            "coverage": 0.8125,
            "accuracy": 0.8846153846153846,
            "confidence_cutoff": 0.8473277470765
          }
        }
      },
      "composition_held_conditional": {
        "n": 32,
        "nll": 0.3332870284344554,
        "acc": 0.875,
        "ece": 0.13686614272503672,
        "brier": 0.21284482647640357,
        "mean_conf": 0.7938492418508554,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.40625,
        "accuracy_at_0_9": 1.0,
        "confidence_bias": -0.08115075814914463,
        "top_bins": {
          "0.9": {
            "n": 13,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 11,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 2,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.7695624232292175
          },
          "0.8": {
            "coverage": 0.8125,
            "accuracy": 0.8461538461538461,
            "confidence_cutoff": 0.6414507818654412
          }
        }
      },
      "composition_held_or_not": {
        "n": 32,
        "nll": 0.15833464456584684,
        "acc": 0.96875,
        "ece": 0.07149179201435074,
        "brier": 0.08199247173625367,
        "mean_conf": 0.8972582079856491,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.625,
        "accuracy_at_0_9": 1.0,
        "confidence_bias": -0.07149179201435085,
        "top_bins": {
          "0.9": {
            "n": 20,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 17,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9687480655265324
          },
          "0.8": {
            "coverage": 0.8125,
            "accuracy": 1.0,
            "confidence_cutoff": 0.791817462688164
          }
        }
      },
      "contrastive_authorization": {
        "n": 40,
        "nll": 0.0003820984206954688,
        "acc": 1.0,
        "ece": 0.0003819875787638427,
        "brier": 4.431386725588458e-07,
        "mean_conf": 0.9996180124212362,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 1.0,
        "accuracy_at_0_9": 1.0,
        "confidence_bias": -0.0003819875787638427,
        "top_bins": {
          "0.9": {
            "n": 40,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 40,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 40,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9997253036940529
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9993739016215527
          }
        }
      },
      "contrastive_deadline": {
        "n": 40,
        "nll": 2.381891379766067,
        "acc": 0.525,
        "ece": 0.4445411347971735,
        "brier": 0.8313373993065791,
        "mean_conf": 0.9505143411607072,
        "confident_error_rate": 0.35,
        "coverage_at_0_9": 0.85,
        "accuracy_at_0_9": 0.5882352941176471,
        "confidence_bias": 0.4255143411607072,
        "top_bins": {
          "0.9": {
            "n": 34,
            "errors": 14,
            "error_rate": 0.4117647058823529
          },
          "0.95": {
            "n": 32,
            "errors": 12,
            "error_rate": 0.375
          },
          "0.99": {
            "n": 25,
            "errors": 8,
            "error_rate": 0.32
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.75,
            "confidence_cutoff": 0.9963926273512684
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.625,
            "confidence_cutoff": 0.9538424791841433
          }
        },
        "score_mae": 0.45259144588683287,
        "ranked_probability_score": 0.20820131499001437
      },
      "emotion": {
        "n": 80,
        "nll": 1.2263539675718254,
        "acc": 0.6625,
        "ece": 0.14963081389980043,
        "brier": 0.5245823410352607,
        "mean_conf": 0.7403260524211139,
        "confident_error_rate": 0.075,
        "coverage_at_0_9": 0.3,
        "accuracy_at_0_9": 0.75,
        "confidence_bias": 0.07782605242111396,
        "top_bins": {
          "0.9": {
            "n": 24,
            "errors": 6,
            "error_rate": 0.25
          },
          "0.95": {
            "n": 15,
            "errors": 3,
            "error_rate": 0.2
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.7,
            "confidence_cutoff": 0.797799825668335
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.71875,
            "confidence_cutoff": 0.5535729084804598
          }
        }
      },
      "mmlu": {
        "n": 80,
        "nll": 0.8274234004446729,
        "acc": 0.65,
        "ece": 0.18294625228578548,
        "brier": 0.42173974468001135,
        "mean_conf": 0.7970014825127686,
        "confident_error_rate": 0.0125,
        "coverage_at_0_9": 0.45,
        "accuracy_at_0_9": 0.9722222222222222,
        "confidence_bias": 0.14700148251276857,
        "top_bins": {
          "0.9": {
            "n": 36,
            "errors": 1,
            "error_rate": 0.027777777777777776
          },
          "0.95": {
            "n": 30,
            "errors": 1,
            "error_rate": 0.03333333333333333
          },
          "0.99": {
            "n": 23,
            "errors": 1,
            "error_rate": 0.043478260869565216
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.925,
            "confidence_cutoff": 0.8614584193836526
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.765625,
            "confidence_cutoff": 0.5891912744154578
          }
        }
      },
      "paws": {
        "n": 80,
        "nll": 0.9617935415055063,
        "acc": 0.725,
        "ece": 0.23949173491457365,
        "brier": 0.4834975530791322,
        "mean_conf": 0.9521085317061271,
        "confident_error_rate": 0.2,
        "coverage_at_0_9": 0.875,
        "accuracy_at_0_9": 0.7714285714285715,
        "confidence_bias": 0.2271085317061271,
        "top_bins": {
          "0.9": {
            "n": 70,
            "errors": 16,
            "error_rate": 0.22857142857142856
          },
          "0.95": {
            "n": 63,
            "errors": 13,
            "error_rate": 0.20634920634920634
          },
          "0.99": {
            "n": 33,
            "errors": 5,
            "error_rate": 0.15151515151515152
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.825,
            "confidence_cutoff": 0.9852659738611202
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.796875,
            "confidence_cutoff": 0.9460779604702808
          }
        }
      },
      "qnli": {
        "n": 80,
        "nll": 0.28925677259060756,
        "acc": 0.8875,
        "ece": 0.07341603957619632,
        "brier": 0.18009551904084561,
        "mean_conf": 0.9343277556724386,
        "confident_error_rate": 0.0625,
        "coverage_at_0_9": 0.8125,
        "accuracy_at_0_9": 0.9230769230769231,
        "confidence_bias": 0.04682775567243869,
        "top_bins": {
          "0.9": {
            "n": 65,
            "errors": 5,
            "error_rate": 0.07692307692307693
          },
          "0.95": {
            "n": 59,
            "errors": 2,
            "error_rate": 0.03389830508474576
          },
          "0.99": {
            "n": 32,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9858154563418124
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.9375,
            "confidence_cutoff": 0.9094368151617287
          }
        }
      },
      "sciq": {
        "n": 80,
        "nll": 0.05181181132785821,
        "acc": 0.9875,
        "ece": 0.03175391680550414,
        "brier": 0.02513979546365842,
        "mean_conf": 0.9682344692294615,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.9,
        "accuracy_at_0_9": 1.0,
        "confidence_bias": -0.019265530770538497,
        "top_bins": {
          "0.9": {
            "n": 72,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 69,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 64,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9995528209501393
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9920047199640097
          }
        }
      },
      "tweet_offensive": {
        "n": 80,
        "nll": 0.6781008210985803,
        "acc": 0.75,
        "ece": 0.15591583779095422,
        "brier": 0.3886805236287897,
        "mean_conf": 0.8634720692805369,
        "confident_error_rate": 0.1125,
        "coverage_at_0_9": 0.575,
        "accuracy_at_0_9": 0.8043478260869565,
        "confidence_bias": 0.11347206928053688,
        "top_bins": {
          "0.9": {
            "n": 46,
            "errors": 9,
            "error_rate": 0.1956521739130435
          },
          "0.95": {
            "n": 39,
            "errors": 5,
            "error_rate": 0.1282051282051282
          },
          "0.99": {
            "n": 19,
            "errors": 2,
            "error_rate": 0.10526315789473684
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.875,
            "confidence_cutoff": 0.9428173514340226
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.796875,
            "confidence_cutoff": 0.7029978842436637
          }
        }
      }
    },
    "variants": {
      "clean": {
        "n": 656,
        "nll": 0.6976569692441479,
        "acc": 0.7896341463414634,
        "ece": 0.10175734176675508,
        "brier": 0.32825872907075376,
        "mean_conf": 0.8873174290671823,
        "confident_error_rate": 0.08231707317073171,
        "coverage_at_0_9": 0.6783536585365854,
        "accuracy_at_0_9": 0.8786516853932584,
        "confidence_bias": 0.0976832827257188,
        "top_bins": {
          "0.9": {
            "n": 445,
            "errors": 54,
            "error_rate": 0.12134831460674157
          },
          "0.95": {
            "n": 400,
            "errors": 39,
            "error_rate": 0.0975
          },
          "0.99": {
            "n": 262,
            "errors": 19,
            "error_rate": 0.07251908396946564
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.9146341463414634,
            "confidence_cutoff": 0.9790200062252951
          },
          "0.8": {
            "coverage": 0.8003048780487805,
            "accuracy": 0.8476190476190476,
            "confidence_cutoff": 0.75841224193573
          }
        },
        "score_mae": 0.45259144588683287,
        "ranked_probability_score": 0.20820131499001437
      },
      "none_absent": {
        "n": 36,
        "nll": 0.33905683993850544,
        "acc": 0.8611111111111112,
        "ece": 0.1400074421282228,
        "brier": 0.21437423073204523,
        "mean_conf": 0.8859076835917248,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.6388888888888888,
        "accuracy_at_0_9": 1.0,
        "confidence_bias": 0.02479657248061362,
        "top_bins": {
          "0.9": {
            "n": 23,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 16,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 8,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9417026261862247
          },
          "0.8": {
            "coverage": 0.8055555555555556,
            "accuracy": 0.9310344827586207,
            "confidence_cutoff": 0.8415947782198402
          }
        }
      },
      "none_present": {
        "n": 36,
        "nll": 1.1768707979298563,
        "acc": 0.5833333333333334,
        "ece": 0.2854879866906561,
        "brier": 0.5694624721478241,
        "mean_conf": 0.8057177842126594,
        "confident_error_rate": 0.08333333333333333,
        "coverage_at_0_9": 0.5277777777777778,
        "accuracy_at_0_9": 0.8421052631578947,
        "confidence_bias": 0.222384450879326,
        "top_bins": {
          "0.9": {
            "n": 19,
            "errors": 3,
            "error_rate": 0.15789473684210525
          },
          "0.95": {
            "n": 17,
            "errors": 2,
            "error_rate": 0.11764705882352941
          },
          "0.99": {
            "n": 9,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.8333333333333334,
            "confidence_cutoff": 0.9343934025851296
          },
          "0.8": {
            "coverage": 0.8055555555555556,
            "accuracy": 0.6206896551724138,
            "confidence_cutoff": 0.6258565324429688
          }
        }
      },
      "permuted": {
        "n": 36,
        "nll": 0.5841450773580558,
        "acc": 0.8055555555555556,
        "ece": 0.09385834965988679,
        "brier": 0.2762947760202843,
        "mean_conf": 0.8743721324628857,
        "confident_error_rate": 0.05555555555555555,
        "coverage_at_0_9": 0.6666666666666666,
        "accuracy_at_0_9": 0.9166666666666666,
        "confidence_bias": 0.06881657690733012,
        "top_bins": {
          "0.9": {
            "n": 24,
            "errors": 2,
            "error_rate": 0.08333333333333333
          },
          "0.95": {
            "n": 22,
            "errors": 2,
            "error_rate": 0.09090909090909091
          },
          "0.99": {
            "n": 14,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.8888888888888888,
            "confidence_cutoff": 0.9793524354721298
          },
          "0.8": {
            "coverage": 0.8055555555555556,
            "accuracy": 0.896551724137931,
            "confidence_cutoff": 0.7735453959337681
          }
        }
      }
    },
    "heldout_tasks": {
      "composition_held_and_or": {
        "n": 32,
        "nll": 0.7456535624236502,
        "acc": 0.78125,
        "ece": 0.17958555475745153,
        "brier": 0.3359556523619844,
        "mean_conf": 0.9225585020071866,
        "confident_error_rate": 0.09375,
        "coverage_at_0_9": 0.78125,
        "accuracy_at_0_9": 0.88,
        "confidence_bias": 0.1413085020071866,
        "top_bins": {
          "0.9": {
            "n": 25,
            "errors": 3,
            "error_rate": 0.12
          },
          "0.95": {
            "n": 25,
            "errors": 3,
            "error_rate": 0.12
          },
          "0.99": {
            "n": 24,
            "errors": 3,
            "error_rate": 0.125
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.9375,
            "confidence_cutoff": 0.997011398040403
          },
          "0.8": {
            "coverage": 0.8125,
            "accuracy": 0.8846153846153846,
            "confidence_cutoff": 0.8473277470765
          }
        }
      },
      "composition_held_conditional": {
        "n": 32,
        "nll": 0.3332870284344554,
        "acc": 0.875,
        "ece": 0.13686614272503672,
        "brier": 0.21284482647640357,
        "mean_conf": 0.7938492418508554,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.40625,
        "accuracy_at_0_9": 1.0,
        "confidence_bias": -0.08115075814914463,
        "top_bins": {
          "0.9": {
            "n": 13,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 11,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 2,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.7695624232292175
          },
          "0.8": {
            "coverage": 0.8125,
            "accuracy": 0.8461538461538461,
            "confidence_cutoff": 0.6414507818654412
          }
        }
      },
      "composition_held_or_not": {
        "n": 32,
        "nll": 0.15833464456584684,
        "acc": 0.96875,
        "ece": 0.07149179201435074,
        "brier": 0.08199247173625367,
        "mean_conf": 0.8972582079856491,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.625,
        "accuracy_at_0_9": 1.0,
        "confidence_bias": -0.07149179201435085,
        "top_bins": {
          "0.9": {
            "n": 20,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 17,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9687480655265324
          },
          "0.8": {
            "coverage": 0.8125,
            "accuracy": 1.0,
            "confidence_cutoff": 0.791817462688164
          }
        }
      },
      "contrastive_authorization": {
        "n": 40,
        "nll": 0.0003820984206954688,
        "acc": 1.0,
        "ece": 0.0003819875787638427,
        "brier": 4.431386725588458e-07,
        "mean_conf": 0.9996180124212362,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 1.0,
        "accuracy_at_0_9": 1.0,
        "confidence_bias": -0.0003819875787638427,
        "top_bins": {
          "0.9": {
            "n": 40,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 40,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 40,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9997253036940529
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9993739016215527
          }
        }
      },
      "contrastive_deadline": {
        "n": 40,
        "nll": 2.381891379766067,
        "acc": 0.525,
        "ece": 0.4445411347971735,
        "brier": 0.8313373993065791,
        "mean_conf": 0.9505143411607072,
        "confident_error_rate": 0.35,
        "coverage_at_0_9": 0.85,
        "accuracy_at_0_9": 0.5882352941176471,
        "confidence_bias": 0.4255143411607072,
        "top_bins": {
          "0.9": {
            "n": 34,
            "errors": 14,
            "error_rate": 0.4117647058823529
          },
          "0.95": {
            "n": 32,
            "errors": 12,
            "error_rate": 0.375
          },
          "0.99": {
            "n": 25,
            "errors": 8,
            "error_rate": 0.32
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.75,
            "confidence_cutoff": 0.9963926273512684
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.625,
            "confidence_cutoff": 0.9538424791841433
          }
        },
        "score_mae": 0.45259144588683287,
        "ranked_probability_score": 0.20820131499001437
      },
      "emotion": {
        "n": 80,
        "nll": 1.2263539675718254,
        "acc": 0.6625,
        "ece": 0.14963081389980043,
        "brier": 0.5245823410352607,
        "mean_conf": 0.7403260524211139,
        "confident_error_rate": 0.075,
        "coverage_at_0_9": 0.3,
        "accuracy_at_0_9": 0.75,
        "confidence_bias": 0.07782605242111396,
        "top_bins": {
          "0.9": {
            "n": 24,
            "errors": 6,
            "error_rate": 0.25
          },
          "0.95": {
            "n": 15,
            "errors": 3,
            "error_rate": 0.2
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.7,
            "confidence_cutoff": 0.797799825668335
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.71875,
            "confidence_cutoff": 0.5535729084804598
          }
        }
      },
      "mmlu": {
        "n": 80,
        "nll": 0.8274234004446729,
        "acc": 0.65,
        "ece": 0.18294625228578548,
        "brier": 0.42173974468001135,
        "mean_conf": 0.7970014825127686,
        "confident_error_rate": 0.0125,
        "coverage_at_0_9": 0.45,
        "accuracy_at_0_9": 0.9722222222222222,
        "confidence_bias": 0.14700148251276857,
        "top_bins": {
          "0.9": {
            "n": 36,
            "errors": 1,
            "error_rate": 0.027777777777777776
          },
          "0.95": {
            "n": 30,
            "errors": 1,
            "error_rate": 0.03333333333333333
          },
          "0.99": {
            "n": 23,
            "errors": 1,
            "error_rate": 0.043478260869565216
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.925,
            "confidence_cutoff": 0.8614584193836526
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.765625,
            "confidence_cutoff": 0.5891912744154578
          }
        }
      },
      "paws": {
        "n": 80,
        "nll": 0.9617935415055063,
        "acc": 0.725,
        "ece": 0.23949173491457365,
        "brier": 0.4834975530791322,
        "mean_conf": 0.9521085317061271,
        "confident_error_rate": 0.2,
        "coverage_at_0_9": 0.875,
        "accuracy_at_0_9": 0.7714285714285715,
        "confidence_bias": 0.2271085317061271,
        "top_bins": {
          "0.9": {
            "n": 70,
            "errors": 16,
            "error_rate": 0.22857142857142856
          },
          "0.95": {
            "n": 63,
            "errors": 13,
            "error_rate": 0.20634920634920634
          },
          "0.99": {
            "n": 33,
            "errors": 5,
            "error_rate": 0.15151515151515152
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.825,
            "confidence_cutoff": 0.9852659738611202
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.796875,
            "confidence_cutoff": 0.9460779604702808
          }
        }
      },
      "qnli": {
        "n": 80,
        "nll": 0.28925677259060756,
        "acc": 0.8875,
        "ece": 0.07341603957619632,
        "brier": 0.18009551904084561,
        "mean_conf": 0.9343277556724386,
        "confident_error_rate": 0.0625,
        "coverage_at_0_9": 0.8125,
        "accuracy_at_0_9": 0.9230769230769231,
        "confidence_bias": 0.04682775567243869,
        "top_bins": {
          "0.9": {
            "n": 65,
            "errors": 5,
            "error_rate": 0.07692307692307693
          },
          "0.95": {
            "n": 59,
            "errors": 2,
            "error_rate": 0.03389830508474576
          },
          "0.99": {
            "n": 32,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9858154563418124
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.9375,
            "confidence_cutoff": 0.9094368151617287
          }
        }
      },
      "sciq": {
        "n": 80,
        "nll": 0.05181181132785821,
        "acc": 0.9875,
        "ece": 0.03175391680550414,
        "brier": 0.02513979546365842,
        "mean_conf": 0.9682344692294615,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.9,
        "accuracy_at_0_9": 1.0,
        "confidence_bias": -0.019265530770538497,
        "top_bins": {
          "0.9": {
            "n": 72,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 69,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 64,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9995528209501393
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9920047199640097
          }
        }
      },
      "tweet_offensive": {
        "n": 80,
        "nll": 0.6781008210985803,
        "acc": 0.75,
        "ece": 0.15591583779095422,
        "brier": 0.3886805236287897,
        "mean_conf": 0.8634720692805369,
        "confident_error_rate": 0.1125,
        "coverage_at_0_9": 0.575,
        "accuracy_at_0_9": 0.8043478260869565,
        "confidence_bias": 0.11347206928053688,
        "top_bins": {
          "0.9": {
            "n": 46,
            "errors": 9,
            "error_rate": 0.1956521739130435
          },
          "0.95": {
            "n": 39,
            "errors": 5,
            "error_rate": 0.1282051282051282
          },
          "0.99": {
            "n": 19,
            "errors": 2,
            "error_rate": 0.10526315789473684
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.875,
            "confidence_cutoff": 0.9428173514340226
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.796875,
            "confidence_cutoff": 0.7029978842436637
          }
        }
      }
    },
    "permutation": {
      "n": 36,
      "mean_max_delta": 0.04950095278980541,
      "flip_rate": 0.05555555555555555
    },
    "temperature": 1.6817928305074288,
    "calibrated_clean": {
      "n": 656,
      "nll": 0.5645439847672864,
      "acc": 0.7896341463414634,
      "ece": 0.0621233006894711,
      "brier": 0.3098930396153164,
      "mean_conf": 0.8183328357646178,
      "confident_error_rate": 0.042682926829268296,
      "coverage_at_0_9": 0.5076219512195121,
      "accuracy_at_0_9": 0.9159159159159159,
      "confidence_bias": 0.028698689423154344,
      "top_bins": {
        "0.9": {
          "n": 333,
          "errors": 28,
          "error_rate": 0.08408408408408409
        },
        "0.95": {
          "n": 224,
          "errors": 14,
          "error_rate": 0.0625
        },
        "0.99": {
          "n": 85,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.9146341463414634,
          "confidence_cutoff": 0.9045152638026881
        },
        "0.8": {
          "coverage": 0.8003048780487805,
          "accuracy": 0.8514285714285714,
          "confidence_cutoff": 0.6315002649515302
        }
      },
      "score_mae": 0.4483170247504605,
      "ranked_probability_score": 0.18781810424878748
    },
    "metric_policy": {
      "nll_floor": 1e-09,
      "renormalize_returned_probabilities": true,
      "raw_sums_outside_1e_5": 0,
      "returned_zeros": 0
    },
    "coverage": {
      "requested_records": 764,
      "requested_questions": 764,
      "evaluated_records": 764,
      "evaluated_questions": 764,
      "rejected_records": 0,
      "truncated_records": 0
    },
    "latency_ms": {
      "median": 44.0643375000036,
      "p95": 52.84930554976199
    },
    "suite_sha256": "31677c2256b406222e7d94ffdc0a02a70ce05746b9efe307876024c4e77291d1"
  },
  "provenance": {
    "config": {
      "epochs": 2,
      "seed": 1,
      "lr": 5e-05,
      "lora": 16,
      "accum": 2,
      "batch": 4,
      "perm_kl": 0.0,
      "perm_frac": 0.3,
      "ord_w": 0.0,
      "p_none": 0.1,
      "p_none_distract": 0.12,
      "p_distract": 0.15,
      "p_none_pair": 0.25,
      "synthetic_repeat": 1,
      "public_frac": 1.0,
      "head_lr": 0.0,
      "weight_decay": 0.01,
      "anchor_w": 0.0,
      "base": "Qwen/Qwen3-4B-Base",
      "dtype": "bf16",
      "checkpointing": 1
    },
    "config_sha256": "7401c9d55a0961c57c94d559125675d98461bff8e78cd47575e279dc6a40d832",
    "suite_sha256": "a8f50e481b7d90b97da049e0ff6a01cee2f1ed204aed61a8265af0edbb5514d2",
    "source_hashes": {
      "kev/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
      "kev/anchors.py": "089d8a5493502bb26f540eb1c5e681780ca0bb01276073d4e1733211f15d0e10",
      "kev/api.py": "cdb0602684d798ccd3fc2c86be622f2a8e7bcdeee64d07f95604c7d3fd4701ec",
      "kev/autoresearch.py": "0a8aa6b57c1c9ba93d25b2cf631b03aed2e686e374c1148002eec48765b7b1fc",
      "kev/benchmark.py": "4dccec28be6becf6c3dcc8c186e8e1908a6482f15f602a60b440cc9cb4ac8a59",
      "kev/compare.py": "bd0445f021de59e35c7bff9304e39dd2e1211e3877a453575594ae7b81b0ada4",
      "kev/composition.py": "f335ed17e18e0a544893db5e22b9a059e6ce1b2e14dbb863ac7d7bbf8f3e0536",
      "kev/contrastive.py": "b1ce87a389b34b4d54a9eaca3ec2ec0809f64ae6640f4fa99785cea9293e3831",
      "kev/data.py": "9729f8f497b02aca542915d6f1bea3ff55956629d7d9610370aea91ee66efd23",
      "kev/evaluate.py": "0567c95bf8e6e6600bd7d356da064aaffcdad475be75c8bdf3a4d2b669a12e35",
      "kev/experiment.py": "c635394fe56c9aa11765f8bbf8f133dcf3e57cbe8f2bce4ebdc7b4e85fa536ff",
      "kev/jev.py": "e0213782359ba2f95adbf045ddaf0a008b08b4162bf6a9b66d4ce55fc91a51cb",
      "kev/model.py": "839dd7632ec291ac4740035c86f780f8f13da2b74b54a146ce90ff0ce8de527b",
      "kev/plot.py": "d689c7dd18cde7f9ea77cb4f55c50ff7da1880b21cb2ecf244348e45842a1382",
      "kev/publish.py": "b88c710140e7717ba42d539fdbe785507ac8b9d56b673641ade61c4136bb7dc9",
      "kev/serve.py": "7c8807379e915ceafc743e663278262c5c9e9e4a13d214c988cff21e11e4ebb5",
      "kev/study_v3.py": "7aa85ef92dd519dcf03a74b07dba400d7c533e9e457e7849b94b19cabafc4fba",
      "kev/suite.py": "5678fdcaf9a3e13efc3a77f514e2ddc5687e7407dddb86487461076f717c96fc",
      "kev/train.py": "e0df6503506d6a0cb5ccd35ae38bb072a6103383adf7beab4269bb1a1f5fcdce",
      "modal_app.py": "64381e069ddf3f560e8a8d8844268a888d7689c72377be899eec2dfc97d4d6d1",
      "pyproject.toml": "ce566ee11b68c0cb587bb591943641f403d761d820817693dae7590faacf9888",
      "uv.lock": "846fdc64a8bd6279a17a4365f18cd4e92ba9622d50d378aafcb5be6604296138"
    },
    "git_commit": "29d71c78368657b3a522729a01c748ea15272abc",
    "platform": "Linux-4.19.0-gvisor-x86_64-with-glibc2.36",
    "torch": "2.8.0+cu128",
    "device": "cuda",
    "gpu": "NVIDIA H200",
    "legacy_checkpoint": false
  },
  "mechanism_checks": {
    "n": 8,
    "packed_max_delta": 0.0,
    "sibling_max_delta": 1.7285346984863281e-06,
    "tolerance": 0.001,
    "passed": true
  },
  "gates": {
    "passed": true,
    "checks": {
      "complete_coverage": true,
      "isolation_and_packing": true,
      "transfer_complete": true,
      "heldout_pairs_at_least_70pct": true,
      "transfer_confident_errors_below_10pct": true
    },
    "policy": "Research screening only: 1e-3 isolation; 5pp task regression; 70% heldout pair correctness; <=10% confident errors; no automatic release."
  },
  "wall_seconds": 2587.115574389,
  "promotable": false,
  "test_evaluated": false,
  "training_resources": {
    "wall_seconds": 2378.645946741104,
    "records_seen": 30530,
    "requested_records": 25152,
    "truncated_records": 0,
    "rejected_records": 0,
    "optimizer_steps": 3144,
    "forward_tokens": 5860024,
    "peak_device_bytes": 23007304704,
    "device": "cuda",
    "dtype": "bf16",
    "batch": 4,
    "peak_rss_bytes": 25029832704
  }
}