{
  "objective": -0.5837225842185488,
  "paired_flip": {
    "pairs": 80,
    "flip_rate": 0.775,
    "both_correct_rate": 0.7625,
    "invariant_pairs": 32,
    "invariance_rate": 1.0,
    "invariant_both_correct_rate": 0.8125
  },
  "unknowable": null,
  "clean": {
    "n": 1264,
    "nll": 0.677277750842542,
    "acc": 0.8251582278481012,
    "ece": 0.10968515087361608,
    "brier": 0.2759729702219201,
    "mean_conf": 0.9329791785729107,
    "confident_error_rate": 0.07911392405063292,
    "coverage_at_0_9": 0.8061708860759493,
    "accuracy_at_0_9": 0.9018645731108931,
    "coverage_at_5pct_error": 0.5632911392405063,
    "coverage_at_1pct_error": 0.17325949367088608,
    "confidence_bias": 0.10782095072480946,
    "top_bins": {
      "0.9": {
        "n": 1019,
        "errors": 100,
        "error_rate": 0.09813542688910697
      },
      "0.95": {
        "n": 934,
        "errors": 75,
        "error_rate": 0.08029978586723768
      },
      "0.99": {
        "n": 786,
        "errors": 43,
        "error_rate": 0.054707379134860054
      }
    },
    "selective": {
      "0.5": {
        "coverage": 0.5,
        "accuracy": 0.9620253164556962,
        "confidence_cutoff": 0.998565945579216
      },
      "0.8": {
        "coverage": 0.8006329113924051,
        "accuracy": 0.9021739130434783,
        "confidence_cutoff": 0.9062608044404319
      }
    },
    "score_mae": 0.49005627978925503,
    "ranked_probability_score": 0.08996848010227619
  },
  "tasks": {
    "agnews": {
      "n": 80,
      "nll": 0.9832496835003219,
      "acc": 0.8375,
      "ece": 0.15513109503024222,
      "brier": 0.3025716783162474,
      "mean_conf": 0.9695536627637408,
      "confident_error_rate": 0.1125,
      "coverage_at_0_9": 0.9125,
      "accuracy_at_0_9": 0.8767123287671232,
      "coverage_at_5pct_error": 0.6375,
      "coverage_at_1pct_error": 0.45,
      "confidence_bias": 0.13205366276374075,
      "top_bins": {
        "0.9": {
          "n": 73,
          "errors": 9,
          "error_rate": 0.1232876712328767
        },
        "0.95": {
          "n": 72,
          "errors": 9,
          "error_rate": 0.125
        },
        "0.99": {
          "n": 64,
          "errors": 7,
          "error_rate": 0.109375
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.975,
          "confidence_cutoff": 0.9999153535825172
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.890625,
          "confidence_cutoff": 0.9912545823258385
        }
      }
    },
    "agnews_yn": {
      "n": 160,
      "nll": 0.3115965548177428,
      "acc": 0.925,
      "ece": 0.0603462185065268,
      "brier": 0.1318512819711422,
      "mean_conf": 0.980261737940278,
      "confident_error_rate": 0.05,
      "coverage_at_0_9": 0.9375,
      "accuracy_at_0_9": 0.9466666666666667,
      "coverage_at_5pct_error": 0.9125,
      "coverage_at_1pct_error": 0.5,
      "confidence_bias": 0.055261737940278,
      "top_bins": {
        "0.9": {
          "n": 150,
          "errors": 8,
          "error_rate": 0.05333333333333334
        },
        "0.95": {
          "n": 147,
          "errors": 8,
          "error_rate": 0.05442176870748299
        },
        "0.99": {
          "n": 136,
          "errors": 4,
          "error_rate": 0.029411764705882353
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9994875802345914
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.96875,
          "confidence_cutoff": 0.9969696708954863
        }
      }
    },
    "amazon": {
      "n": 80,
      "nll": 1.2366158624032093,
      "acc": 0.575,
      "ece": 0.20630787161160322,
      "brier": 0.5776284368946147,
      "mean_conf": 0.7813078716116031,
      "confident_error_rate": 0.05,
      "coverage_at_0_9": 0.3625,
      "accuracy_at_0_9": 0.8620689655172413,
      "coverage_at_5pct_error": 0.15,
      "coverage_at_1pct_error": 0.15,
      "confidence_bias": 0.20630787161160313,
      "top_bins": {
        "0.9": {
          "n": 29,
          "errors": 4,
          "error_rate": 0.13793103448275862
        },
        "0.95": {
          "n": 20,
          "errors": 3,
          "error_rate": 0.15
        },
        "0.99": {
          "n": 5,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.8,
          "confidence_cutoff": 0.804787614863595
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.65625,
          "confidence_cutoff": 0.6037764923189102
        }
      },
      "score_mae": 0.5011906275246643,
      "ranked_probability_score": 0.08998982899356231
    },
    "banking77": {
      "n": 80,
      "nll": 0.4023205818047388,
      "acc": 0.85,
      "ece": 0.08929501496409659,
      "brier": 0.2009334459047487,
      "mean_conf": 0.8971821022144549,
      "confident_error_rate": 0.0125,
      "coverage_at_0_9": 0.75,
      "accuracy_at_0_9": 0.9833333333333333,
      "coverage_at_5pct_error": 0.8,
      "coverage_at_1pct_error": 0.725,
      "confidence_bias": 0.04718210221445496,
      "top_bins": {
        "0.9": {
          "n": 60,
          "errors": 1,
          "error_rate": 0.016666666666666666
        },
        "0.95": {
          "n": 58,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 48,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9989341306330574
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.953125,
          "confidence_cutoff": 0.8131230570950913
        }
      }
    },
    "boolq": {
      "n": 80,
      "nll": 1.1931741060605592,
      "acc": 0.7625,
      "ece": 0.18683346049861238,
      "brier": 0.4130432350526035,
      "mean_conf": 0.9465267519654077,
      "confident_error_rate": 0.1625,
      "coverage_at_0_9": 0.8125,
      "accuracy_at_0_9": 0.8,
      "coverage_at_5pct_error": 0.0125,
      "coverage_at_1pct_error": 0.0125,
      "confidence_bias": 0.18402675196540774,
      "top_bins": {
        "0.9": {
          "n": 65,
          "errors": 13,
          "error_rate": 0.2
        },
        "0.95": {
          "n": 61,
          "errors": 12,
          "error_rate": 0.19672131147540983
        },
        "0.99": {
          "n": 51,
          "errors": 10,
          "error_rate": 0.19607843137254902
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.825,
          "confidence_cutoff": 0.9969401287601459
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.8125,
          "confidence_cutoff": 0.9391503613934029
        }
      }
    },
    "composition_atom": {
      "n": 16,
      "nll": 0.0018204253681370606,
      "acc": 1.0,
      "ece": 0.0018176200570680923,
      "brier": 1.1192189656453611e-05,
      "mean_conf": 0.9981823799429319,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 1.0,
      "accuracy_at_0_9": 1.0,
      "coverage_at_5pct_error": 1.0,
      "coverage_at_1pct_error": 1.0,
      "confidence_bias": -0.0018176200570680923,
      "top_bins": {
        "0.9": {
          "n": 16,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 16,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 16,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9979979161178252
        },
        "0.8": {
          "coverage": 0.8125,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9979044254752123
        }
      }
    },
    "composition_conditional": {
      "n": 16,
      "nll": 1.407375578542251,
      "acc": 0.75,
      "ece": 0.3408496346386551,
      "brier": 0.5686769779830284,
      "mean_conf": 0.8849633266106525,
      "confident_error_rate": 0.25,
      "coverage_at_0_9": 0.6875,
      "accuracy_at_0_9": 0.6363636363636364,
      "coverage_at_5pct_error": 0.0625,
      "coverage_at_1pct_error": 0.0625,
      "confidence_bias": 0.13496332661065247,
      "top_bins": {
        "0.9": {
          "n": 11,
          "errors": 4,
          "error_rate": 0.36363636363636365
        },
        "0.95": {
          "n": 11,
          "errors": 4,
          "error_rate": 0.36363636363636365
        },
        "0.99": {
          "n": 7,
          "errors": 3,
          "error_rate": 0.42857142857142855
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.5,
          "confidence_cutoff": 0.9663209643099036
        },
        "0.8": {
          "coverage": 0.8125,
          "accuracy": 0.6923076923076923,
          "confidence_cutoff": 0.786160076283213
        }
      }
    },
    "composition_conjunction": {
      "n": 16,
      "nll": 0.33671509514484377,
      "acc": 0.9375,
      "ece": 0.037238846975122586,
      "brier": 0.1268404957604162,
      "mean_conf": 0.9747388469751226,
      "confident_error_rate": 0.0625,
      "coverage_at_0_9": 1.0,
      "accuracy_at_0_9": 0.9375,
      "coverage_at_5pct_error": 0.5625,
      "coverage_at_1pct_error": 0.5625,
      "confidence_bias": 0.037238846975122586,
      "top_bins": {
        "0.9": {
          "n": 16,
          "errors": 1,
          "error_rate": 0.0625
        },
        "0.95": {
          "n": 12,
          "errors": 1,
          "error_rate": 0.08333333333333333
        },
        "0.99": {
          "n": 10,
          "errors": 1,
          "error_rate": 0.1
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9947336048940597
        },
        "0.8": {
          "coverage": 0.8125,
          "accuracy": 0.9230769230769231,
          "confidence_cutoff": 0.9366926690443258
        }
      }
    },
    "composition_disjunction": {
      "n": 16,
      "nll": 0.7980323464815665,
      "acc": 0.625,
      "ece": 0.34042637355738437,
      "brier": 0.5754585999291262,
      "mean_conf": 0.9236151581808647,
      "confident_error_rate": 0.125,
      "coverage_at_0_9": 0.6875,
      "accuracy_at_0_9": 0.8181818181818182,
      "coverage_at_5pct_error": 0.5625,
      "coverage_at_1pct_error": 0.5625,
      "confidence_bias": 0.2986151581808647,
      "top_bins": {
        "0.9": {
          "n": 11,
          "errors": 2,
          "error_rate": 0.18181818181818182
        },
        "0.95": {
          "n": 9,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 5,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9873694740706329
        },
        "0.8": {
          "coverage": 0.875,
          "accuracy": 0.6428571428571429,
          "confidence_cutoff": 0.8276452175484533
        }
      }
    },
    "composition_exception": {
      "n": 16,
      "nll": 0.5900095558121554,
      "acc": 0.5625,
      "ece": 0.2816931411850216,
      "brier": 0.39583397870207226,
      "mean_conf": 0.8441931411850218,
      "confident_error_rate": 0.0625,
      "coverage_at_0_9": 0.625,
      "accuracy_at_0_9": 0.9,
      "coverage_at_5pct_error": 0.5,
      "coverage_at_1pct_error": 0.5,
      "confidence_bias": 0.28169314118502176,
      "top_bins": {
        "0.9": {
          "n": 10,
          "errors": 1,
          "error_rate": 0.1
        },
        "0.95": {
          "n": 10,
          "errors": 1,
          "error_rate": 0.1
        },
        "0.99": {
          "n": 5,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9760010333359338
        },
        "0.8": {
          "coverage": 0.8125,
          "accuracy": 0.6923076923076923,
          "confidence_cutoff": 0.631696857911161
        }
      }
    },
    "composition_negation": {
      "n": 16,
      "nll": 0.06080032875137842,
      "acc": 1.0,
      "ece": 0.05737638597491783,
      "brier": 0.01223893910076251,
      "mean_conf": 0.9426236140250821,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.875,
      "accuracy_at_0_9": 1.0,
      "coverage_at_5pct_error": 1.0,
      "coverage_at_1pct_error": 1.0,
      "confidence_bias": -0.0573763859749179,
      "top_bins": {
        "0.9": {
          "n": 14,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 11,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 1,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9676642634197532
        },
        "0.8": {
          "coverage": 0.8125,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9302116355236877
        }
      }
    },
    "composition_nested_and": {
      "n": 16,
      "nll": 0.6256150963330884,
      "acc": 0.8125,
      "ece": 0.14464532099308625,
      "brier": 0.348125633054877,
      "mean_conf": 0.9571453209930862,
      "confident_error_rate": 0.1875,
      "coverage_at_0_9": 1.0,
      "accuracy_at_0_9": 0.8125,
      "coverage_at_5pct_error": 0.3125,
      "coverage_at_1pct_error": 0.3125,
      "confidence_bias": 0.14464532099308625,
      "top_bins": {
        "0.9": {
          "n": 16,
          "errors": 3,
          "error_rate": 0.1875
        },
        "0.95": {
          "n": 10,
          "errors": 3,
          "error_rate": 0.3
        },
        "0.99": {
          "n": 4,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.875,
          "confidence_cutoff": 0.9574078703506022
        },
        "0.8": {
          "coverage": 0.8125,
          "accuracy": 0.7692307692307693,
          "confidence_cutoff": 0.9318581472390091
        }
      }
    },
    "composition_nested_or": {
      "n": 16,
      "nll": 0.07625810559742767,
      "acc": 0.9375,
      "ece": 0.05914285391270866,
      "brier": 0.04200748661529065,
      "mean_conf": 0.9485557788640286,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.9375,
      "accuracy_at_0_9": 1.0,
      "coverage_at_5pct_error": 0.9375,
      "coverage_at_1pct_error": 0.9375,
      "confidence_bias": 0.011055778864028643,
      "top_bins": {
        "0.9": {
          "n": 15,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 12,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 6,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9898232160226875
        },
        "0.8": {
          "coverage": 0.8125,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9273196502137393
        }
      }
    },
    "contrastive_age_eligibility": {
      "n": 24,
      "nll": 1.8499446775205282e-05,
      "acc": 1.0,
      "ece": 1.8499237824265435e-05,
      "brier": 8.357891696338058e-10,
      "mean_conf": 0.9999815007621757,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 1.0,
      "accuracy_at_0_9": 1.0,
      "coverage_at_5pct_error": 1.0,
      "coverage_at_1pct_error": 1.0,
      "confidence_bias": -1.8499237824265435e-05,
      "top_bins": {
        "0.9": {
          "n": 24,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 24,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 24,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9999830436286485
        },
        "0.8": {
          "coverage": 0.8333333333333334,
          "accuracy": 1.0,
          "confidence_cutoff": 0.999974184122501
        }
      }
    },
    "contrastive_quantity_limit": {
      "n": 24,
      "nll": 0.1125956044001396,
      "acc": 0.9583333333333334,
      "ece": 0.03624299722303426,
      "brier": 0.07197842498551321,
      "mean_conf": 0.9945763305563676,
      "confident_error_rate": 0.041666666666666664,
      "coverage_at_0_9": 1.0,
      "accuracy_at_0_9": 0.9583333333333334,
      "coverage_at_5pct_error": 1.0,
      "coverage_at_1pct_error": 0.9583333333333334,
      "confidence_bias": 0.03624299722303426,
      "top_bins": {
        "0.9": {
          "n": 24,
          "errors": 1,
          "error_rate": 0.041666666666666664
        },
        "0.95": {
          "n": 23,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 21,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9999334638396701
        },
        "0.8": {
          "coverage": 0.8333333333333334,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9981896653224097
        }
      }
    },
    "contrastive_return_window": {
      "n": 24,
      "nll": 0.6051529398630384,
      "acc": 0.6666666666666666,
      "ece": 0.21995455952093168,
      "brier": 0.371917448872205,
      "mean_conf": 0.8675996478133303,
      "confident_error_rate": 0.08333333333333333,
      "coverage_at_0_9": 0.625,
      "accuracy_at_0_9": 0.8666666666666667,
      "coverage_at_5pct_error": 0.041666666666666664,
      "coverage_at_1pct_error": 0.041666666666666664,
      "confidence_bias": 0.2009329811466637,
      "top_bins": {
        "0.9": {
          "n": 15,
          "errors": 2,
          "error_rate": 0.13333333333333333
        },
        "0.95": {
          "n": 12,
          "errors": 2,
          "error_rate": 0.16666666666666666
        },
        "0.99": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.8333333333333334,
          "confidence_cutoff": 0.9521516320165855
        },
        "0.8": {
          "coverage": 0.8333333333333334,
          "accuracy": 0.8,
          "confidence_cutoff": 0.7327820281023392
        }
      }
    },
    "contrastive_spend_threshold": {
      "n": 24,
      "nll": 0.00030279504157641773,
      "acc": 1.0,
      "ece": 0.0003026639771004369,
      "brier": 5.21775523659121e-07,
      "mean_conf": 0.9996973360228996,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 1.0,
      "accuracy_at_0_9": 1.0,
      "coverage_at_5pct_error": 1.0,
      "coverage_at_1pct_error": 1.0,
      "confidence_bias": -0.0003026639771004369,
      "top_bins": {
        "0.9": {
          "n": 24,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 24,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 24,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9999916121909319
        },
        "0.8": {
          "coverage": 0.8333333333333334,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9993623150563549
        }
      }
    },
    "dbpedia14": {
      "n": 80,
      "nll": 0.4604112284196398,
      "acc": 0.9625,
      "ece": 0.03734234636353262,
      "brier": 0.0749167806142053,
      "mean_conf": 0.9998423463635326,
      "confident_error_rate": 0.0375,
      "coverage_at_0_9": 1.0,
      "accuracy_at_0_9": 0.9625,
      "coverage_at_5pct_error": 1.0,
      "coverage_at_1pct_error": 0.35,
      "confidence_bias": 0.03734234636353262,
      "top_bins": {
        "0.9": {
          "n": 80,
          "errors": 3,
          "error_rate": 0.0375
        },
        "0.95": {
          "n": 80,
          "errors": 3,
          "error_rate": 0.0375
        },
        "0.99": {
          "n": 80,
          "errors": 3,
          "error_rate": 0.0375
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.975,
          "confidence_cutoff": 0.9999999475161038
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.984375,
          "confidence_cutoff": 0.9999997520431655
        }
      }
    },
    "imdb": {
      "n": 80,
      "nll": 0.5716301725699824,
      "acc": 0.8875,
      "ece": 0.09875640553643103,
      "brier": 0.19712326669594268,
      "mean_conf": 0.9785553763225823,
      "confident_error_rate": 0.0875,
      "coverage_at_0_9": 0.9375,
      "accuracy_at_0_9": 0.9066666666666666,
      "coverage_at_5pct_error": 0.8375,
      "coverage_at_1pct_error": 0.3625,
      "confidence_bias": 0.09105537632258232,
      "top_bins": {
        "0.9": {
          "n": 75,
          "errors": 7,
          "error_rate": 0.09333333333333334
        },
        "0.95": {
          "n": 72,
          "errors": 5,
          "error_rate": 0.06944444444444445
        },
        "0.99": {
          "n": 68,
          "errors": 4,
          "error_rate": 0.058823529411764705
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.975,
          "confidence_cutoff": 0.9999773407237053
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.953125,
          "confidence_cutoff": 0.9974316348485517
        }
      }
    },
    "mnli": {
      "n": 80,
      "nll": 1.1387845715116893,
      "acc": 0.8125,
      "ece": 0.16543833890354487,
      "brier": 0.3362546680191699,
      "mean_conf": 0.9719655603456216,
      "confident_error_rate": 0.1625,
      "coverage_at_0_9": 0.95,
      "accuracy_at_0_9": 0.8289473684210527,
      "coverage_at_5pct_error": 0.25,
      "coverage_at_1pct_error": 0.1875,
      "confidence_bias": 0.15946556034562165,
      "top_bins": {
        "0.9": {
          "n": 76,
          "errors": 13,
          "error_rate": 0.17105263157894737
        },
        "0.95": {
          "n": 71,
          "errors": 12,
          "error_rate": 0.16901408450704225
        },
        "0.99": {
          "n": 64,
          "errors": 6,
          "error_rate": 0.09375
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.9,
          "confidence_cutoff": 0.9992974611653868
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.90625,
          "confidence_cutoff": 0.9905674125869428
        }
      }
    },
    "sst5": {
      "n": 80,
      "nll": 1.5534444317420577,
      "acc": 0.5875,
      "ece": 0.23933171099649792,
      "brier": 0.7022323612305323,
      "mean_conf": 0.8124146265105597,
      "confident_error_rate": 0.2,
      "coverage_at_0_9": 0.4125,
      "accuracy_at_0_9": 0.5151515151515151,
      "coverage_at_5pct_error": 0.0375,
      "coverage_at_1pct_error": 0.0375,
      "confidence_bias": 0.2249146265105597,
      "top_bins": {
        "0.9": {
          "n": 33,
          "errors": 16,
          "error_rate": 0.48484848484848486
        },
        "0.95": {
          "n": 13,
          "errors": 5,
          "error_rate": 0.38461538461538464
        },
        "0.99": {
          "n": 2,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.575,
          "confidence_cutoff": 0.8458941242777314
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.578125,
          "confidence_cutoff": 0.6599644726386572
        }
      },
      "score_mae": 0.5693146229296997,
      "ranked_probability_score": 0.10965566832128679
    },
    "trec": {
      "n": 80,
      "nll": 0.2886405792332978,
      "acc": 0.9625,
      "ece": 0.039050754551288716,
      "brier": 0.07288648289502976,
      "mean_conf": 0.9895815825818847,
      "confident_error_rate": 0.025,
      "coverage_at_0_9": 0.975,
      "accuracy_at_0_9": 0.9743589743589743,
      "coverage_at_5pct_error": 1.0,
      "coverage_at_1pct_error": 0.6875,
      "confidence_bias": 0.027081582581884645,
      "top_bins": {
        "0.9": {
          "n": 78,
          "errors": 2,
          "error_rate": 0.02564102564102564
        },
        "0.95": {
          "n": 76,
          "errors": 2,
          "error_rate": 0.02631578947368421
        },
        "0.99": {
          "n": 76,
          "errors": 2,
          "error_rate": 0.02631578947368421
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9999995069938787
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.984375,
          "confidence_cutoff": 0.9999976468822103
        }
      }
    },
    "yelp": {
      "n": 80,
      "nll": 0.9293500772388761,
      "acc": 0.65,
      "ece": 0.20917757536782644,
      "brier": 0.5126908269440805,
      "mean_conf": 0.8206727370156578,
      "confident_error_rate": 0.0875,
      "coverage_at_0_9": 0.425,
      "accuracy_at_0_9": 0.7941176470588235,
      "coverage_at_5pct_error": 0.2625,
      "coverage_at_1pct_error": 0.175,
      "confidence_bias": 0.17067273701565777,
      "top_bins": {
        "0.9": {
          "n": 34,
          "errors": 7,
          "error_rate": 0.20588235294117646
        },
        "0.95": {
          "n": 22,
          "errors": 2,
          "error_rate": 0.09090909090909091
        },
        "0.99": {
          "n": 8,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.775,
          "confidence_cutoff": 0.8554426543815328
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.671875,
          "confidence_cutoff": 0.6485221260511311
        }
      },
      "score_mae": 0.3996635889134011,
      "ranked_probability_score": 0.07025994299197942
    },
    "yelp_yn": {
      "n": 80,
      "nll": 0.32542780116067804,
      "acc": 0.8875,
      "ece": 0.08498293375955343,
      "brier": 0.15938160338912247,
      "mean_conf": 0.9595849699746,
      "confident_error_rate": 0.0375,
      "coverage_at_0_9": 0.875,
      "accuracy_at_0_9": 0.9571428571428572,
      "coverage_at_5pct_error": 0.875,
      "coverage_at_1pct_error": 0.625,
      "confidence_bias": 0.07208496997460001,
      "top_bins": {
        "0.9": {
          "n": 70,
          "errors": 3,
          "error_rate": 0.04285714285714286
        },
        "0.95": {
          "n": 68,
          "errors": 3,
          "error_rate": 0.04411764705882353
        },
        "0.99": {
          "n": 61,
          "errors": 3,
          "error_rate": 0.04918032786885246
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9996372033356943
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.953125,
          "confidence_cutoff": 0.9804108300621437
        }
      }
    }
  },
  "variants": {
    "clean": {
      "n": 1264,
      "nll": 0.677277750842542,
      "acc": 0.8251582278481012,
      "ece": 0.10968515087361608,
      "brier": 0.2759729702219201,
      "mean_conf": 0.9329791785729107,
      "confident_error_rate": 0.07911392405063292,
      "coverage_at_0_9": 0.8061708860759493,
      "accuracy_at_0_9": 0.9018645731108931,
      "coverage_at_5pct_error": 0.5632911392405063,
      "coverage_at_1pct_error": 0.17325949367088608,
      "confidence_bias": 0.10782095072480946,
      "top_bins": {
        "0.9": {
          "n": 1019,
          "errors": 100,
          "error_rate": 0.09813542688910697
        },
        "0.95": {
          "n": 934,
          "errors": 75,
          "error_rate": 0.08029978586723768
        },
        "0.99": {
          "n": 786,
          "errors": 43,
          "error_rate": 0.054707379134860054
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.9620253164556962,
          "confidence_cutoff": 0.998565945579216
        },
        "0.8": {
          "coverage": 0.8006329113924051,
          "accuracy": 0.9021739130434783,
          "confidence_cutoff": 0.9062608044404319
        }
      },
      "score_mae": 0.49005627978925503,
      "ranked_probability_score": 0.08996848010227619
    },
    "none_absent": {
      "n": 60,
      "nll": 0.5141479457352874,
      "acc": 0.85,
      "ece": 0.11106580603119935,
      "brier": 0.22718497312140562,
      "mean_conf": 0.9610658060311994,
      "confident_error_rate": 0.08333333333333333,
      "coverage_at_0_9": 0.9,
      "accuracy_at_0_9": 0.9074074074074074,
      "coverage_at_5pct_error": 0.6666666666666666,
      "coverage_at_1pct_error": 0.23333333333333334,
      "confidence_bias": 0.11106580603119942,
      "top_bins": {
        "0.9": {
          "n": 54,
          "errors": 5,
          "error_rate": 0.09259259259259259
        },
        "0.95": {
          "n": 51,
          "errors": 5,
          "error_rate": 0.09803921568627451
        },
        "0.99": {
          "n": 36,
          "errors": 2,
          "error_rate": 0.05555555555555555
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.9666666666666667,
          "confidence_cutoff": 0.998077132042754
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.8958333333333334,
          "confidence_cutoff": 0.957759179102288
        }
      }
    },
    "none_present": {
      "n": 60,
      "nll": 0.8304065371572367,
      "acc": 0.8333333333333334,
      "ece": 0.11153206529511675,
      "brier": 0.27340139863288243,
      "mean_conf": 0.9448653986284501,
      "confident_error_rate": 0.06666666666666667,
      "coverage_at_0_9": 0.8166666666666667,
      "accuracy_at_0_9": 0.9183673469387755,
      "coverage_at_5pct_error": 0.6333333333333333,
      "coverage_at_1pct_error": 0.03333333333333333,
      "confidence_bias": 0.11153206529511672,
      "top_bins": {
        "0.9": {
          "n": 49,
          "errors": 4,
          "error_rate": 0.08163265306122448
        },
        "0.95": {
          "n": 43,
          "errors": 4,
          "error_rate": 0.09302325581395349
        },
        "0.99": {
          "n": 32,
          "errors": 1,
          "error_rate": 0.03125
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.9666666666666667,
          "confidence_cutoff": 0.992714404701043
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.9166666666666666,
          "confidence_cutoff": 0.9168633320077479
        }
      }
    },
    "permuted": {
      "n": 84,
      "nll": 0.48436865419846886,
      "acc": 0.9047619047619048,
      "ece": 0.07211805581863377,
      "brier": 0.15448812106112633,
      "mean_conf": 0.9768799605805386,
      "confident_error_rate": 0.05952380952380952,
      "coverage_at_0_9": 0.9404761904761905,
      "accuracy_at_0_9": 0.9367088607594937,
      "coverage_at_5pct_error": 0.9047619047619048,
      "coverage_at_1pct_error": 0.09523809523809523,
      "confidence_bias": 0.07211805581863384,
      "top_bins": {
        "0.9": {
          "n": 79,
          "errors": 5,
          "error_rate": 0.06329113924050633
        },
        "0.95": {
          "n": 77,
          "errors": 4,
          "error_rate": 0.05194805194805195
        },
        "0.99": {
          "n": 73,
          "errors": 3,
          "error_rate": 0.0410958904109589
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.9761904761904762,
          "confidence_cutoff": 0.9998084861230152
        },
        "0.8": {
          "coverage": 0.8095238095238095,
          "accuracy": 0.9705882352941176,
          "confidence_cutoff": 0.9936823144468198
        }
      }
    }
  },
  "heldout_tasks": {},
  "permutation": {
    "n": 60,
    "mean_max_delta": 0.02025688803068432,
    "flip_rate": 0.016666666666666666
  },
  "temperature": 2.6390158215457884,
  "calibrated_clean": {
    "n": 1264,
    "nll": 0.46507794327280627,
    "acc": 0.8251582278481012,
    "ece": 0.024966358925870097,
    "brier": 0.2496238544020784,
    "mean_conf": 0.8187350664838222,
    "confident_error_rate": 0.022151898734177215,
    "coverage_at_0_9": 0.5158227848101266,
    "accuracy_at_0_9": 0.9570552147239264,
    "coverage_at_5pct_error": 0.5435126582278481,
    "coverage_at_1pct_error": 0.15822784810126583,
    "confidence_bias": -0.006423161364279006,
    "top_bins": {
      "0.9": {
        "n": 652,
        "errors": 28,
        "error_rate": 0.04294478527607362
      },
      "0.95": {
        "n": 483,
        "errors": 13,
        "error_rate": 0.026915113871635612
      },
      "0.99": {
        "n": 142,
        "errors": 1,
        "error_rate": 0.007042253521126761
      }
    },
    "selective": {
      "0.5": {
        "coverage": 0.5,
        "accuracy": 0.9572784810126582,
        "confidence_cutoff": 0.9093676413006413
      },
      "0.8": {
        "coverage": 0.8006329113924051,
        "accuracy": 0.900197628458498,
        "confidence_cutoff": 0.628961610919671
      }
    },
    "score_mae": 0.5430814474849706,
    "ranked_probability_score": 0.08275479929730008
  },
  "metric_policy": {
    "nll_floor": 1e-09,
    "renormalize_returned_probabilities": true,
    "raw_sums_outside_1e_5": 0,
    "returned_zeros": 0
  },
  "coverage": {
    "requested_records": 1204,
    "requested_questions": 1468,
    "evaluated_records": 1204,
    "evaluated_questions": 1468,
    "rejected_records": 0,
    "truncated_records": 0
  },
  "latency_ms": {
    "median": 41.915772000010065,
    "p95": 44.87425409996604
  },
  "transfer": {
    "objective": -0.9002227022737674,
    "paired_flip": {
      "pairs": 64,
      "flip_rate": 0.484375,
      "both_correct_rate": 0.421875,
      "invariant_pairs": 24,
      "invariance_rate": 0.9583333333333334,
      "invariant_both_correct_rate": 0.5416666666666666
    },
    "unknowable": null,
    "clean": {
      "n": 656,
      "nll": 0.9222275811844616,
      "acc": 0.6524390243902439,
      "ece": 0.17939440499990641,
      "brier": 0.49870669415283536,
      "mean_conf": 0.8314353498515511,
      "confident_error_rate": 0.09908536585365854,
      "coverage_at_0_9": 0.5060975609756098,
      "accuracy_at_0_9": 0.8042168674698795,
      "coverage_at_5pct_error": 0.22865853658536586,
      "coverage_at_1pct_error": 0.08841463414634146,
      "confidence_bias": 0.17899632546130717,
      "top_bins": {
        "0.9": {
          "n": 332,
          "errors": 65,
          "error_rate": 0.19578313253012047
        },
        "0.95": {
          "n": 249,
          "errors": 35,
          "error_rate": 0.14056224899598393
        },
        "0.99": {
          "n": 157,
          "errors": 8,
          "error_rate": 0.050955414012738856
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.8079268292682927,
          "confidence_cutoff": 0.9024031758308411
        },
        "0.8": {
          "coverage": 0.8003048780487805,
          "accuracy": 0.7219047619047619,
          "confidence_cutoff": 0.6487010024681538
        }
      },
      "score_mae": 0.6515860158402881,
      "ranked_probability_score": 0.23189794760605645
    },
    "tasks": {
      "composition_held_and_or": {
        "n": 32,
        "nll": 1.1899194065298853,
        "acc": 0.65625,
        "ece": 0.3062826763573561,
        "brier": 0.6297875816783778,
        "mean_conf": 0.9625326763573561,
        "confident_error_rate": 0.3125,
        "coverage_at_0_9": 0.9375,
        "accuracy_at_0_9": 0.6666666666666666,
        "coverage_at_5pct_error": 0.125,
        "coverage_at_1pct_error": 0.125,
        "confidence_bias": 0.30628267635735607,
        "top_bins": {
          "0.9": {
            "n": 30,
            "errors": 10,
            "error_rate": 0.3333333333333333
          },
          "0.95": {
            "n": 24,
            "errors": 8,
            "error_rate": 0.3333333333333333
          },
          "0.99": {
            "n": 4,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.8125,
            "confidence_cutoff": 0.976970766787314
          },
          "0.8": {
            "coverage": 0.8125,
            "accuracy": 0.6153846153846154,
            "confidence_cutoff": 0.9484939888568524
          }
        }
      },
      "composition_held_conditional": {
        "n": 32,
        "nll": 0.675725781843872,
        "acc": 0.59375,
        "ece": 0.18786636617174748,
        "brier": 0.4705037394069092,
        "mean_conf": 0.7816163661717475,
        "confident_error_rate": 0.0625,
        "coverage_at_0_9": 0.25,
        "accuracy_at_0_9": 0.75,
        "coverage_at_5pct_error": 0.1875,
        "coverage_at_1pct_error": 0.1875,
        "confidence_bias": 0.18786636617174746,
        "top_bins": {
          "0.9": {
            "n": 8,
            "errors": 2,
            "error_rate": 0.25
          },
          "0.95": {
            "n": 3,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.8125,
            "confidence_cutoff": 0.8704544098878388
          },
          "0.8": {
            "coverage": 0.84375,
            "accuracy": 0.6666666666666666,
            "confidence_cutoff": 0.5778257422991319
          }
        }
      },
      "composition_held_or_not": {
        "n": 32,
        "nll": 0.8207520004606448,
        "acc": 0.5625,
        "ece": 0.2816247209338629,
        "brier": 0.5616030854083928,
        "mean_conf": 0.8278648881415858,
        "confident_error_rate": 0.03125,
        "coverage_at_0_9": 0.3125,
        "accuracy_at_0_9": 0.9,
        "coverage_at_5pct_error": 0.09375,
        "coverage_at_1pct_error": 0.09375,
        "confidence_bias": 0.26536488814158576,
        "top_bins": {
          "0.9": {
            "n": 10,
            "errors": 1,
            "error_rate": 0.1
          },
          "0.95": {
            "n": 5,
            "errors": 1,
            "error_rate": 0.2
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.53125,
            "accuracy": 0.7058823529411765,
            "confidence_cutoff": 0.867844091834961
          },
          "0.8": {
            "coverage": 0.8125,
            "accuracy": 0.6538461538461539,
            "confidence_cutoff": 0.6750653184930885
          }
        }
      },
      "contrastive_authorization": {
        "n": 40,
        "nll": 0.06778782153149417,
        "acc": 0.975,
        "ece": 0.031234135570157376,
        "brier": 0.036192439631432066,
        "mean_conf": 0.9497480518201697,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.85,
        "accuracy_at_0_9": 1.0,
        "coverage_at_5pct_error": 1.0,
        "coverage_at_1pct_error": 0.95,
        "confidence_bias": -0.02525194817983023,
        "top_bins": {
          "0.9": {
            "n": 34,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 31,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 27,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.99714271867781
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9445846571260915
          }
        }
      },
      "contrastive_deadline": {
        "n": 40,
        "nll": 1.3889026704649357,
        "acc": 0.375,
        "ece": 0.3829928525155385,
        "brier": 0.7922274946031982,
        "mean_conf": 0.7042415556863528,
        "confident_error_rate": 0.1,
        "coverage_at_0_9": 0.225,
        "accuracy_at_0_9": 0.5555555555555556,
        "coverage_at_5pct_error": 0.0,
        "coverage_at_1pct_error": 0.0,
        "confidence_bias": 0.3292415556863528,
        "top_bins": {
          "0.9": {
            "n": 9,
            "errors": 4,
            "error_rate": 0.4444444444444444
          },
          "0.95": {
            "n": 5,
            "errors": 2,
            "error_rate": 0.4
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.55,
            "confidence_cutoff": 0.6421262312885925
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.40625,
            "confidence_cutoff": 0.5615110481010533
          }
        },
        "score_mae": 0.6515860158402881,
        "ranked_probability_score": 0.23189794760605645
      },
      "emotion": {
        "n": 80,
        "nll": 1.6769960423140386,
        "acc": 0.5375,
        "ece": 0.22586976855913896,
        "brier": 0.6678512282134752,
        "mean_conf": 0.7472712517372814,
        "confident_error_rate": 0.075,
        "coverage_at_0_9": 0.3125,
        "accuracy_at_0_9": 0.76,
        "coverage_at_5pct_error": 0.0,
        "coverage_at_1pct_error": 0.0,
        "confidence_bias": 0.2097712517372814,
        "top_bins": {
          "0.9": {
            "n": 25,
            "errors": 6,
            "error_rate": 0.24
          },
          "0.95": {
            "n": 19,
            "errors": 4,
            "error_rate": 0.21052631578947367
          },
          "0.99": {
            "n": 9,
            "errors": 3,
            "error_rate": 0.3333333333333333
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.65,
            "confidence_cutoff": 0.7725859213122825
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.578125,
            "confidence_cutoff": 0.56208528383785
          }
        }
      },
      "mmlu": {
        "n": 80,
        "nll": 1.406507662100838,
        "acc": 0.425,
        "ece": 0.23086071722009255,
        "brier": 0.7337000166926988,
        "mean_conf": 0.6558607172200925,
        "confident_error_rate": 0.05,
        "coverage_at_0_9": 0.225,
        "accuracy_at_0_9": 0.7777777777777778,
        "coverage_at_5pct_error": 0.1,
        "coverage_at_1pct_error": 0.1,
        "confidence_bias": 0.23086071722009255,
        "top_bins": {
          "0.9": {
            "n": 18,
            "errors": 4,
            "error_rate": 0.2222222222222222
          },
          "0.95": {
            "n": 16,
            "errors": 3,
            "error_rate": 0.1875
          },
          "0.99": {
            "n": 11,
            "errors": 1,
            "error_rate": 0.09090909090909091
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.525,
            "confidence_cutoff": 0.618546923871794
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.453125,
            "confidence_cutoff": 0.4300697519593815
          }
        }
      },
      "paws": {
        "n": 80,
        "nll": 1.1248226915361372,
        "acc": 0.55,
        "ece": 0.3200331593407043,
        "brier": 0.7235656563851223,
        "mean_conf": 0.8654864054853849,
        "confident_error_rate": 0.2625,
        "coverage_at_0_9": 0.5,
        "accuracy_at_0_9": 0.475,
        "coverage_at_5pct_error": 0.0125,
        "coverage_at_1pct_error": 0.0125,
        "confidence_bias": 0.31548640548538487,
        "top_bins": {
          "0.9": {
            "n": 40,
            "errors": 21,
            "error_rate": 0.525
          },
          "0.95": {
            "n": 12,
            "errors": 5,
            "error_rate": 0.4166666666666667
          },
          "0.99": {
            "n": 2,
            "errors": 1,
            "error_rate": 0.5
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.475,
            "confidence_cutoff": 0.9014056727918538
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.546875,
            "confidence_cutoff": 0.8111088751091231
          }
        }
      },
      "qnli": {
        "n": 80,
        "nll": 0.4122149692919013,
        "acc": 0.85,
        "ece": 0.09433166058006281,
        "brier": 0.24752108114076724,
        "mean_conf": 0.8990905626919042,
        "confident_error_rate": 0.0375,
        "coverage_at_0_9": 0.6625,
        "accuracy_at_0_9": 0.9433962264150944,
        "coverage_at_5pct_error": 0.5125,
        "coverage_at_1pct_error": 0.3625,
        "confidence_bias": 0.04909056269190426,
        "top_bins": {
          "0.9": {
            "n": 53,
            "errors": 3,
            "error_rate": 0.05660377358490566
          },
          "0.95": {
            "n": 43,
            "errors": 3,
            "error_rate": 0.06976744186046512
          },
          "0.99": {
            "n": 29,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.95,
            "confidence_cutoff": 0.9583688735354272
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.890625,
            "confidence_cutoff": 0.793002498045601
          }
        }
      },
      "sciq": {
        "n": 80,
        "nll": 0.24132037549325877,
        "acc": 0.9125,
        "ece": 0.04025150891820237,
        "brier": 0.12820506248919616,
        "mean_conf": 0.9387513504714097,
        "confident_error_rate": 0.025,
        "coverage_at_0_9": 0.8375,
        "accuracy_at_0_9": 0.9701492537313433,
        "coverage_at_5pct_error": 0.925,
        "coverage_at_1pct_error": 0.7,
        "confidence_bias": 0.026251350471409696,
        "top_bins": {
          "0.9": {
            "n": 67,
            "errors": 2,
            "error_rate": 0.029850746268656716
          },
          "0.95": {
            "n": 63,
            "errors": 1,
            "error_rate": 0.015873015873015872
          },
          "0.99": {
            "n": 58,
            "errors": 1,
            "error_rate": 0.017241379310344827
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9998130113129569
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.96875,
            "confidence_cutoff": 0.9417403771936594
          }
        }
      },
      "tweet_offensive": {
        "n": 80,
        "nll": 0.8975003034444349,
        "acc": 0.675,
        "ece": 0.18809786512896348,
        "brier": 0.5095841174172036,
        "mean_conf": 0.8555092051551096,
        "confident_error_rate": 0.15,
        "coverage_at_0_9": 0.475,
        "accuracy_at_0_9": 0.6842105263157895,
        "coverage_at_5pct_error": 0.0375,
        "coverage_at_1pct_error": 0.0375,
        "confidence_bias": 0.1805092051551096,
        "top_bins": {
          "0.9": {
            "n": 38,
            "errors": 12,
            "error_rate": 0.3157894736842105
          },
          "0.95": {
            "n": 28,
            "errors": 8,
            "error_rate": 0.2857142857142857
          },
          "0.99": {
            "n": 17,
            "errors": 2,
            "error_rate": 0.11764705882352941
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.675,
            "confidence_cutoff": 0.8908563720881537
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.703125,
            "confidence_cutoff": 0.7286377114869528
          }
        }
      }
    },
    "variants": {
      "clean": {
        "n": 656,
        "nll": 0.9222275811844616,
        "acc": 0.6524390243902439,
        "ece": 0.17939440499990641,
        "brier": 0.49870669415283536,
        "mean_conf": 0.8314353498515511,
        "confident_error_rate": 0.09908536585365854,
        "coverage_at_0_9": 0.5060975609756098,
        "accuracy_at_0_9": 0.8042168674698795,
        "coverage_at_5pct_error": 0.22865853658536586,
        "coverage_at_1pct_error": 0.08841463414634146,
        "confidence_bias": 0.17899632546130717,
        "top_bins": {
          "0.9": {
            "n": 332,
            "errors": 65,
            "error_rate": 0.19578313253012047
          },
          "0.95": {
            "n": 249,
            "errors": 35,
            "error_rate": 0.14056224899598393
          },
          "0.99": {
            "n": 157,
            "errors": 8,
            "error_rate": 0.050955414012738856
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.8079268292682927,
            "confidence_cutoff": 0.9024031758308411
          },
          "0.8": {
            "coverage": 0.8003048780487805,
            "accuracy": 0.7219047619047619,
            "confidence_cutoff": 0.6487010024681538
          }
        },
        "score_mae": 0.6515860158402881,
        "ranked_probability_score": 0.23189794760605645
      },
      "none_absent": {
        "n": 36,
        "nll": 0.38820769746181283,
        "acc": 0.8333333333333334,
        "ece": 0.08916457948776643,
        "brier": 0.22282945335191004,
        "mean_conf": 0.8007323164732282,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.4444444444444444,
        "accuracy_at_0_9": 1.0,
        "coverage_at_5pct_error": 0.75,
        "coverage_at_1pct_error": 0.4444444444444444,
        "confidence_bias": -0.03260101686010519,
        "top_bins": {
          "0.9": {
            "n": 16,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 13,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 6,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.9444444444444444,
            "confidence_cutoff": 0.8828300404102825
          },
          "0.8": {
            "coverage": 0.8055555555555556,
            "accuracy": 0.9310344827586207,
            "confidence_cutoff": 0.6286187226089864
          }
        }
      },
      "none_present": {
        "n": 36,
        "nll": 1.264206963123794,
        "acc": 0.4444444444444444,
        "ece": 0.31744480152703275,
        "brier": 0.653531262440586,
        "mean_conf": 0.7618892459714771,
        "confident_error_rate": 0.05555555555555555,
        "coverage_at_0_9": 0.3888888888888889,
        "accuracy_at_0_9": 0.8571428571428571,
        "coverage_at_5pct_error": 0.3055555555555556,
        "coverage_at_1pct_error": 0.3055555555555556,
        "confidence_bias": 0.3174448015270327,
        "top_bins": {
          "0.9": {
            "n": 14,
            "errors": 2,
            "error_rate": 0.14285714285714285
          },
          "0.95": {
            "n": 12,
            "errors": 1,
            "error_rate": 0.08333333333333333
          },
          "0.99": {
            "n": 8,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.7777777777777778,
            "confidence_cutoff": 0.8565133594916252
          },
          "0.8": {
            "coverage": 0.8055555555555556,
            "accuracy": 0.5517241379310345,
            "confidence_cutoff": 0.5275042540090179
          }
        }
      },
      "permuted": {
        "n": 36,
        "nll": 0.6298818160485206,
        "acc": 0.6944444444444444,
        "ece": 0.16106824698543198,
        "brier": 0.36143960629508687,
        "mean_conf": 0.7996874698756644,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.5277777777777778,
        "accuracy_at_0_9": 1.0,
        "coverage_at_5pct_error": 0.6111111111111112,
        "coverage_at_1pct_error": 0.5833333333333334,
        "confidence_bias": 0.10524302543122,
        "top_bins": {
          "0.9": {
            "n": 19,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 17,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 13,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9428915688779251
          },
          "0.8": {
            "coverage": 0.8055555555555556,
            "accuracy": 0.7586206896551724,
            "confidence_cutoff": 0.5394178434772673
          }
        }
      }
    },
    "heldout_tasks": {
      "composition_held_and_or": {
        "n": 32,
        "nll": 1.1899194065298853,
        "acc": 0.65625,
        "ece": 0.3062826763573561,
        "brier": 0.6297875816783778,
        "mean_conf": 0.9625326763573561,
        "confident_error_rate": 0.3125,
        "coverage_at_0_9": 0.9375,
        "accuracy_at_0_9": 0.6666666666666666,
        "coverage_at_5pct_error": 0.125,
        "coverage_at_1pct_error": 0.125,
        "confidence_bias": 0.30628267635735607,
        "top_bins": {
          "0.9": {
            "n": 30,
            "errors": 10,
            "error_rate": 0.3333333333333333
          },
          "0.95": {
            "n": 24,
            "errors": 8,
            "error_rate": 0.3333333333333333
          },
          "0.99": {
            "n": 4,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.8125,
            "confidence_cutoff": 0.976970766787314
          },
          "0.8": {
            "coverage": 0.8125,
            "accuracy": 0.6153846153846154,
            "confidence_cutoff": 0.9484939888568524
          }
        }
      },
      "composition_held_conditional": {
        "n": 32,
        "nll": 0.675725781843872,
        "acc": 0.59375,
        "ece": 0.18786636617174748,
        "brier": 0.4705037394069092,
        "mean_conf": 0.7816163661717475,
        "confident_error_rate": 0.0625,
        "coverage_at_0_9": 0.25,
        "accuracy_at_0_9": 0.75,
        "coverage_at_5pct_error": 0.1875,
        "coverage_at_1pct_error": 0.1875,
        "confidence_bias": 0.18786636617174746,
        "top_bins": {
          "0.9": {
            "n": 8,
            "errors": 2,
            "error_rate": 0.25
          },
          "0.95": {
            "n": 3,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.8125,
            "confidence_cutoff": 0.8704544098878388
          },
          "0.8": {
            "coverage": 0.84375,
            "accuracy": 0.6666666666666666,
            "confidence_cutoff": 0.5778257422991319
          }
        }
      },
      "composition_held_or_not": {
        "n": 32,
        "nll": 0.8207520004606448,
        "acc": 0.5625,
        "ece": 0.2816247209338629,
        "brier": 0.5616030854083928,
        "mean_conf": 0.8278648881415858,
        "confident_error_rate": 0.03125,
        "coverage_at_0_9": 0.3125,
        "accuracy_at_0_9": 0.9,
        "coverage_at_5pct_error": 0.09375,
        "coverage_at_1pct_error": 0.09375,
        "confidence_bias": 0.26536488814158576,
        "top_bins": {
          "0.9": {
            "n": 10,
            "errors": 1,
            "error_rate": 0.1
          },
          "0.95": {
            "n": 5,
            "errors": 1,
            "error_rate": 0.2
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.53125,
            "accuracy": 0.7058823529411765,
            "confidence_cutoff": 0.867844091834961
          },
          "0.8": {
            "coverage": 0.8125,
            "accuracy": 0.6538461538461539,
            "confidence_cutoff": 0.6750653184930885
          }
        }
      },
      "contrastive_authorization": {
        "n": 40,
        "nll": 0.06778782153149417,
        "acc": 0.975,
        "ece": 0.031234135570157376,
        "brier": 0.036192439631432066,
        "mean_conf": 0.9497480518201697,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.85,
        "accuracy_at_0_9": 1.0,
        "coverage_at_5pct_error": 1.0,
        "coverage_at_1pct_error": 0.95,
        "confidence_bias": -0.02525194817983023,
        "top_bins": {
          "0.9": {
            "n": 34,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 31,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 27,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.99714271867781
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9445846571260915
          }
        }
      },
      "contrastive_deadline": {
        "n": 40,
        "nll": 1.3889026704649357,
        "acc": 0.375,
        "ece": 0.3829928525155385,
        "brier": 0.7922274946031982,
        "mean_conf": 0.7042415556863528,
        "confident_error_rate": 0.1,
        "coverage_at_0_9": 0.225,
        "accuracy_at_0_9": 0.5555555555555556,
        "coverage_at_5pct_error": 0.0,
        "coverage_at_1pct_error": 0.0,
        "confidence_bias": 0.3292415556863528,
        "top_bins": {
          "0.9": {
            "n": 9,
            "errors": 4,
            "error_rate": 0.4444444444444444
          },
          "0.95": {
            "n": 5,
            "errors": 2,
            "error_rate": 0.4
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.55,
            "confidence_cutoff": 0.6421262312885925
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.40625,
            "confidence_cutoff": 0.5615110481010533
          }
        },
        "score_mae": 0.6515860158402881,
        "ranked_probability_score": 0.23189794760605645
      },
      "emotion": {
        "n": 80,
        "nll": 1.6769960423140386,
        "acc": 0.5375,
        "ece": 0.22586976855913896,
        "brier": 0.6678512282134752,
        "mean_conf": 0.7472712517372814,
        "confident_error_rate": 0.075,
        "coverage_at_0_9": 0.3125,
        "accuracy_at_0_9": 0.76,
        "coverage_at_5pct_error": 0.0,
        "coverage_at_1pct_error": 0.0,
        "confidence_bias": 0.2097712517372814,
        "top_bins": {
          "0.9": {
            "n": 25,
            "errors": 6,
            "error_rate": 0.24
          },
          "0.95": {
            "n": 19,
            "errors": 4,
            "error_rate": 0.21052631578947367
          },
          "0.99": {
            "n": 9,
            "errors": 3,
            "error_rate": 0.3333333333333333
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.65,
            "confidence_cutoff": 0.7725859213122825
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.578125,
            "confidence_cutoff": 0.56208528383785
          }
        }
      },
      "mmlu": {
        "n": 80,
        "nll": 1.406507662100838,
        "acc": 0.425,
        "ece": 0.23086071722009255,
        "brier": 0.7337000166926988,
        "mean_conf": 0.6558607172200925,
        "confident_error_rate": 0.05,
        "coverage_at_0_9": 0.225,
        "accuracy_at_0_9": 0.7777777777777778,
        "coverage_at_5pct_error": 0.1,
        "coverage_at_1pct_error": 0.1,
        "confidence_bias": 0.23086071722009255,
        "top_bins": {
          "0.9": {
            "n": 18,
            "errors": 4,
            "error_rate": 0.2222222222222222
          },
          "0.95": {
            "n": 16,
            "errors": 3,
            "error_rate": 0.1875
          },
          "0.99": {
            "n": 11,
            "errors": 1,
            "error_rate": 0.09090909090909091
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.525,
            "confidence_cutoff": 0.618546923871794
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.453125,
            "confidence_cutoff": 0.4300697519593815
          }
        }
      },
      "paws": {
        "n": 80,
        "nll": 1.1248226915361372,
        "acc": 0.55,
        "ece": 0.3200331593407043,
        "brier": 0.7235656563851223,
        "mean_conf": 0.8654864054853849,
        "confident_error_rate": 0.2625,
        "coverage_at_0_9": 0.5,
        "accuracy_at_0_9": 0.475,
        "coverage_at_5pct_error": 0.0125,
        "coverage_at_1pct_error": 0.0125,
        "confidence_bias": 0.31548640548538487,
        "top_bins": {
          "0.9": {
            "n": 40,
            "errors": 21,
            "error_rate": 0.525
          },
          "0.95": {
            "n": 12,
            "errors": 5,
            "error_rate": 0.4166666666666667
          },
          "0.99": {
            "n": 2,
            "errors": 1,
            "error_rate": 0.5
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.475,
            "confidence_cutoff": 0.9014056727918538
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.546875,
            "confidence_cutoff": 0.8111088751091231
          }
        }
      },
      "qnli": {
        "n": 80,
        "nll": 0.4122149692919013,
        "acc": 0.85,
        "ece": 0.09433166058006281,
        "brier": 0.24752108114076724,
        "mean_conf": 0.8990905626919042,
        "confident_error_rate": 0.0375,
        "coverage_at_0_9": 0.6625,
        "accuracy_at_0_9": 0.9433962264150944,
        "coverage_at_5pct_error": 0.5125,
        "coverage_at_1pct_error": 0.3625,
        "confidence_bias": 0.04909056269190426,
        "top_bins": {
          "0.9": {
            "n": 53,
            "errors": 3,
            "error_rate": 0.05660377358490566
          },
          "0.95": {
            "n": 43,
            "errors": 3,
            "error_rate": 0.06976744186046512
          },
          "0.99": {
            "n": 29,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.95,
            "confidence_cutoff": 0.9583688735354272
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.890625,
            "confidence_cutoff": 0.793002498045601
          }
        }
      },
      "sciq": {
        "n": 80,
        "nll": 0.24132037549325877,
        "acc": 0.9125,
        "ece": 0.04025150891820237,
        "brier": 0.12820506248919616,
        "mean_conf": 0.9387513504714097,
        "confident_error_rate": 0.025,
        "coverage_at_0_9": 0.8375,
        "accuracy_at_0_9": 0.9701492537313433,
        "coverage_at_5pct_error": 0.925,
        "coverage_at_1pct_error": 0.7,
        "confidence_bias": 0.026251350471409696,
        "top_bins": {
          "0.9": {
            "n": 67,
            "errors": 2,
            "error_rate": 0.029850746268656716
          },
          "0.95": {
            "n": 63,
            "errors": 1,
            "error_rate": 0.015873015873015872
          },
          "0.99": {
            "n": 58,
            "errors": 1,
            "error_rate": 0.017241379310344827
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9998130113129569
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.96875,
            "confidence_cutoff": 0.9417403771936594
          }
        }
      },
      "tweet_offensive": {
        "n": 80,
        "nll": 0.8975003034444349,
        "acc": 0.675,
        "ece": 0.18809786512896348,
        "brier": 0.5095841174172036,
        "mean_conf": 0.8555092051551096,
        "confident_error_rate": 0.15,
        "coverage_at_0_9": 0.475,
        "accuracy_at_0_9": 0.6842105263157895,
        "coverage_at_5pct_error": 0.0375,
        "coverage_at_1pct_error": 0.0375,
        "confidence_bias": 0.1805092051551096,
        "top_bins": {
          "0.9": {
            "n": 38,
            "errors": 12,
            "error_rate": 0.3157894736842105
          },
          "0.95": {
            "n": 28,
            "errors": 8,
            "error_rate": 0.2857142857142857
          },
          "0.99": {
            "n": 17,
            "errors": 2,
            "error_rate": 0.11764705882352941
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.675,
            "confidence_cutoff": 0.8908563720881537
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.703125,
            "confidence_cutoff": 0.7286377114869528
          }
        }
      }
    },
    "permutation": {
      "n": 36,
      "mean_max_delta": 0.0699928005819115,
      "flip_rate": 0.08333333333333333
    },
    "temperature": 2.6390158215457884,
    "calibrated_clean": {
      "n": 656,
      "nll": 0.71295343189403,
      "acc": 0.6524390243902439,
      "ece": 0.04090334952670928,
      "brier": 0.42880721405783623,
      "mean_conf": 0.672610937208296,
      "confident_error_rate": 0.003048780487804878,
      "coverage_at_0_9": 0.15548780487804878,
      "accuracy_at_0_9": 0.9803921568627451,
      "coverage_at_5pct_error": 0.22865853658536586,
      "coverage_at_1pct_error": 0.14939024390243902,
      "confidence_bias": 0.020171912818052085,
      "top_bins": {
        "0.9": {
          "n": 102,
          "errors": 2,
          "error_rate": 0.0196078431372549
        },
        "0.95": {
          "n": 52,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 8,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.8079268292682927,
          "confidence_cutoff": 0.6812517987895854
        },
        "0.8": {
          "coverage": 0.8003048780487805,
          "accuracy": 0.7257142857142858,
          "confidence_cutoff": 0.5086893876402309
        }
      },
      "score_mae": 0.69030504615076,
      "ranked_probability_score": 0.21277812800381835
    },
    "metric_policy": {
      "nll_floor": 1e-09,
      "renormalize_returned_probabilities": true,
      "raw_sums_outside_1e_5": 0,
      "returned_zeros": 0
    },
    "coverage": {
      "requested_records": 764,
      "requested_questions": 764,
      "evaluated_records": 764,
      "evaluated_questions": 764,
      "rejected_records": 0,
      "truncated_records": 0
    },
    "latency_ms": {
      "median": 40.60463100006473,
      "p95": 43.923992149950664
    },
    "suite_sha256": "31677c2256b406222e7d94ffdc0a02a70ce05746b9efe307876024c4e77291d1"
  },
  "provenance": {
    "config": {
      "epochs": 1,
      "seed": 1,
      "lr": 4e-05,
      "lora": 16,
      "accum": 1,
      "batch": 8,
      "perm_kl": 0.0,
      "perm_frac": 0.3,
      "ord_w": 0.0,
      "p_none": 0.1,
      "p_none_distract": 0.12,
      "p_distract": 0.15,
      "p_none_pair": 0.25,
      "synthetic_repeat": 1,
      "public_frac": 1.0,
      "head_lr": 0.0,
      "weight_decay": 0.01,
      "anchor_w": 0.0,
      "dtype": "bf16",
      "replay": 2000,
      "base": "Qwen/Qwen3.5-0.8B-Base",
      "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68",
      "init_from": "jaredpalmer/kev-0.8b",
      "data": "evals/night2/dates_unknowable.jsonl"
    },
    "config_sha256": "0e130e9f338c8a4c911c6dc37493b36e6be7ca972cbafbb42466043b6daa7a4b",
    "suite_sha256": "a8f50e481b7d90b97da049e0ff6a01cee2f1ed204aed61a8265af0edbb5514d2",
    "source_hashes": {
      "kev/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
      "kev/anchors.py": "089d8a5493502bb26f540eb1c5e681780ca0bb01276073d4e1733211f15d0e10",
      "kev/api.py": "c9eebbdb6c625563d33f030299ce0fbe8b50493dd1e92592df9514c30f5720af",
      "kev/autoresearch.py": "0a8aa6b57c1c9ba93d25b2cf631b03aed2e686e374c1148002eec48765b7b1fc",
      "kev/benchmark.py": "4192ec3b26b065452f84bde38a091e6854a28fe185d2e0f39a0c91b2efe69df7",
      "kev/compare.py": "bd0445f021de59e35c7bff9304e39dd2e1211e3877a453575594ae7b81b0ada4",
      "kev/composition.py": "f335ed17e18e0a544893db5e22b9a059e6ce1b2e14dbb863ac7d7bbf8f3e0536",
      "kev/contrastive.py": "cbb979aa5d40265ad0e64695f94b281d91751fa405811ddfa8212fede111edcf",
      "kev/data.py": "997c31d737c2a130ade49edd6534aa47d910786c98af883745b1a97ec858a704",
      "kev/evaluate.py": "1b20e3f9edf417aa8dae924b1526e52f74b710cadf7213c5ec68334f6e7f8fe1",
      "kev/experiment.py": "ba034856f79bd500d8b2eeeb7382e4db3fccb30b437f9f3cf55cf1a7c565a6c9",
      "kev/jev.py": "e0213782359ba2f95adbf045ddaf0a008b08b4162bf6a9b66d4ce55fc91a51cb",
      "kev/model.py": "a17d57a52da3fe6ebef7146ce548bd3cedc09e21cc2b7738d9077771f5f16989",
      "kev/plot.py": "d689c7dd18cde7f9ea77cb4f55c50ff7da1880b21cb2ecf244348e45842a1382",
      "kev/publish.py": "8f23cda767008756e0f169249eb44577db76bd6c1e12e9cf64c302bad04c8f16",
      "kev/serve.py": "d095bbee3a210fa8807d1a7b073b56181c93aa69b00f2e243a323ab0a19fa8c9",
      "kev/study_v3.py": "9fc44d44dad09a4f1ee29a7bcb2eb3c7aa373d09186d0e9533666b69ec95c401",
      "kev/suite.py": "44858f9a99df086a47d1ae36141631fab6fb6a8293a9e1d0c6ad397ddcfaf94a",
      "kev/train.py": "c2a3770072b73dd4c0769c8f188ddedbb402f2f8195df6e10b11e5112cab734b",
      "kev/transfer_v9.py": "588aa2ff3ec0823c2e31733bef9a3748b263849c966ef6a80ebd686539c605e9",
      "modal_app.py": "d3700b2914be5ef6baa6d588a967d3a242903e1ed96d0f5cc525a8908aa58340",
      "pyproject.toml": "52da5eea3efc6f2b1c0589acebad62e56a214294bb02c1a4218c93efd4af3182",
      "uv.lock": "18b3e5ea0f25d2e8546fab81f16cb965ae05c3289fffaaa1ce27d114adee47f3"
    },
    "git_commit": "19dcae9b6e3e1a48200c5825aad9fc200d31e20a",
    "platform": "Linux-4.19.0-gvisor-x86_64-with-glibc2.36",
    "torch": "2.8.0+cu128",
    "device": "cuda",
    "gpu": "NVIDIA H100 80GB HBM3",
    "legacy_checkpoint": false
  },
  "mechanism_checks": {
    "n": 8,
    "packed_max_delta": 0.0,
    "sibling_max_delta": 0.0008976459503173828,
    "tolerance": 0.001,
    "passed": true
  },
  "gates": {
    "passed": false,
    "checks": {
      "complete_coverage": true,
      "isolation_and_packing": true,
      "transfer_complete": true,
      "heldout_pairs_at_least_70pct": false,
      "transfer_confident_errors_below_10pct": true
    },
    "policy": "Research screening only: 1e-3 isolation; 5pp task regression; 70% heldout pair correctness; <=10% confident errors; no automatic release."
  },
  "wall_seconds": 546.105594389,
  "promotable": false,
  "test_evaluated": false,
  "training_resources": {
    "wall_seconds": 313.94338822364807,
    "records_seen": 3937,
    "requested_records": 3425,
    "truncated_records": 0,
    "rejected_records": 0,
    "optimizer_steps": 429,
    "forward_tokens": 640184,
    "peak_device_bytes": 43896273920,
    "device": "cuda",
    "dtype": "bf16",
    "batch": 8,
    "peak_rss_bytes": 10316816384
  }
}