diff --git "a/metrics/benchmark.json" "b/metrics/benchmark.json"
--- "a/metrics/benchmark.json"
+++ "b/metrics/benchmark.json"
@@ -8,7 +8,7 @@
     "transfer_v9_test": 0.15
   },
   "protocol_sha256": "ab97f652ab890eaefdd5373dd2deeb1e26588815f1f339eee4bcaa610846b0af",
-  "statistics_sha256": "c73006e701434d215264f2d623160f213cebf2a8ff9b4f8c0d3402b3b99436d3",
+  "statistics_sha256": "8406aea215dc1c4ee2645130c94472b336bb065ba46c6efac4f742589499463b",
   "source_per_model": {
     "Jev": {
       "statistics_sha256": "b319a993ac569ea1fa8da027ccb2195868557e4efe232a5d06091db2884086be",
@@ -119,9 +119,9 @@
       }
     },
     "Lux": {
-      "statistics_sha256": "49c1195fcd88860620fa7d5eb48bb77d6bc759cbac6db89ab11a6faf485644fa",
-      "source_model_key": "Lux-semantic-format-candidate",
-      "manifest_sha256": "485e1f7505d390fb6e84c34edca5812c6803f4d7a48b3f752f7784decdf58559",
+      "statistics_sha256": "71251a31ed6435d8c94df6d2a5241e9183613c19a87f8970145afec8ec948efc",
+      "source_model_key": "Lux-reading-source-candidate",
+      "manifest_sha256": "7107a2a8264a0416fa75fe40e57f3d06e4cb0395527eaedfa7b7335d8e37109a",
       "qualified_runtime": {
         "python": "3.12.13",
         "numpy": "2.3.5"
@@ -5296,59 +5296,59 @@
     "Lux": {
       "panels": {
         "old_core": {
-          "point": 0.8417410714285715,
-          "exact_fraction": "3771/4480",
+          "point": 0.8438244047619048,
+          "exact_fraction": "11341/13440",
           "ci95": [
-            0.8145452008928572,
-            0.8688244047619047
+            0.8172237723214285,
+            0.8705357142857142
           ]
         },
         "v3_core": {
           "point": 0.5275,
           "exact_fraction": "211/400",
           "ci95": [
-            0.49416666666666675,
-            0.5608333333333333
+            0.49375,
+            0.56125
           ]
         },
         "v4": {
-          "point": 0.896875,
-          "exact_fraction": "287/320",
+          "point": 0.9015625,
+          "exact_fraction": "577/640",
           "ci95": [
-            0.8629459433229814,
-            0.9277075669696847
+            0.8683564268867926,
+            0.9324785297746995
           ]
         },
         "v5": {
-          "point": 0.9125,
-          "exact_fraction": "73/80",
+          "point": 0.9145833333333333,
+          "exact_fraction": "439/480",
           "ci95": [
-            0.8866503163407948,
-            0.9366505343021726
+            0.8889180415782575,
+            0.9378879473136665
           ]
         },
         "transfer_v9_test": {
-          "point": 0.7762906309751434,
-          "exact_fraction": "406/523",
+          "point": 0.777246653919694,
+          "exact_fraction": "813/1046",
           "ci95": [
-            0.7482989865877792,
-            0.8032481563248626
+            0.7490309134919917,
+            0.8043278866211298
           ]
         },
         "original_four_panel_mean": {
-          "point": 0.7946540178571428,
-          "exact_fraction": "71201/89600",
+          "point": 0.7968675595238095,
+          "exact_fraction": "107099/134400",
           "ci95": [
-            0.7794567024393497,
-            0.8092736519074929
+            0.7818603628416357,
+            0.8112560296673489
           ]
         },
         "weighted_mean": {
-          "point": 0.7722471660748429,
-          "exact_fraction": "904703/1171520",
+          "point": 0.7740311945165256,
+          "exact_fraction": "36271721/46860800",
           "ci95": [
-            0.7583454502909792,
-            0.785859954609448
+            0.7601225402834362,
+            0.7877175850518867
           ]
         }
       },
@@ -5383,15 +5383,15 @@
               "requested": 64
             },
             "relational_composition": {
-              "correct": 57,
+              "correct": 58,
               "requested": 96
             },
             "scoped_evidence": {
-              "correct": 91,
+              "correct": 93,
               "requested": 96
             },
             "state_tracking": {
-              "correct": 28,
+              "correct": 27,
               "requested": 96
             },
             "unknown_rejection": {
@@ -5410,7 +5410,7 @@
               "requested": 80
             },
             "capacitated_assignment": {
-              "correct": 58,
+              "correct": 59,
               "requested": 80
             },
             "constraint_assignment": {
@@ -5430,7 +5430,7 @@
               "requested": 80
             },
             "ordinal_service_loss": {
-              "correct": 22,
+              "correct": 21,
               "requested": 80
             },
             "paraconsistent_rule_closure": {
@@ -5438,11 +5438,11 @@
               "requested": 80
             },
             "temporal_exclusion": {
-              "correct": 31,
+              "correct": 30,
               "requested": 80
             },
             "transaction_recovery": {
-              "correct": 42,
+              "correct": 43,
               "requested": 80
             }
           }
@@ -5453,7 +5453,7 @@
           "valid_probability_rows": 480,
           "families": {
             "boolq": {
-              "correct": 142,
+              "correct": 143,
               "requested": 160
             },
             "belebele_en": {
@@ -5461,7 +5461,7 @@
               "requested": 160
             },
             "belebele_zh": {
-              "correct": 142,
+              "correct": 143,
               "requested": 160
             }
           }
@@ -5484,7 +5484,7 @@
               "requested": 120
             },
             "qasc": {
-              "correct": 116,
+              "correct": 117,
               "requested": 120
             }
           }
@@ -5494,101 +5494,101 @@
         "clean": {
           "requested": 1046,
           "valid_probability_rows": 1046,
-          "correct": 812,
-          "accuracy": 0.7762906309751434,
+          "correct": 813,
+          "accuracy": 0.777246653919694,
           "full_probability_metrics": {
             "n": 1046,
-            "nll": 0.5940603022446113,
-            "acc": 0.7762906309751434,
-            "ece": 0.05558262070294892,
-            "brier": 0.2937157813621403,
-            "mean_conf": 0.8318732516780923,
-            "confident_error_rate": 0.02390057361376673,
-            "coverage_at_0_9": 0.5860420650095602,
-            "accuracy_at_0_9": 0.9592169657422512,
-            "coverage_at_5pct_error": 0.6347992351816444,
-            "coverage_at_1pct_error": 0.3030592734225621,
-            "aurc": 0.05816033320114623,
-            "error_rate_at_0_9": 0.040783034257748776,
-            "confidence_bias": 0.05558262070294895,
+            "nll": 0.5870477618581573,
+            "acc": 0.777246653919694,
+            "ece": 0.04794282897297868,
+            "brier": 0.2910675481135292,
+            "mean_conf": 0.8251894828926727,
+            "confident_error_rate": 0.02198852772466539,
+            "coverage_at_0_9": 0.5717017208413002,
+            "accuracy_at_0_9": 0.9615384615384616,
+            "coverage_at_5pct_error": 0.6338432122370937,
+            "coverage_at_1pct_error": 0.29923518164435947,
+            "aurc": 0.057865847250986556,
+            "error_rate_at_0_9": 0.038461538461538464,
+            "confidence_bias": 0.047942828972978635,
             "top_bins": {
               "0.9": {
-                "n": 613,
-                "errors": 25,
-                "error_rate": 0.040783034257748776
+                "n": 598,
+                "errors": 23,
+                "error_rate": 0.038461538461538464
               },
               "0.95": {
-                "n": 514,
-                "errors": 12,
-                "error_rate": 0.023346303501945526
+                "n": 491,
+                "errors": 9,
+                "error_rate": 0.018329938900203666
               },
               "0.99": {
-                "n": 207,
+                "n": 174,
                 "errors": 3,
-                "error_rate": 0.014492753623188406
+                "error_rate": 0.017241379310344827
               }
             },
             "selective": {
               "0.5": {
-                "coverage": 0.5009560229445507,
-                "accuracy": 0.9770992366412213,
-                "confidence_cutoff": 0.947606086730957
+                "coverage": 0.5,
+                "accuracy": 0.9751434034416826,
+                "confidence_cutoff": 0.9408829808235168
               },
               "0.8": {
                 "coverage": 0.8001912045889101,
-                "accuracy": 0.8781362007168458,
-                "confidence_cutoff": 0.647434909565297
+                "accuracy": 0.8793309438470729,
+                "confidence_cutoff": 0.6510422047867755
               }
             },
-            "score_mae": 0.18149603870298436,
-            "ranked_probability_score": 0.051063424067515496
+            "score_mae": 0.18602619774583048,
+            "ranked_probability_score": 0.050882127676710784
           },
           "covered_only_probability_metrics": {
             "n": 1046,
-            "nll": 0.5940603022446113,
-            "acc": 0.7762906309751434,
-            "ece": 0.05558262070294892,
-            "brier": 0.2937157813621403,
-            "mean_conf": 0.8318732516780923,
-            "confident_error_rate": 0.02390057361376673,
-            "coverage_at_0_9": 0.5860420650095602,
-            "accuracy_at_0_9": 0.9592169657422512,
-            "coverage_at_5pct_error": 0.6347992351816444,
-            "coverage_at_1pct_error": 0.3030592734225621,
-            "aurc": 0.05816033320114623,
-            "error_rate_at_0_9": 0.040783034257748776,
-            "confidence_bias": 0.05558262070294895,
+            "nll": 0.5870477618581573,
+            "acc": 0.777246653919694,
+            "ece": 0.04794282897297868,
+            "brier": 0.2910675481135292,
+            "mean_conf": 0.8251894828926727,
+            "confident_error_rate": 0.02198852772466539,
+            "coverage_at_0_9": 0.5717017208413002,
+            "accuracy_at_0_9": 0.9615384615384616,
+            "coverage_at_5pct_error": 0.6338432122370937,
+            "coverage_at_1pct_error": 0.29923518164435947,
+            "aurc": 0.057865847250986556,
+            "error_rate_at_0_9": 0.038461538461538464,
+            "confidence_bias": 0.047942828972978635,
             "top_bins": {
               "0.9": {
-                "n": 613,
-                "errors": 25,
-                "error_rate": 0.040783034257748776
+                "n": 598,
+                "errors": 23,
+                "error_rate": 0.038461538461538464
               },
               "0.95": {
-                "n": 514,
-                "errors": 12,
-                "error_rate": 0.023346303501945526
+                "n": 491,
+                "errors": 9,
+                "error_rate": 0.018329938900203666
               },
               "0.99": {
-                "n": 207,
+                "n": 174,
                 "errors": 3,
-                "error_rate": 0.014492753623188406
+                "error_rate": 0.017241379310344827
               }
             },
             "selective": {
               "0.5": {
-                "coverage": 0.5009560229445507,
-                "accuracy": 0.9770992366412213,
-                "confidence_cutoff": 0.947606086730957
+                "coverage": 0.5,
+                "accuracy": 0.9751434034416826,
+                "confidence_cutoff": 0.9408829808235168
               },
               "0.8": {
                 "coverage": 0.8001912045889101,
-                "accuracy": 0.8781362007168458,
-                "confidence_cutoff": 0.647434909565297
+                "accuracy": 0.8793309438470729,
+                "confidence_cutoff": 0.6510422047867755
               }
             },
-            "score_mae": 0.18149603870298436,
-            "ranked_probability_score": 0.051063424067515496
+            "score_mae": 0.18602619774583048,
+            "ranked_probability_score": 0.050882127676710784
           },
           "coverage": 1.0
         },
@@ -5600,11 +5600,11 @@
             "accuracy": 0.45,
             "full_probability_metrics": {
               "n": 20,
-              "nll": 1.4899537928777824,
+              "nll": 1.4638562023564279,
               "acc": 0.45,
-              "ece": 0.3017003229117818,
-              "brier": 0.6913230901539468,
-              "mean_conf": 0.6857035964205832,
+              "ece": 0.22400769818984456,
+              "brier": 0.685438069283081,
+              "mean_conf": 0.6594526435269211,
               "confident_error_rate": 0.05,
               "coverage_at_0_9": 0.2,
               "accuracy_at_0_9": 0.75,
@@ -5612,7 +5612,7 @@
               "coverage_at_1pct_error": 0.05,
               "aurc": 0.3734132508358514,
               "error_rate_at_0_9": 0.25,
-              "confidence_bias": 0.2357035964205832,
+              "confidence_bias": 0.2094526435269211,
               "top_bins": {
                 "0.9": {
                   "n": 4,
@@ -5634,22 +5634,22 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.6,
-                  "confidence_cutoff": 0.720178043538323
+                  "confidence_cutoff": 0.6973127263072573
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.49892304084621675
+                  "confidence_cutoff": 0.483508165065625
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 20,
-              "nll": 1.4899537928777824,
+              "nll": 1.4638562023564279,
               "acc": 0.45,
-              "ece": 0.3017003229117818,
-              "brier": 0.6913230901539468,
-              "mean_conf": 0.6857035964205832,
+              "ece": 0.22400769818984456,
+              "brier": 0.685438069283081,
+              "mean_conf": 0.6594526435269211,
               "confident_error_rate": 0.05,
               "coverage_at_0_9": 0.2,
               "accuracy_at_0_9": 0.75,
@@ -5657,7 +5657,7 @@
               "coverage_at_1pct_error": 0.05,
               "aurc": 0.3734132508358514,
               "error_rate_at_0_9": 0.25,
-              "confidence_bias": 0.2357035964205832,
+              "confidence_bias": 0.2094526435269211,
               "top_bins": {
                 "0.9": {
                   "n": 4,
@@ -5679,12 +5679,12 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.6,
-                  "confidence_cutoff": 0.720178043538323
+                  "confidence_cutoff": 0.6973127263072573
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.49892304084621675
+                  "confidence_cutoff": 0.483508165065625
                 }
               }
             },
@@ -5697,24 +5697,24 @@
             "accuracy": 0.8,
             "full_probability_metrics": {
               "n": 20,
-              "nll": 0.391154446854581,
+              "nll": 0.36403248618468526,
               "acc": 0.8,
-              "ece": 0.0909007219853577,
-              "brier": 0.24269024392468377,
-              "mean_conf": 0.8773054444371697,
+              "ece": 0.1295361487173242,
+              "brier": 0.2257044216488208,
+              "mean_conf": 0.8831790381288052,
               "confident_error_rate": 0.05,
-              "coverage_at_0_9": 0.7,
-              "accuracy_at_0_9": 0.9285714285714286,
+              "coverage_at_0_9": 0.65,
+              "accuracy_at_0_9": 0.9230769230769231,
               "coverage_at_5pct_error": 0.45,
               "coverage_at_1pct_error": 0.45,
-              "aurc": 0.06282346007965203,
-              "error_rate_at_0_9": 0.07142857142857142,
-              "confidence_bias": 0.07730544443716969,
+              "aurc": 0.05397950583128602,
+              "error_rate_at_0_9": 0.07692307692307693,
+              "confidence_bias": 0.08317903812880512,
               "top_bins": {
                 "0.9": {
-                  "n": 14,
+                  "n": 13,
                   "errors": 1,
-                  "error_rate": 0.07142857142857142
+                  "error_rate": 0.07692307692307693
                 },
                 "0.95": {
                   "n": 12,
@@ -5731,35 +5731,35 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.9,
-                  "confidence_cutoff": 0.9649345170900093
+                  "confidence_cutoff": 0.9546051514751633
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.875,
-                  "confidence_cutoff": 0.7085450713959379
+                  "accuracy": 0.9375,
+                  "confidence_cutoff": 0.7867595084452922
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 20,
-              "nll": 0.391154446854581,
+              "nll": 0.36403248618468526,
               "acc": 0.8,
-              "ece": 0.0909007219853577,
-              "brier": 0.24269024392468377,
-              "mean_conf": 0.8773054444371697,
+              "ece": 0.1295361487173242,
+              "brier": 0.2257044216488208,
+              "mean_conf": 0.8831790381288052,
               "confident_error_rate": 0.05,
-              "coverage_at_0_9": 0.7,
-              "accuracy_at_0_9": 0.9285714285714286,
+              "coverage_at_0_9": 0.65,
+              "accuracy_at_0_9": 0.9230769230769231,
               "coverage_at_5pct_error": 0.45,
               "coverage_at_1pct_error": 0.45,
-              "aurc": 0.06282346007965203,
-              "error_rate_at_0_9": 0.07142857142857142,
-              "confidence_bias": 0.07730544443716969,
+              "aurc": 0.05397950583128602,
+              "error_rate_at_0_9": 0.07692307692307693,
+              "confidence_bias": 0.08317903812880512,
               "top_bins": {
                 "0.9": {
-                  "n": 14,
+                  "n": 13,
                   "errors": 1,
-                  "error_rate": 0.07142857142857142
+                  "error_rate": 0.07692307692307693
                 },
                 "0.95": {
                   "n": 12,
@@ -5776,12 +5776,12 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.9,
-                  "confidence_cutoff": 0.9649345170900093
+                  "confidence_cutoff": 0.9546051514751633
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.875,
-                  "confidence_cutoff": 0.7085450713959379
+                  "accuracy": 0.9375,
+                  "confidence_cutoff": 0.7867595084452922
                 }
               }
             },
@@ -5794,32 +5794,32 @@
             "accuracy": 0.9,
             "full_probability_metrics": {
               "n": 20,
-              "nll": 0.173920037838875,
+              "nll": 0.18031139864262885,
               "acc": 0.9,
-              "ece": 0.08884237104933625,
-              "brier": 0.11334577453222641,
-              "mean_conf": 0.9453143823740051,
+              "ece": 0.0817012190955263,
+              "brier": 0.11604628935827062,
+              "mean_conf": 0.9409710011124263,
               "confident_error_rate": 0.0,
-              "coverage_at_0_9": 0.85,
+              "coverage_at_0_9": 0.8,
               "accuracy_at_0_9": 1.0,
               "coverage_at_5pct_error": 0.9,
               "coverage_at_1pct_error": 0.9,
               "aurc": 0.0076315789473684215,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": 0.04531438237400509,
+              "confidence_bias": 0.04097100111242624,
               "top_bins": {
                 "0.9": {
-                  "n": 17,
+                  "n": 16,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.95": {
-                  "n": 16,
+                  "n": 15,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 2,
+                  "n": 1,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -5828,43 +5828,43 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9793051012887353
+                  "confidence_cutoff": 0.9736818072730682
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9588658106453812
+                  "confidence_cutoff": 0.9485171543268541
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 20,
-              "nll": 0.173920037838875,
+              "nll": 0.18031139864262885,
               "acc": 0.9,
-              "ece": 0.08884237104933625,
-              "brier": 0.11334577453222641,
-              "mean_conf": 0.9453143823740051,
+              "ece": 0.0817012190955263,
+              "brier": 0.11604628935827062,
+              "mean_conf": 0.9409710011124263,
               "confident_error_rate": 0.0,
-              "coverage_at_0_9": 0.85,
+              "coverage_at_0_9": 0.8,
               "accuracy_at_0_9": 1.0,
               "coverage_at_5pct_error": 0.9,
               "coverage_at_1pct_error": 0.9,
               "aurc": 0.0076315789473684215,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": 0.04531438237400509,
+              "confidence_bias": 0.04097100111242624,
               "top_bins": {
                 "0.9": {
-                  "n": 17,
+                  "n": 16,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.95": {
-                  "n": 16,
+                  "n": 15,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 2,
+                  "n": 1,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -5873,12 +5873,12 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9793051012887353
+                  "confidence_cutoff": 0.9736818072730682
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9588658106453812
+                  "confidence_cutoff": 0.9485171543268541
                 }
               }
             },
@@ -5891,19 +5891,19 @@
             "accuracy": 0.4,
             "full_probability_metrics": {
               "n": 20,
-              "nll": 0.963421060535776,
+              "nll": 0.9361317272289519,
               "acc": 0.4,
-              "ece": 0.3598356256038071,
-              "brier": 0.7060594827987038,
-              "mean_conf": 0.7594032993432073,
+              "ece": 0.3467800262809543,
+              "brier": 0.6890225027879284,
+              "mean_conf": 0.7467800262809542,
               "confident_error_rate": 0.05,
               "coverage_at_0_9": 0.1,
               "accuracy_at_0_9": 0.5,
               "coverage_at_5pct_error": 0.05,
               "coverage_at_1pct_error": 0.05,
-              "aurc": 0.45470466259746445,
+              "aurc": 0.46809751974032154,
               "error_rate_at_0_9": 0.5,
-              "confidence_bias": 0.35940329934320725,
+              "confidence_bias": 0.34678002628095417,
               "top_bins": {
                 "0.9": {
                   "n": 2,
@@ -5911,9 +5911,9 @@
                   "error_rate": 0.5
                 },
                 "0.95": {
-                  "n": 2,
-                  "errors": 1,
-                  "error_rate": 0.5
+                  "n": 1,
+                  "errors": 0,
+                  "error_rate": 0.0
                 },
                 "0.99": {
                   "n": 0,
@@ -5925,30 +5925,30 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.7584096306325441
+                  "confidence_cutoff": 0.7481542379764177
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.4375,
-                  "confidence_cutoff": 0.6582031053840188
+                  "confidence_cutoff": 0.6360844184135074
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 20,
-              "nll": 0.963421060535776,
+              "nll": 0.9361317272289519,
               "acc": 0.4,
-              "ece": 0.3598356256038071,
-              "brier": 0.7060594827987038,
-              "mean_conf": 0.7594032993432073,
+              "ece": 0.3467800262809543,
+              "brier": 0.6890225027879284,
+              "mean_conf": 0.7467800262809542,
               "confident_error_rate": 0.05,
               "coverage_at_0_9": 0.1,
               "accuracy_at_0_9": 0.5,
               "coverage_at_5pct_error": 0.05,
               "coverage_at_1pct_error": 0.05,
-              "aurc": 0.45470466259746445,
+              "aurc": 0.46809751974032154,
               "error_rate_at_0_9": 0.5,
-              "confidence_bias": 0.35940329934320725,
+              "confidence_bias": 0.34678002628095417,
               "top_bins": {
                 "0.9": {
                   "n": 2,
@@ -5956,9 +5956,9 @@
                   "error_rate": 0.5
                 },
                 "0.95": {
-                  "n": 2,
-                  "errors": 1,
-                  "error_rate": 0.5
+                  "n": 1,
+                  "errors": 0,
+                  "error_rate": 0.0
                 },
                 "0.99": {
                   "n": 0,
@@ -5970,12 +5970,12 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.7584096306325441
+                  "confidence_cutoff": 0.7481542379764177
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.4375,
-                  "confidence_cutoff": 0.6582031053840188
+                  "confidence_cutoff": 0.6360844184135074
                 }
               }
             },
@@ -5988,19 +5988,19 @@
             "accuracy": 0.96875,
             "full_probability_metrics": {
               "n": 32,
-              "nll": 0.13034778836964267,
+              "nll": 0.12979322912048136,
               "acc": 0.96875,
-              "ece": 0.07387770926294471,
-              "brier": 0.06436468242012063,
-              "mean_conf": 0.9252340474017305,
+              "ece": 0.05865632527071586,
+              "brier": 0.06221598573997203,
+              "mean_conf": 0.9181713626206109,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.75,
               "accuracy_at_0_9": 1.0,
               "coverage_at_5pct_error": 1.0,
-              "coverage_at_1pct_error": 0.84375,
-              "aurc": 0.005186349167726045,
+              "coverage_at_1pct_error": 0.875,
+              "aurc": 0.004070277739154617,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.04351595259826946,
+              "confidence_bias": -0.05057863737938906,
               "top_bins": {
                 "0.9": {
                   "n": 24,
@@ -6008,7 +6008,7 @@
                   "error_rate": 0.0
                 },
                 "0.95": {
-                  "n": 17,
+                  "n": 11,
                   "errors": 0,
                   "error_rate": 0.0
                 },
@@ -6020,32 +6020,32 @@
               },
               "selective": {
                 "0.5": {
-                  "coverage": 0.53125,
+                  "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9517537497245334
+                  "confidence_cutoff": 0.9449616248177756
                 },
                 "0.8": {
                   "coverage": 0.8125,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.8932077222078463
+                  "confidence_cutoff": 0.8779250187145542
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 32,
-              "nll": 0.13034778836964267,
+              "nll": 0.12979322912048136,
               "acc": 0.96875,
-              "ece": 0.07387770926294471,
-              "brier": 0.06436468242012063,
-              "mean_conf": 0.9252340474017305,
+              "ece": 0.05865632527071586,
+              "brier": 0.06221598573997203,
+              "mean_conf": 0.9181713626206109,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.75,
               "accuracy_at_0_9": 1.0,
               "coverage_at_5pct_error": 1.0,
-              "coverage_at_1pct_error": 0.84375,
-              "aurc": 0.005186349167726045,
+              "coverage_at_1pct_error": 0.875,
+              "aurc": 0.004070277739154617,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.04351595259826946,
+              "confidence_bias": -0.05057863737938906,
               "top_bins": {
                 "0.9": {
                   "n": 24,
@@ -6053,7 +6053,7 @@
                   "error_rate": 0.0
                 },
                 "0.95": {
-                  "n": 17,
+                  "n": 11,
                   "errors": 0,
                   "error_rate": 0.0
                 },
@@ -6065,14 +6065,14 @@
               },
               "selective": {
                 "0.5": {
-                  "coverage": 0.53125,
+                  "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9517537497245334
+                  "confidence_cutoff": 0.9449616248177756
                 },
                 "0.8": {
                   "coverage": 0.8125,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.8932077222078463
+                  "confidence_cutoff": 0.8779250187145542
                 }
               }
             },
@@ -6085,19 +6085,19 @@
             "accuracy": 0.90625,
             "full_probability_metrics": {
               "n": 32,
-              "nll": 0.24646259109043608,
+              "nll": 0.24455502641087398,
               "acc": 0.90625,
-              "ece": 0.08165686202673392,
-              "brier": 0.13219691386769145,
-              "mean_conf": 0.9344170796872766,
+              "ece": 0.079792014916701,
+              "brier": 0.13577654408819648,
+              "mean_conf": 0.9357865299742769,
               "confident_error_rate": 0.03125,
               "coverage_at_0_9": 0.75,
               "accuracy_at_0_9": 0.9583333333333334,
               "coverage_at_5pct_error": 0.9375,
               "coverage_at_1pct_error": 0.375,
-              "aurc": 0.033345170986070205,
+              "aurc": 0.033363772176546395,
               "error_rate_at_0_9": 0.041666666666666664,
-              "confidence_bias": 0.02816707968727661,
+              "confidence_bias": 0.029536529974276915,
               "top_bins": {
                 "0.9": {
                   "n": 24,
@@ -6110,7 +6110,7 @@
                   "error_rate": 0.05555555555555555
                 },
                 "0.99": {
-                  "n": 6,
+                  "n": 5,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -6119,30 +6119,30 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.9375,
-                  "confidence_cutoff": 0.9769813800539988
+                  "confidence_cutoff": 0.9711615479408262
                 },
                 "0.8": {
                   "coverage": 0.8125,
                   "accuracy": 0.9615384615384616,
-                  "confidence_cutoff": 0.8745144680431802
+                  "confidence_cutoff": 0.8868939148352457
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 32,
-              "nll": 0.24646259109043608,
+              "nll": 0.24455502641087398,
               "acc": 0.90625,
-              "ece": 0.08165686202673392,
-              "brier": 0.13219691386769145,
-              "mean_conf": 0.9344170796872766,
+              "ece": 0.079792014916701,
+              "brier": 0.13577654408819648,
+              "mean_conf": 0.9357865299742769,
               "confident_error_rate": 0.03125,
               "coverage_at_0_9": 0.75,
               "accuracy_at_0_9": 0.9583333333333334,
               "coverage_at_5pct_error": 0.9375,
               "coverage_at_1pct_error": 0.375,
-              "aurc": 0.033345170986070205,
+              "aurc": 0.033363772176546395,
               "error_rate_at_0_9": 0.041666666666666664,
-              "confidence_bias": 0.02816707968727661,
+              "confidence_bias": 0.029536529974276915,
               "top_bins": {
                 "0.9": {
                   "n": 24,
@@ -6155,7 +6155,7 @@
                   "error_rate": 0.05555555555555555
                 },
                 "0.99": {
-                  "n": 6,
+                  "n": 5,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -6164,12 +6164,12 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.9375,
-                  "confidence_cutoff": 0.9769813800539988
+                  "confidence_cutoff": 0.9711615479408262
                 },
                 "0.8": {
                   "coverage": 0.8125,
                   "accuracy": 0.9615384615384616,
-                  "confidence_cutoff": 0.8745144680431802
+                  "confidence_cutoff": 0.8868939148352457
                 }
               }
             },
@@ -6182,11 +6182,11 @@
             "accuracy": 0.96875,
             "full_probability_metrics": {
               "n": 32,
-              "nll": 0.17366660442192522,
+              "nll": 0.1799321903230246,
               "acc": 0.96875,
-              "ece": 0.139280642091471,
-              "brier": 0.09349301051833991,
-              "mean_conf": 0.8684549516140397,
+              "ece": 0.10182879137977313,
+              "brier": 0.09774414276889377,
+              "mean_conf": 0.8675840743934221,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.5625,
               "accuracy_at_0_9": 1.0,
@@ -6194,7 +6194,7 @@
               "coverage_at_1pct_error": 0.875,
               "aurc": 0.004072377873563219,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.10029504838596026,
+              "confidence_bias": -0.10116592560657789,
               "top_bins": {
                 "0.9": {
                   "n": 18,
@@ -6207,7 +6207,7 @@
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 5,
+                  "n": 4,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -6216,22 +6216,22 @@
                 "0.5": {
                   "coverage": 0.53125,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9398574478142043
+                  "confidence_cutoff": 0.9276751556299673
                 },
                 "0.8": {
                   "coverage": 0.8125,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.7353225569523143
+                  "confidence_cutoff": 0.7040075273516038
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 32,
-              "nll": 0.17366660442192522,
+              "nll": 0.1799321903230246,
               "acc": 0.96875,
-              "ece": 0.139280642091471,
-              "brier": 0.09349301051833991,
-              "mean_conf": 0.8684549516140397,
+              "ece": 0.10182879137977313,
+              "brier": 0.09774414276889377,
+              "mean_conf": 0.8675840743934221,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.5625,
               "accuracy_at_0_9": 1.0,
@@ -6239,7 +6239,7 @@
               "coverage_at_1pct_error": 0.875,
               "aurc": 0.004072377873563219,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.10029504838596026,
+              "confidence_bias": -0.10116592560657789,
               "top_bins": {
                 "0.9": {
                   "n": 18,
@@ -6252,7 +6252,7 @@
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 5,
+                  "n": 4,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -6261,12 +6261,12 @@
                 "0.5": {
                   "coverage": 0.53125,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9398574478142043
+                  "confidence_cutoff": 0.9276751556299673
                 },
                 "0.8": {
                   "coverage": 0.8125,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.7353225569523143
+                  "confidence_cutoff": 0.7040075273516038
                 }
               }
             },
@@ -6279,11 +6279,11 @@
             "accuracy": 1.0,
             "full_probability_metrics": {
               "n": 40,
-              "nll": 0.010156044928884202,
+              "nll": 0.012461149245409892,
               "acc": 1.0,
-              "ece": 0.010082175827252016,
-              "brier": 0.0002922128455344832,
-              "mean_conf": 0.989917824172748,
+              "ece": 0.012352988030602807,
+              "brier": 0.0004269628768902182,
+              "mean_conf": 0.9876470119693972,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -6291,7 +6291,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.010082175827252016,
+              "confidence_bias": -0.012352988030602807,
               "top_bins": {
                 "0.9": {
                   "n": 40,
@@ -6313,22 +6313,22 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.994282735900115
+                  "confidence_cutoff": 0.9926770167945137
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9830883529872735
+                  "confidence_cutoff": 0.9794385287249308
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 40,
-              "nll": 0.010156044928884202,
+              "nll": 0.012461149245409892,
               "acc": 1.0,
-              "ece": 0.010082175827252016,
-              "brier": 0.0002922128455344832,
-              "mean_conf": 0.989917824172748,
+              "ece": 0.012352988030602807,
+              "brier": 0.0004269628768902182,
+              "mean_conf": 0.9876470119693972,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -6336,7 +6336,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.010082175827252016,
+              "confidence_bias": -0.012352988030602807,
               "top_bins": {
                 "0.9": {
                   "n": 40,
@@ -6358,12 +6358,12 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.994282735900115
+                  "confidence_cutoff": 0.9926770167945137
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9830883529872735
+                  "confidence_cutoff": 0.9794385287249308
                 }
               }
             },
@@ -6376,32 +6376,32 @@
             "accuracy": 0.9,
             "full_probability_metrics": {
               "n": 40,
-              "nll": 0.2578333370423695,
+              "nll": 0.26407278866821826,
               "acc": 0.9,
-              "ece": 0.08339734125029008,
-              "brier": 0.1539668312012304,
-              "mean_conf": 0.8928125003966263,
+              "ece": 0.05245202539688848,
+              "brier": 0.15778553734614284,
+              "mean_conf": 0.8829211192449822,
               "confident_error_rate": 0.0,
-              "coverage_at_0_9": 0.6,
+              "coverage_at_0_9": 0.575,
               "accuracy_at_0_9": 1.0,
-              "coverage_at_5pct_error": 0.7,
+              "coverage_at_5pct_error": 0.725,
               "coverage_at_1pct_error": 0.65,
-              "aurc": 0.02394744920629697,
+              "aurc": 0.02308538024077973,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.007187499603373726,
+              "confidence_bias": -0.017078880755017845,
               "top_bins": {
                 "0.9": {
-                  "n": 24,
+                  "n": 23,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.95": {
-                  "n": 16,
+                  "n": 15,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 8,
+                  "n": 5,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -6410,45 +6410,45 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9320149317531109
+                  "confidence_cutoff": 0.9200535815570202
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.9375,
-                  "confidence_cutoff": 0.8366438894497547
+                  "confidence_cutoff": 0.8251958593609416
                 }
               },
-              "score_mae": 0.16359197313913096,
-              "ranked_probability_score": 0.03911601810015416
+              "score_mae": 0.17291673608415475,
+              "ranked_probability_score": 0.04024927146404126
             },
             "covered_only_probability_metrics": {
               "n": 40,
-              "nll": 0.2578333370423695,
+              "nll": 0.26407278866821826,
               "acc": 0.9,
-              "ece": 0.08339734125029008,
-              "brier": 0.1539668312012304,
-              "mean_conf": 0.8928125003966263,
+              "ece": 0.05245202539688848,
+              "brier": 0.15778553734614284,
+              "mean_conf": 0.8829211192449822,
               "confident_error_rate": 0.0,
-              "coverage_at_0_9": 0.6,
+              "coverage_at_0_9": 0.575,
               "accuracy_at_0_9": 1.0,
-              "coverage_at_5pct_error": 0.7,
+              "coverage_at_5pct_error": 0.725,
               "coverage_at_1pct_error": 0.65,
-              "aurc": 0.02394744920629697,
+              "aurc": 0.02308538024077973,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.007187499603373726,
+              "confidence_bias": -0.017078880755017845,
               "top_bins": {
                 "0.9": {
-                  "n": 24,
+                  "n": 23,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.95": {
-                  "n": 16,
+                  "n": 15,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 8,
+                  "n": 5,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -6457,111 +6457,111 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9320149317531109
+                  "confidence_cutoff": 0.9200535815570202
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.9375,
-                  "confidence_cutoff": 0.8366438894497547
+                  "confidence_cutoff": 0.8251958593609416
                 }
               },
-              "score_mae": 0.16359197313913096,
-              "ranked_probability_score": 0.03911601810015416
+              "score_mae": 0.17291673608415475,
+              "ranked_probability_score": 0.04024927146404126
             },
             "coverage": 1.0
           },
           "emotion": {
             "requested": 80,
             "valid_probability_rows": 80,
-            "correct": 49,
-            "accuracy": 0.6125,
+            "correct": 48,
+            "accuracy": 0.6,
             "full_probability_metrics": {
               "n": 80,
-              "nll": 1.3250366616414988,
-              "acc": 0.6125,
-              "ece": 0.2033677770446442,
-              "brier": 0.578098657406022,
-              "mean_conf": 0.7837558603474908,
-              "confident_error_rate": 0.075,
-              "coverage_at_0_9": 0.45,
-              "accuracy_at_0_9": 0.8333333333333334,
+              "nll": 1.2865095689254165,
+              "acc": 0.6,
+              "ece": 0.19167859824830136,
+              "brier": 0.5660599536581878,
+              "mean_conf": 0.7759389293425664,
+              "confident_error_rate": 0.0625,
+              "coverage_at_0_9": 0.4,
+              "accuracy_at_0_9": 0.84375,
               "coverage_at_5pct_error": 0.025,
               "coverage_at_1pct_error": 0.025,
-              "aurc": 0.2465828605574525,
-              "error_rate_at_0_9": 0.16666666666666666,
-              "confidence_bias": 0.17125586034749074,
+              "aurc": 0.24689075981896816,
+              "error_rate_at_0_9": 0.15625,
+              "confidence_bias": 0.17593892934256639,
               "top_bins": {
                 "0.9": {
-                  "n": 36,
-                  "errors": 6,
-                  "error_rate": 0.16666666666666666
+                  "n": 32,
+                  "errors": 5,
+                  "error_rate": 0.15625
                 },
                 "0.95": {
-                  "n": 24,
+                  "n": 22,
                   "errors": 3,
-                  "error_rate": 0.125
+                  "error_rate": 0.13636363636363635
                 },
                 "0.99": {
-                  "n": 7,
+                  "n": 5,
                   "errors": 2,
-                  "error_rate": 0.2857142857142857
+                  "error_rate": 0.4
                 }
               },
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.8,
-                  "confidence_cutoff": 0.8534444506954815
+                  "confidence_cutoff": 0.8358595503583666
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.65625,
-                  "confidence_cutoff": 0.5796189437721434
+                  "accuracy": 0.671875,
+                  "confidence_cutoff": 0.5660683528503451
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 80,
-              "nll": 1.3250366616414988,
-              "acc": 0.6125,
-              "ece": 0.2033677770446442,
-              "brier": 0.578098657406022,
-              "mean_conf": 0.7837558603474908,
-              "confident_error_rate": 0.075,
-              "coverage_at_0_9": 0.45,
-              "accuracy_at_0_9": 0.8333333333333334,
+              "nll": 1.2865095689254165,
+              "acc": 0.6,
+              "ece": 0.19167859824830136,
+              "brier": 0.5660599536581878,
+              "mean_conf": 0.7759389293425664,
+              "confident_error_rate": 0.0625,
+              "coverage_at_0_9": 0.4,
+              "accuracy_at_0_9": 0.84375,
               "coverage_at_5pct_error": 0.025,
               "coverage_at_1pct_error": 0.025,
-              "aurc": 0.2465828605574525,
-              "error_rate_at_0_9": 0.16666666666666666,
-              "confidence_bias": 0.17125586034749074,
+              "aurc": 0.24689075981896816,
+              "error_rate_at_0_9": 0.15625,
+              "confidence_bias": 0.17593892934256639,
               "top_bins": {
                 "0.9": {
-                  "n": 36,
-                  "errors": 6,
-                  "error_rate": 0.16666666666666666
+                  "n": 32,
+                  "errors": 5,
+                  "error_rate": 0.15625
                 },
                 "0.95": {
-                  "n": 24,
+                  "n": 22,
                   "errors": 3,
-                  "error_rate": 0.125
+                  "error_rate": 0.13636363636363635
                 },
                 "0.99": {
-                  "n": 7,
+                  "n": 5,
                   "errors": 2,
-                  "error_rate": 0.2857142857142857
+                  "error_rate": 0.4
                 }
               },
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.8,
-                  "confidence_cutoff": 0.8534444506954815
+                  "confidence_cutoff": 0.8358595503583666
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.65625,
-                  "confidence_cutoff": 0.5796189437721434
+                  "accuracy": 0.671875,
+                  "confidence_cutoff": 0.5660683528503451
                 }
               }
             },
@@ -6570,23 +6570,23 @@
           "mmlu": {
             "requested": 80,
             "valid_probability_rows": 80,
-            "correct": 59,
-            "accuracy": 0.7375,
+            "correct": 60,
+            "accuracy": 0.75,
             "full_probability_metrics": {
               "n": 80,
-              "nll": 0.6466641422955599,
-              "acc": 0.7375,
-              "ece": 0.10575227809040644,
-              "brier": 0.3381993704155934,
-              "mean_conf": 0.8239887221865458,
+              "nll": 0.64319962297488,
+              "acc": 0.75,
+              "ece": 0.07583444934027661,
+              "brier": 0.336846797840859,
+              "mean_conf": 0.815404454455512,
               "confident_error_rate": 0.025,
               "coverage_at_0_9": 0.5125,
               "accuracy_at_0_9": 0.9512195121951219,
-              "coverage_at_5pct_error": 0.5875,
-              "coverage_at_1pct_error": 0.4125,
-              "aurc": 0.07075415004878534,
+              "coverage_at_5pct_error": 0.575,
+              "coverage_at_1pct_error": 0.425,
+              "aurc": 0.06990146624747551,
               "error_rate_at_0_9": 0.04878048780487805,
-              "confidence_bias": 0.0864887221865458,
+              "confidence_bias": 0.06540445445551202,
               "top_bins": {
                 "0.9": {
                   "n": 41,
@@ -6594,12 +6594,12 @@
                   "error_rate": 0.04878048780487805
                 },
                 "0.95": {
-                  "n": 31,
+                  "n": 29,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 9,
+                  "n": 6,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -6608,30 +6608,30 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.95,
-                  "confidence_cutoff": 0.9179506124738418
+                  "confidence_cutoff": 0.9107307477881781
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.859375,
-                  "confidence_cutoff": 0.6428546144492293
+                  "confidence_cutoff": 0.631165847975779
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 80,
-              "nll": 0.6466641422955599,
-              "acc": 0.7375,
-              "ece": 0.10575227809040644,
-              "brier": 0.3381993704155934,
-              "mean_conf": 0.8239887221865458,
+              "nll": 0.64319962297488,
+              "acc": 0.75,
+              "ece": 0.07583444934027661,
+              "brier": 0.336846797840859,
+              "mean_conf": 0.815404454455512,
               "confident_error_rate": 0.025,
               "coverage_at_0_9": 0.5125,
               "accuracy_at_0_9": 0.9512195121951219,
-              "coverage_at_5pct_error": 0.5875,
-              "coverage_at_1pct_error": 0.4125,
-              "aurc": 0.07075415004878534,
+              "coverage_at_5pct_error": 0.575,
+              "coverage_at_1pct_error": 0.425,
+              "aurc": 0.06990146624747551,
               "error_rate_at_0_9": 0.04878048780487805,
-              "confidence_bias": 0.0864887221865458,
+              "confidence_bias": 0.06540445445551202,
               "top_bins": {
                 "0.9": {
                   "n": 41,
@@ -6639,12 +6639,12 @@
                   "error_rate": 0.04878048780487805
                 },
                 "0.95": {
-                  "n": 31,
+                  "n": 29,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 9,
+                  "n": 6,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -6653,12 +6653,12 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.95,
-                  "confidence_cutoff": 0.9179506124738418
+                  "confidence_cutoff": 0.9107307477881781
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.859375,
-                  "confidence_cutoff": 0.6428546144492293
+                  "confidence_cutoff": 0.631165847975779
                 }
               }
             },
@@ -6671,32 +6671,32 @@
             "accuracy": 0.53,
             "full_probability_metrics": {
               "n": 200,
-              "nll": 1.3111522675312284,
+              "nll": 1.3020212799561357,
               "acc": 0.53,
-              "ece": 0.08739805659958974,
-              "brier": 0.5728987396461885,
-              "mean_conf": 0.5899912710608897,
+              "ece": 0.057886156442029546,
+              "brier": 0.5699198585015737,
+              "mean_conf": 0.5796763608829093,
               "confident_error_rate": 0.005,
-              "coverage_at_0_9": 0.23,
-              "accuracy_at_0_9": 0.9782608695652174,
-              "coverage_at_5pct_error": 0.25,
-              "coverage_at_1pct_error": 0.19,
-              "aurc": 0.21980746104696547,
-              "error_rate_at_0_9": 0.021739130434782608,
-              "confidence_bias": 0.05999127106088964,
+              "coverage_at_0_9": 0.21,
+              "accuracy_at_0_9": 0.9761904761904762,
+              "coverage_at_5pct_error": 0.255,
+              "coverage_at_1pct_error": 0.185,
+              "aurc": 0.22101453807685856,
+              "error_rate_at_0_9": 0.023809523809523808,
+              "confidence_bias": 0.049676360882909276,
               "top_bins": {
                 "0.9": {
-                  "n": 46,
+                  "n": 42,
                   "errors": 1,
-                  "error_rate": 0.021739130434782608
+                  "error_rate": 0.023809523809523808
                 },
                 "0.95": {
-                  "n": 29,
+                  "n": 28,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 6,
+                  "n": 5,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -6704,44 +6704,44 @@
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
-                  "accuracy": 0.75,
-                  "confidence_cutoff": 0.57725688415424
+                  "accuracy": 0.76,
+                  "confidence_cutoff": 0.5622010178659016
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.625,
-                  "confidence_cutoff": 0.26950159367730925
+                  "accuracy": 0.6125,
+                  "confidence_cutoff": 0.28172954890715934
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 200,
-              "nll": 1.3111522675312284,
+              "nll": 1.3020212799561357,
               "acc": 0.53,
-              "ece": 0.08739805659958974,
-              "brier": 0.5728987396461885,
-              "mean_conf": 0.5899912710608897,
+              "ece": 0.057886156442029546,
+              "brier": 0.5699198585015737,
+              "mean_conf": 0.5796763608829093,
               "confident_error_rate": 0.005,
-              "coverage_at_0_9": 0.23,
-              "accuracy_at_0_9": 0.9782608695652174,
-              "coverage_at_5pct_error": 0.25,
-              "coverage_at_1pct_error": 0.19,
-              "aurc": 0.21980746104696547,
-              "error_rate_at_0_9": 0.021739130434782608,
-              "confidence_bias": 0.05999127106088964,
+              "coverage_at_0_9": 0.21,
+              "accuracy_at_0_9": 0.9761904761904762,
+              "coverage_at_5pct_error": 0.255,
+              "coverage_at_1pct_error": 0.185,
+              "aurc": 0.22101453807685856,
+              "error_rate_at_0_9": 0.023809523809523808,
+              "confidence_bias": 0.049676360882909276,
               "top_bins": {
                 "0.9": {
-                  "n": 46,
+                  "n": 42,
                   "errors": 1,
-                  "error_rate": 0.021739130434782608
+                  "error_rate": 0.023809523809523808
                 },
                 "0.95": {
-                  "n": 29,
+                  "n": 28,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 6,
+                  "n": 5,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -6749,13 +6749,13 @@
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
-                  "accuracy": 0.75,
-                  "confidence_cutoff": 0.57725688415424
+                  "accuracy": 0.76,
+                  "confidence_cutoff": 0.5622010178659016
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.625,
-                  "confidence_cutoff": 0.26950159367730925
+                  "accuracy": 0.6125,
+                  "confidence_cutoff": 0.28172954890715934
                 }
               }
             },
@@ -6764,95 +6764,95 @@
           "paws": {
             "requested": 80,
             "valid_probability_rows": 80,
-            "correct": 72,
-            "accuracy": 0.9,
+            "correct": 73,
+            "accuracy": 0.9125,
             "full_probability_metrics": {
               "n": 80,
-              "nll": 0.28314251837212323,
-              "acc": 0.9,
-              "ece": 0.06247439765758463,
-              "brier": 0.15885879977500503,
-              "mean_conf": 0.9412949163036171,
-              "confident_error_rate": 0.0625,
-              "coverage_at_0_9": 0.8375,
-              "accuracy_at_0_9": 0.9253731343283582,
-              "coverage_at_5pct_error": 0.775,
-              "coverage_at_1pct_error": 0.2875,
-              "aurc": 0.030912620564059478,
-              "error_rate_at_0_9": 0.07462686567164178,
-              "confidence_bias": 0.041294916303617035,
+              "nll": 0.2779227201992138,
+              "acc": 0.9125,
+              "ece": 0.05493591159180497,
+              "brier": 0.15753943063635653,
+              "mean_conf": 0.9357409581036927,
+              "confident_error_rate": 0.05,
+              "coverage_at_0_9": 0.8125,
+              "accuracy_at_0_9": 0.9384615384615385,
+              "coverage_at_5pct_error": 0.7625,
+              "coverage_at_1pct_error": 0.3,
+              "aurc": 0.0308044058290765,
+              "error_rate_at_0_9": 0.06153846153846154,
+              "confidence_bias": 0.023240958103692755,
               "top_bins": {
                 "0.9": {
-                  "n": 67,
-                  "errors": 5,
-                  "error_rate": 0.07462686567164178
+                  "n": 65,
+                  "errors": 4,
+                  "error_rate": 0.06153846153846154
                 },
                 "0.95": {
-                  "n": 60,
+                  "n": 58,
                   "errors": 2,
-                  "error_rate": 0.03333333333333333
+                  "error_rate": 0.034482758620689655
                 },
                 "0.99": {
-                  "n": 2,
+                  "n": 0,
                   "errors": 0,
-                  "error_rate": 0.0
+                  "error_rate": null
                 }
               },
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.975,
-                  "confidence_cutoff": 0.9773598813644135
+                  "confidence_cutoff": 0.9736310261741858
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.9375,
-                  "confidence_cutoff": 0.9395250222016129
+                  "confidence_cutoff": 0.9301026006058246
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 80,
-              "nll": 0.28314251837212323,
-              "acc": 0.9,
-              "ece": 0.06247439765758463,
-              "brier": 0.15885879977500503,
-              "mean_conf": 0.9412949163036171,
-              "confident_error_rate": 0.0625,
-              "coverage_at_0_9": 0.8375,
-              "accuracy_at_0_9": 0.9253731343283582,
-              "coverage_at_5pct_error": 0.775,
-              "coverage_at_1pct_error": 0.2875,
-              "aurc": 0.030912620564059478,
-              "error_rate_at_0_9": 0.07462686567164178,
-              "confidence_bias": 0.041294916303617035,
+              "nll": 0.2779227201992138,
+              "acc": 0.9125,
+              "ece": 0.05493591159180497,
+              "brier": 0.15753943063635653,
+              "mean_conf": 0.9357409581036927,
+              "confident_error_rate": 0.05,
+              "coverage_at_0_9": 0.8125,
+              "accuracy_at_0_9": 0.9384615384615385,
+              "coverage_at_5pct_error": 0.7625,
+              "coverage_at_1pct_error": 0.3,
+              "aurc": 0.0308044058290765,
+              "error_rate_at_0_9": 0.06153846153846154,
+              "confidence_bias": 0.023240958103692755,
               "top_bins": {
                 "0.9": {
-                  "n": 67,
-                  "errors": 5,
-                  "error_rate": 0.07462686567164178
+                  "n": 65,
+                  "errors": 4,
+                  "error_rate": 0.06153846153846154
                 },
                 "0.95": {
-                  "n": 60,
+                  "n": 58,
                   "errors": 2,
-                  "error_rate": 0.03333333333333333
+                  "error_rate": 0.034482758620689655
                 },
                 "0.99": {
-                  "n": 2,
+                  "n": 0,
                   "errors": 0,
-                  "error_rate": 0.0
+                  "error_rate": null
                 }
               },
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.975,
-                  "confidence_cutoff": 0.9773598813644135
+                  "confidence_cutoff": 0.9736310261741858
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.9375,
-                  "confidence_cutoff": 0.9395250222016129
+                  "confidence_cutoff": 0.9301026006058246
                 }
               }
             },
@@ -6865,91 +6865,91 @@
             "accuracy": 0.9125,
             "full_probability_metrics": {
               "n": 80,
-              "nll": 0.2161076588398619,
+              "nll": 0.21231177025039338,
               "acc": 0.9125,
-              "ece": 0.038010106954782505,
-              "brier": 0.11745397306383834,
-              "mean_conf": 0.9376866478309165,
+              "ece": 0.026039786769557725,
+              "brier": 0.11561259420716938,
+              "mean_conf": 0.9322130681411511,
               "confident_error_rate": 0.025,
-              "coverage_at_0_9": 0.8375,
-              "accuracy_at_0_9": 0.9701492537313433,
+              "coverage_at_0_9": 0.825,
+              "accuracy_at_0_9": 0.9696969696969697,
               "coverage_at_5pct_error": 0.925,
-              "coverage_at_1pct_error": 0.075,
-              "aurc": 0.039172223012800614,
-              "error_rate_at_0_9": 0.029850746268656716,
-              "confidence_bias": 0.02518664783091651,
+              "coverage_at_1pct_error": 0.1,
+              "aurc": 0.03569781457299058,
+              "error_rate_at_0_9": 0.030303030303030304,
+              "confidence_bias": 0.019713068141151147,
               "top_bins": {
                 "0.9": {
-                  "n": 67,
+                  "n": 66,
                   "errors": 2,
-                  "error_rate": 0.029850746268656716
+                  "error_rate": 0.030303030303030304
                 },
                 "0.95": {
-                  "n": 62,
-                  "errors": 2,
-                  "error_rate": 0.03225806451612903
+                  "n": 61,
+                  "errors": 1,
+                  "error_rate": 0.01639344262295082
                 },
                 "0.99": {
-                  "n": 26,
+                  "n": 14,
                   "errors": 1,
-                  "error_rate": 0.038461538461538464
+                  "error_rate": 0.07142857142857142
                 }
               },
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.975,
-                  "confidence_cutoff": 0.9851611733370504
+                  "confidence_cutoff": 0.9814442414444764
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.96875,
-                  "confidence_cutoff": 0.9458952863255798
+                  "confidence_cutoff": 0.9395185848998813
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 80,
-              "nll": 0.2161076588398619,
+              "nll": 0.21231177025039338,
               "acc": 0.9125,
-              "ece": 0.038010106954782505,
-              "brier": 0.11745397306383834,
-              "mean_conf": 0.9376866478309165,
+              "ece": 0.026039786769557725,
+              "brier": 0.11561259420716938,
+              "mean_conf": 0.9322130681411511,
               "confident_error_rate": 0.025,
-              "coverage_at_0_9": 0.8375,
-              "accuracy_at_0_9": 0.9701492537313433,
+              "coverage_at_0_9": 0.825,
+              "accuracy_at_0_9": 0.9696969696969697,
               "coverage_at_5pct_error": 0.925,
-              "coverage_at_1pct_error": 0.075,
-              "aurc": 0.039172223012800614,
-              "error_rate_at_0_9": 0.029850746268656716,
-              "confidence_bias": 0.02518664783091651,
+              "coverage_at_1pct_error": 0.1,
+              "aurc": 0.03569781457299058,
+              "error_rate_at_0_9": 0.030303030303030304,
+              "confidence_bias": 0.019713068141151147,
               "top_bins": {
                 "0.9": {
-                  "n": 67,
+                  "n": 66,
                   "errors": 2,
-                  "error_rate": 0.029850746268656716
+                  "error_rate": 0.030303030303030304
                 },
                 "0.95": {
-                  "n": 62,
-                  "errors": 2,
-                  "error_rate": 0.03225806451612903
+                  "n": 61,
+                  "errors": 1,
+                  "error_rate": 0.01639344262295082
                 },
                 "0.99": {
-                  "n": 26,
+                  "n": 14,
                   "errors": 1,
-                  "error_rate": 0.038461538461538464
+                  "error_rate": 0.07142857142857142
                 }
               },
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.975,
-                  "confidence_cutoff": 0.9851611733370504
+                  "confidence_cutoff": 0.9814442414444764
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.96875,
-                  "confidence_cutoff": 0.9458952863255798
+                  "confidence_cutoff": 0.9395185848998813
                 }
               }
             },
@@ -6962,11 +6962,11 @@
             "accuracy": 1.0,
             "full_probability_metrics": {
               "n": 80,
-              "nll": 0.006236348098711358,
+              "nll": 0.007801897547958209,
               "acc": 1.0,
-              "ece": 0.006149372309216639,
-              "brier": 0.00030747770792780715,
-              "mean_conf": 0.9938506276907834,
+              "ece": 0.007676312134330843,
+              "brier": 0.0004369807984377848,
+              "mean_conf": 0.9923236878656692,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -6974,7 +6974,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.006149372309216639,
+              "confidence_bias": -0.007676312134330843,
               "top_bins": {
                 "0.9": {
                   "n": 80,
@@ -6982,12 +6982,12 @@
                   "error_rate": 0.0
                 },
                 "0.95": {
-                  "n": 79,
+                  "n": 78,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 69,
+                  "n": 65,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -6996,22 +6996,22 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9978451209130044
+                  "confidence_cutoff": 0.9965300683221596
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9938674921230647
+                  "confidence_cutoff": 0.9921908745945206
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 80,
-              "nll": 0.006236348098711358,
+              "nll": 0.007801897547958209,
               "acc": 1.0,
-              "ece": 0.006149372309216639,
-              "brier": 0.00030747770792780715,
-              "mean_conf": 0.9938506276907834,
+              "ece": 0.007676312134330843,
+              "brier": 0.0004369807984377848,
+              "mean_conf": 0.9923236878656692,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -7019,7 +7019,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.006149372309216639,
+              "confidence_bias": -0.007676312134330843,
               "top_bins": {
                 "0.9": {
                   "n": 80,
@@ -7027,12 +7027,12 @@
                   "error_rate": 0.0
                 },
                 "0.95": {
-                  "n": 79,
+                  "n": 78,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 69,
+                  "n": 65,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -7041,12 +7041,12 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9978451209130044
+                  "confidence_cutoff": 0.9965300683221596
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9938674921230647
+                  "confidence_cutoff": 0.9921908745945206
                 }
               }
             },
@@ -7059,19 +7059,19 @@
             "accuracy": 0.75,
             "full_probability_metrics": {
               "n": 80,
-              "nll": 0.47556818540480367,
+              "nll": 0.4720489807336611,
               "acc": 0.75,
-              "ece": 0.07643617030229378,
-              "brier": 0.3199264336587134,
-              "mean_conf": 0.8258139805835978,
+              "ece": 0.0761936963725907,
+              "brier": 0.3177754348396661,
+              "mean_conf": 0.8169291651727064,
               "confident_error_rate": 0.0375,
               "coverage_at_0_9": 0.4125,
               "accuracy_at_0_9": 0.9090909090909091,
               "coverage_at_5pct_error": 0.3625,
-              "coverage_at_1pct_error": 0.35,
-              "aurc": 0.09222008593150528,
+              "coverage_at_1pct_error": 0.3375,
+              "aurc": 0.09282282377572246,
               "error_rate_at_0_9": 0.09090909090909091,
-              "confidence_bias": 0.07581398058359778,
+              "confidence_bias": 0.06692916517270642,
               "top_bins": {
                 "0.9": {
                   "n": 33,
@@ -7079,7 +7079,7 @@
                   "error_rate": 0.09090909090909091
                 },
                 "0.95": {
-                  "n": 24,
+                  "n": 21,
                   "errors": 0,
                   "error_rate": 0.0
                 },
@@ -7092,31 +7092,31 @@
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
-                  "accuracy": 0.9,
-                  "confidence_cutoff": 0.8609209785002616
+                  "accuracy": 0.925,
+                  "confidence_cutoff": 0.8560822733652931
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.828125,
-                  "confidence_cutoff": 0.6948713064193726
+                  "confidence_cutoff": 0.6797098123940769
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 80,
-              "nll": 0.47556818540480367,
+              "nll": 0.4720489807336611,
               "acc": 0.75,
-              "ece": 0.07643617030229378,
-              "brier": 0.3199264336587134,
-              "mean_conf": 0.8258139805835978,
+              "ece": 0.0761936963725907,
+              "brier": 0.3177754348396661,
+              "mean_conf": 0.8169291651727064,
               "confident_error_rate": 0.0375,
               "coverage_at_0_9": 0.4125,
               "accuracy_at_0_9": 0.9090909090909091,
               "coverage_at_5pct_error": 0.3625,
-              "coverage_at_1pct_error": 0.35,
-              "aurc": 0.09222008593150528,
+              "coverage_at_1pct_error": 0.3375,
+              "aurc": 0.09282282377572246,
               "error_rate_at_0_9": 0.09090909090909091,
-              "confidence_bias": 0.07581398058359778,
+              "confidence_bias": 0.06692916517270642,
               "top_bins": {
                 "0.9": {
                   "n": 33,
@@ -7124,7 +7124,7 @@
                   "error_rate": 0.09090909090909091
                 },
                 "0.95": {
-                  "n": 24,
+                  "n": 21,
                   "errors": 0,
                   "error_rate": 0.0
                 },
@@ -7137,13 +7137,13 @@
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
-                  "accuracy": 0.9,
-                  "confidence_cutoff": 0.8609209785002616
+                  "accuracy": 0.925,
+                  "confidence_cutoff": 0.8560822733652931
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.828125,
-                  "confidence_cutoff": 0.6948713064193726
+                  "confidence_cutoff": 0.6797098123940769
                 }
               }
             },
@@ -7156,11 +7156,11 @@
             "accuracy": 1.0,
             "full_probability_metrics": {
               "n": 10,
-              "nll": 0.007841155789564139,
+              "nll": 0.01010806429581184,
               "acc": 1.0,
-              "ece": 0.007806247293806301,
-              "brier": 0.00013872505147745316,
-              "mean_conf": 0.9921937527061937,
+              "ece": 0.010050520182894429,
+              "brier": 0.00022828427851469854,
+              "mean_conf": 0.9899494798171056,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -7168,7 +7168,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.007806247293806301,
+              "confidence_bias": -0.010050520182894429,
               "top_bins": {
                 "0.9": {
                   "n": 10,
@@ -7181,7 +7181,7 @@
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 6,
+                  "n": 5,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -7190,22 +7190,22 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9939010793872055
+                  "confidence_cutoff": 0.9920052497980926
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9896593136903282
+                  "confidence_cutoff": 0.9867894292404178
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 10,
-              "nll": 0.007841155789564139,
+              "nll": 0.01010806429581184,
               "acc": 1.0,
-              "ece": 0.007806247293806301,
-              "brier": 0.00013872505147745316,
-              "mean_conf": 0.9921937527061937,
+              "ece": 0.010050520182894429,
+              "brier": 0.00022828427851469854,
+              "mean_conf": 0.9899494798171056,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -7213,7 +7213,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.007806247293806301,
+              "confidence_bias": -0.010050520182894429,
               "top_bins": {
                 "0.9": {
                   "n": 10,
@@ -7226,7 +7226,7 @@
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 6,
+                  "n": 5,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -7235,12 +7235,12 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9939010793872055
+                  "confidence_cutoff": 0.9920052497980926
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9896593136903282
+                  "confidence_cutoff": 0.9867894292404178
                 }
               }
             },
@@ -7253,11 +7253,11 @@
             "accuracy": 1.0,
             "full_probability_metrics": {
               "n": 10,
-              "nll": 0.0107929267557592,
+              "nll": 0.013336190464884707,
               "acc": 1.0,
-              "ece": 0.010701462639340464,
-              "brier": 0.00036120660545170313,
-              "mean_conf": 0.9892985373606595,
+              "ece": 0.013199482511126015,
+              "brier": 0.0005383885458560314,
+              "mean_conf": 0.986800517488874,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -7265,7 +7265,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.010701462639340464,
+              "confidence_bias": -0.013199482511126015,
               "top_bins": {
                 "0.9": {
                   "n": 10,
@@ -7287,22 +7287,22 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9969034000449554
+                  "confidence_cutoff": 0.9957628189842604
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9816944573491596
+                  "confidence_cutoff": 0.9774085096482822
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 10,
-              "nll": 0.0107929267557592,
+              "nll": 0.013336190464884707,
               "acc": 1.0,
-              "ece": 0.010701462639340464,
-              "brier": 0.00036120660545170313,
-              "mean_conf": 0.9892985373606595,
+              "ece": 0.013199482511126015,
+              "brier": 0.0005383885458560314,
+              "mean_conf": 0.986800517488874,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -7310,7 +7310,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.010701462639340464,
+              "confidence_bias": -0.013199482511126015,
               "top_bins": {
                 "0.9": {
                   "n": 10,
@@ -7332,12 +7332,12 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9969034000449554
+                  "confidence_cutoff": 0.9957628189842604
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9816944573491596
+                  "confidence_cutoff": 0.9774085096482822
                 }
               }
             },
@@ -7350,11 +7350,11 @@
             "accuracy": 0.8,
             "full_probability_metrics": {
               "n": 10,
-              "nll": 0.2138260506584651,
+              "nll": 0.22694873211843464,
               "acc": 0.8,
-              "ece": 0.15741611133328517,
-              "brier": 0.1276425479499806,
-              "mean_conf": 0.856580016184909,
+              "ece": 0.16555386795989538,
+              "brier": 0.13550345474895487,
+              "mean_conf": 0.8535695538437231,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.6,
               "accuracy_at_0_9": 1.0,
@@ -7362,7 +7362,7 @@
               "coverage_at_1pct_error": 0.8,
               "aurc": 0.03111111111111111,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": 0.05658001618490893,
+              "confidence_bias": 0.05356955384372308,
               "top_bins": {
                 "0.9": {
                   "n": 6,
@@ -7370,12 +7370,12 @@
                   "error_rate": 0.0
                 },
                 "0.95": {
-                  "n": 4,
+                  "n": 3,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 2,
+                  "n": 1,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -7384,24 +7384,24 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9426117959877142
+                  "confidence_cutoff": 0.9367303412067545
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.8415699130530643
+                  "confidence_cutoff": 0.8305143644338201
                 }
               },
-              "score_mae": 0.16071754616245987,
-              "ranked_probability_score": 0.03275971187271286
+              "score_mae": 0.16955440723819443,
+              "ranked_probability_score": 0.034895014874681045
             },
             "covered_only_probability_metrics": {
               "n": 10,
-              "nll": 0.2138260506584651,
+              "nll": 0.22694873211843464,
               "acc": 0.8,
-              "ece": 0.15741611133328517,
-              "brier": 0.1276425479499806,
-              "mean_conf": 0.856580016184909,
+              "ece": 0.16555386795989538,
+              "brier": 0.13550345474895487,
+              "mean_conf": 0.8535695538437231,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.6,
               "accuracy_at_0_9": 1.0,
@@ -7409,7 +7409,7 @@
               "coverage_at_1pct_error": 0.8,
               "aurc": 0.03111111111111111,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": 0.05658001618490893,
+              "confidence_bias": 0.05356955384372308,
               "top_bins": {
                 "0.9": {
                   "n": 6,
@@ -7417,12 +7417,12 @@
                   "error_rate": 0.0
                 },
                 "0.95": {
-                  "n": 4,
+                  "n": 3,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 2,
+                  "n": 1,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -7431,16 +7431,16 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9426117959877142
+                  "confidence_cutoff": 0.9367303412067545
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.8415699130530643
+                  "confidence_cutoff": 0.8305143644338201
                 }
               },
-              "score_mae": 0.16071754616245987,
-              "ranked_probability_score": 0.03275971187271286
+              "score_mae": 0.16955440723819443,
+              "ranked_probability_score": 0.034895014874681045
             },
             "coverage": 1.0
           },
@@ -7451,11 +7451,11 @@
             "accuracy": 1.0,
             "full_probability_metrics": {
               "n": 10,
-              "nll": 0.07005694097555375,
+              "nll": 0.061294700563837144,
               "acc": 1.0,
-              "ece": 0.0540813816797954,
-              "brier": 0.04110395968042234,
-              "mean_conf": 0.9459186183202046,
+              "ece": 0.04980564296050656,
+              "brier": 0.03092495951104666,
+              "mean_conf": 0.9501943570394934,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.9,
               "accuracy_at_0_9": 1.0,
@@ -7463,7 +7463,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.05408138167979537,
+              "confidence_bias": -0.0498056429605066,
               "top_bins": {
                 "0.9": {
                   "n": 9,
@@ -7476,7 +7476,7 @@
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 6,
+                  "n": 5,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -7485,24 +7485,24 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.993819476814992
+                  "confidence_cutoff": 0.9916381797053619
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9798565648382314
+                  "confidence_cutoff": 0.9777806624356902
                 }
               },
-              "score_mae": 0.0487152458265909,
-              "ranked_probability_score": 0.00975869637261514
+              "score_mae": 0.04333117262198813,
+              "ranked_probability_score": 0.007196645402378658
             },
             "covered_only_probability_metrics": {
               "n": 10,
-              "nll": 0.07005694097555375,
+              "nll": 0.061294700563837144,
               "acc": 1.0,
-              "ece": 0.0540813816797954,
-              "brier": 0.04110395968042234,
-              "mean_conf": 0.9459186183202046,
+              "ece": 0.04980564296050656,
+              "brier": 0.03092495951104666,
+              "mean_conf": 0.9501943570394934,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.9,
               "accuracy_at_0_9": 1.0,
@@ -7510,7 +7510,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.05408138167979537,
+              "confidence_bias": -0.0498056429605066,
               "top_bins": {
                 "0.9": {
                   "n": 9,
@@ -7523,7 +7523,7 @@
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 6,
+                  "n": 5,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -7532,16 +7532,16 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.993819476814992
+                  "confidence_cutoff": 0.9916381797053619
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9798565648382314
+                  "confidence_cutoff": 0.9777806624356902
                 }
               },
-              "score_mae": 0.0487152458265909,
-              "ranked_probability_score": 0.00975869637261514
+              "score_mae": 0.04333117262198813,
+              "ranked_probability_score": 0.007196645402378658
             },
             "coverage": 1.0
           },
@@ -7552,22 +7552,22 @@
             "accuracy": 1.0,
             "full_probability_metrics": {
               "n": 10,
-              "nll": 0.10819644087102691,
+              "nll": 0.12423624782442941,
               "acc": 1.0,
-              "ece": 0.0954442527194344,
-              "brier": 0.039610537719924295,
-              "mean_conf": 0.9045557472805654,
+              "ece": 0.10880820844844666,
+              "brier": 0.04779494768609947,
+              "mean_conf": 0.8911917915515533,
               "confident_error_rate": 0.0,
-              "coverage_at_0_9": 0.7,
+              "coverage_at_0_9": 0.6,
               "accuracy_at_0_9": 1.0,
               "coverage_at_5pct_error": 1.0,
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.09544425271943457,
+              "confidence_bias": -0.10880820844844674,
               "top_bins": {
                 "0.9": {
-                  "n": 7,
+                  "n": 6,
                   "errors": 0,
                   "error_rate": 0.0
                 },
@@ -7586,33 +7586,33 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9656626177441163
+                  "confidence_cutoff": 0.9558210808739257
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.881241046212756
+                  "confidence_cutoff": 0.8866819137686461
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 10,
-              "nll": 0.10819644087102691,
+              "nll": 0.12423624782442941,
               "acc": 1.0,
-              "ece": 0.0954442527194344,
-              "brier": 0.039610537719924295,
-              "mean_conf": 0.9045557472805654,
+              "ece": 0.10880820844844666,
+              "brier": 0.04779494768609947,
+              "mean_conf": 0.8911917915515533,
               "confident_error_rate": 0.0,
-              "coverage_at_0_9": 0.7,
+              "coverage_at_0_9": 0.6,
               "accuracy_at_0_9": 1.0,
               "coverage_at_5pct_error": 1.0,
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.09544425271943457,
+              "confidence_bias": -0.10880820844844674,
               "top_bins": {
                 "0.9": {
-                  "n": 7,
+                  "n": 6,
                   "errors": 0,
                   "error_rate": 0.0
                 },
@@ -7631,12 +7631,12 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9656626177441163
+                  "confidence_cutoff": 0.9558210808739257
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.881241046212756
+                  "confidence_cutoff": 0.8866819137686461
                 }
               }
             },
@@ -7649,11 +7649,11 @@
             "accuracy": 0.6,
             "full_probability_metrics": {
               "n": 10,
-              "nll": 0.8939128624245644,
+              "nll": 0.8828182838634889,
               "acc": 0.6,
-              "ece": 0.39838756334240605,
-              "brier": 0.6386225586932958,
-              "mean_conf": 0.8386086717206818,
+              "ece": 0.3920235500266451,
+              "brier": 0.6280874258674091,
+              "mean_conf": 0.8410658551406985,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.2,
               "accuracy_at_0_9": 1.0,
@@ -7661,7 +7661,7 @@
               "coverage_at_1pct_error": 0.2,
               "aurc": 0.288968253968254,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": 0.2386086717206818,
+              "confidence_bias": 0.24106585514069856,
               "top_bins": {
                 "0.9": {
                   "n": 2,
@@ -7683,22 +7683,22 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.8,
-                  "confidence_cutoff": 0.8853410746258817
+                  "confidence_cutoff": 0.8732116352500265
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.8282266608409089
+                  "confidence_cutoff": 0.8179686895920922
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 10,
-              "nll": 0.8939128624245644,
+              "nll": 0.8828182838634889,
               "acc": 0.6,
-              "ece": 0.39838756334240605,
-              "brier": 0.6386225586932958,
-              "mean_conf": 0.8386086717206818,
+              "ece": 0.3920235500266451,
+              "brier": 0.6280874258674091,
+              "mean_conf": 0.8410658551406985,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.2,
               "accuracy_at_0_9": 1.0,
@@ -7706,7 +7706,7 @@
               "coverage_at_1pct_error": 0.2,
               "aurc": 0.288968253968254,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": 0.2386086717206818,
+              "confidence_bias": 0.24106585514069856,
               "top_bins": {
                 "0.9": {
                   "n": 2,
@@ -7728,12 +7728,12 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.8,
-                  "confidence_cutoff": 0.8853410746258817
+                  "confidence_cutoff": 0.8732116352500265
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.8282266608409089
+                  "confidence_cutoff": 0.8179686895920922
                 }
               }
             },
@@ -7746,11 +7746,11 @@
             "accuracy": 0.9,
             "full_probability_metrics": {
               "n": 10,
-              "nll": 0.3878790410817737,
+              "nll": 0.38549807963539046,
               "acc": 0.9,
-              "ece": 0.20366310890419706,
-              "brier": 0.23894788485153878,
-              "mean_conf": 0.8019263309455292,
+              "ece": 0.20288794178646427,
+              "brier": 0.23489325574634798,
+              "mean_conf": 0.8015680467880623,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.3,
               "accuracy_at_0_9": 1.0,
@@ -7758,7 +7758,7 @@
               "coverage_at_1pct_error": 0.5,
               "aurc": 0.06456349206349207,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.09807366905447079,
+              "confidence_bias": -0.09843195321193776,
               "top_bins": {
                 "0.9": {
                   "n": 3,
@@ -7780,24 +7780,24 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.8513163369222139
+                  "confidence_cutoff": 0.8456577690156466
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.875,
-                  "confidence_cutoff": 0.7305872644461858
+                  "confidence_cutoff": 0.6947201852727033
                 }
               },
-              "score_mae": 0.2577528085044144,
-              "ranked_probability_score": 0.05968224629533503
+              "score_mae": 0.2564750859760038,
+              "ranked_probability_score": 0.05882371330201126
             },
             "covered_only_probability_metrics": {
               "n": 10,
-              "nll": 0.3878790410817737,
+              "nll": 0.38549807963539046,
               "acc": 0.9,
-              "ece": 0.20366310890419706,
-              "brier": 0.23894788485153878,
-              "mean_conf": 0.8019263309455292,
+              "ece": 0.20288794178646427,
+              "brier": 0.23489325574634798,
+              "mean_conf": 0.8015680467880623,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.3,
               "accuracy_at_0_9": 1.0,
@@ -7805,7 +7805,7 @@
               "coverage_at_1pct_error": 0.5,
               "aurc": 0.06456349206349207,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.09807366905447079,
+              "confidence_bias": -0.09843195321193776,
               "top_bins": {
                 "0.9": {
                   "n": 3,
@@ -7827,16 +7827,16 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.8513163369222139
+                  "confidence_cutoff": 0.8456577690156466
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.875,
-                  "confidence_cutoff": 0.7305872644461858
+                  "confidence_cutoff": 0.6947201852727033
                 }
               },
-              "score_mae": 0.2577528085044144,
-              "ranked_probability_score": 0.05968224629533503
+              "score_mae": 0.2564750859760038,
+              "ranked_probability_score": 0.05882371330201126
             },
             "coverage": 1.0
           },
@@ -7847,11 +7847,11 @@
             "accuracy": 0.7,
             "full_probability_metrics": {
               "n": 10,
-              "nll": 0.7470384623310196,
+              "nll": 0.7292834955292916,
               "acc": 0.7,
-              "ece": 0.22029693644423792,
-              "brier": 0.4309736445646065,
-              "mean_conf": 0.9202969364442379,
+              "ece": 0.21088673885292936,
+              "brier": 0.4247531105085889,
+              "mean_conf": 0.9108867388529293,
               "confident_error_rate": 0.1,
               "coverage_at_0_9": 0.7,
               "accuracy_at_0_9": 0.8571428571428571,
@@ -7859,7 +7859,7 @@
               "coverage_at_1pct_error": 0.4,
               "aurc": 0.12817460317460316,
               "error_rate_at_0_9": 0.14285714285714285,
-              "confidence_bias": 0.22029693644423798,
+              "confidence_bias": 0.2108867388529293,
               "top_bins": {
                 "0.9": {
                   "n": 7,
@@ -7881,24 +7881,24 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.8,
-                  "confidence_cutoff": 0.9773512719149
+                  "confidence_cutoff": 0.9757163386049921
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.75,
-                  "confidence_cutoff": 0.8938853735717587
+                  "confidence_cutoff": 0.8745092806607159
                 }
               },
-              "score_mae": 0.2732574139836782,
-              "ranked_probability_score": 0.10706854764238136
+              "score_mae": 0.27669455495743667,
+              "ranked_probability_score": 0.1053871341351588
             },
             "covered_only_probability_metrics": {
               "n": 10,
-              "nll": 0.7470384623310196,
+              "nll": 0.7292834955292916,
               "acc": 0.7,
-              "ece": 0.22029693644423792,
-              "brier": 0.4309736445646065,
-              "mean_conf": 0.9202969364442379,
+              "ece": 0.21088673885292936,
+              "brier": 0.4247531105085889,
+              "mean_conf": 0.9108867388529293,
               "confident_error_rate": 0.1,
               "coverage_at_0_9": 0.7,
               "accuracy_at_0_9": 0.8571428571428571,
@@ -7906,7 +7906,7 @@
               "coverage_at_1pct_error": 0.4,
               "aurc": 0.12817460317460316,
               "error_rate_at_0_9": 0.14285714285714285,
-              "confidence_bias": 0.22029693644423798,
+              "confidence_bias": 0.2108867388529293,
               "top_bins": {
                 "0.9": {
                   "n": 7,
@@ -7928,16 +7928,16 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.8,
-                  "confidence_cutoff": 0.9773512719149
+                  "confidence_cutoff": 0.9757163386049921
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.75,
-                  "confidence_cutoff": 0.8938853735717587
+                  "confidence_cutoff": 0.8745092806607159
                 }
               },
-              "score_mae": 0.2732574139836782,
-              "ranked_probability_score": 0.10706854764238136
+              "score_mae": 0.27669455495743667,
+              "ranked_probability_score": 0.1053871341351588
             },
             "coverage": 1.0
           },
@@ -7948,11 +7948,11 @@
             "accuracy": 1.0,
             "full_probability_metrics": {
               "n": 10,
-              "nll": 0.004226216609183238,
+              "nll": 0.005872511904691692,
               "acc": 1.0,
-              "ece": 0.004215976897349094,
-              "brier": 3.4649667131696104e-05,
-              "mean_conf": 0.9957840231026509,
+              "ece": 0.005852885976918243,
+              "brier": 6.553204955281063e-05,
+              "mean_conf": 0.9941471140230818,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -7960,7 +7960,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.004215976897349094,
+              "confidence_bias": -0.005852885976918243,
               "top_bins": {
                 "0.9": {
                   "n": 10,
@@ -7982,22 +7982,22 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9969052065210677
+                  "confidence_cutoff": 0.9956017537902697
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9942030038674581
+                  "confidence_cutoff": 0.992077102727452
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 10,
-              "nll": 0.004226216609183238,
+              "nll": 0.005872511904691692,
               "acc": 1.0,
-              "ece": 0.004215976897349094,
-              "brier": 3.4649667131696104e-05,
-              "mean_conf": 0.9957840231026509,
+              "ece": 0.005852885976918243,
+              "brier": 6.553204955281063e-05,
+              "mean_conf": 0.9941471140230818,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -8005,7 +8005,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.004215976897349094,
+              "confidence_bias": -0.005852885976918243,
               "top_bins": {
                 "0.9": {
                   "n": 10,
@@ -8027,12 +8027,12 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9969052065210677
+                  "confidence_cutoff": 0.9956017537902697
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9942030038674581
+                  "confidence_cutoff": 0.992077102727452
                 }
               }
             },
@@ -8045,11 +8045,11 @@
             "accuracy": 1.0,
             "full_probability_metrics": {
               "n": 10,
-              "nll": 0.007053870282788843,
+              "nll": 0.009442501422597043,
               "acc": 1.0,
-              "ece": 0.007024441570579243,
-              "brier": 9.776258511132903e-05,
-              "mean_conf": 0.9929755584294208,
+              "ece": 0.009390689401267194,
+              "brier": 0.00017066799723336928,
+              "mean_conf": 0.9906093105987328,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -8057,7 +8057,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.007024441570579243,
+              "confidence_bias": -0.009390689401267194,
               "top_bins": {
                 "0.9": {
                   "n": 10,
@@ -8079,24 +8079,24 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9932447768419814
+                  "confidence_cutoff": 0.9912854619430778
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9898739137651561
+                  "confidence_cutoff": 0.9857135990468268
                 }
               },
-              "score_mae": 0.007660911551517352,
-              "ranked_probability_score": 2.757223086398964e-05
+              "score_mae": 0.010231745432055298,
+              "ranked_probability_score": 4.863796909247521e-05
             },
             "covered_only_probability_metrics": {
               "n": 10,
-              "nll": 0.007053870282788843,
+              "nll": 0.009442501422597043,
               "acc": 1.0,
-              "ece": 0.007024441570579243,
-              "brier": 9.776258511132903e-05,
-              "mean_conf": 0.9929755584294208,
+              "ece": 0.009390689401267194,
+              "brier": 0.00017066799723336928,
+              "mean_conf": 0.9906093105987328,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -8104,7 +8104,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.007024441570579243,
+              "confidence_bias": -0.009390689401267194,
               "top_bins": {
                 "0.9": {
                   "n": 10,
@@ -8126,16 +8126,16 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9932447768419814
+                  "confidence_cutoff": 0.9912854619430778
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9898739137651561
+                  "confidence_cutoff": 0.9857135990468268
                 }
               },
-              "score_mae": 0.007660911551517352,
-              "ranked_probability_score": 2.757223086398964e-05
+              "score_mae": 0.010231745432055298,
+              "ranked_probability_score": 4.863796909247521e-05
             },
             "coverage": 1.0
           },
@@ -8146,11 +8146,11 @@
             "accuracy": 0.5,
             "full_probability_metrics": {
               "n": 10,
-              "nll": 0.9724116266401464,
+              "nll": 0.9490781913515873,
               "acc": 0.5,
-              "ece": 0.37317330379992186,
-              "brier": 0.592356663184495,
-              "mean_conf": 0.7285382769948201,
+              "ece": 0.36923739486967955,
+              "brier": 0.5785088804997336,
+              "mean_conf": 0.710935585802415,
               "confident_error_rate": 0.1,
               "coverage_at_0_9": 0.2,
               "accuracy_at_0_9": 0.5,
@@ -8158,7 +8158,7 @@
               "coverage_at_1pct_error": 0.1,
               "aurc": 0.356468253968254,
               "error_rate_at_0_9": 0.5,
-              "confidence_bias": 0.2285382769948201,
+              "confidence_bias": 0.210935585802415,
               "top_bins": {
                 "0.9": {
                   "n": 2,
@@ -8166,9 +8166,9 @@
                   "error_rate": 0.5
                 },
                 "0.95": {
-                  "n": 2,
-                  "errors": 1,
-                  "error_rate": 0.5
+                  "n": 1,
+                  "errors": 0,
+                  "error_rate": 0.0
                 },
                 "0.99": {
                   "n": 0,
@@ -8180,24 +8180,24 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.6,
-                  "confidence_cutoff": 0.7507937755021719
+                  "confidence_cutoff": 0.6925948236413493
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.625,
-                  "confidence_cutoff": 0.597999206215155
+                  "confidence_cutoff": 0.586704671382904
                 }
               },
-              "score_mae": 0.41248856844465875,
-              "ranked_probability_score": 0.14487339386062986
+              "score_mae": 0.41230806689600685,
+              "ranked_probability_score": 0.14147304522762058
             },
             "covered_only_probability_metrics": {
               "n": 10,
-              "nll": 0.9724116266401464,
+              "nll": 0.9490781913515873,
               "acc": 0.5,
-              "ece": 0.37317330379992186,
-              "brier": 0.592356663184495,
-              "mean_conf": 0.7285382769948201,
+              "ece": 0.36923739486967955,
+              "brier": 0.5785088804997336,
+              "mean_conf": 0.710935585802415,
               "confident_error_rate": 0.1,
               "coverage_at_0_9": 0.2,
               "accuracy_at_0_9": 0.5,
@@ -8205,7 +8205,7 @@
               "coverage_at_1pct_error": 0.1,
               "aurc": 0.356468253968254,
               "error_rate_at_0_9": 0.5,
-              "confidence_bias": 0.2285382769948201,
+              "confidence_bias": 0.210935585802415,
               "top_bins": {
                 "0.9": {
                   "n": 2,
@@ -8213,9 +8213,9 @@
                   "error_rate": 0.5
                 },
                 "0.95": {
-                  "n": 2,
-                  "errors": 1,
-                  "error_rate": 0.5
+                  "n": 1,
+                  "errors": 0,
+                  "error_rate": 0.0
                 },
                 "0.99": {
                   "n": 0,
@@ -8227,42 +8227,42 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.6,
-                  "confidence_cutoff": 0.7507937755021719
+                  "confidence_cutoff": 0.6925948236413493
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.625,
-                  "confidence_cutoff": 0.597999206215155
+                  "confidence_cutoff": 0.586704671382904
                 }
               },
-              "score_mae": 0.41248856844465875,
-              "ranked_probability_score": 0.14487339386062986
+              "score_mae": 0.41230806689600685,
+              "ranked_probability_score": 0.14147304522762058
             },
             "coverage": 1.0
           }
         },
-        "clean_task_macro_accuracy": 0.8235648148148148,
+        "clean_task_macro_accuracy": 0.8240277777777778,
         "permutation": {
           "requested_pairs": 36,
           "valid_pairs": 36,
           "both_correct_requested": 0.7777777777777778,
           "flip_rate": 0.08333333333333333,
           "covered_only_flip_rate": 0.08333333333333333,
-          "half_l1": 0.08515840387623191
+          "half_l1": 0.08228615038271361
         },
         "unknowable": {
           "requested": 110,
           "valid": 110,
           "accuracy": null,
           "full_confidence_metrics": {
-            "mean_max_p": 0.6485130260679773,
-            "share_at_0_9": 0.21818181818181817,
-            "normalized_entropy": 0.7184893503275872
+            "mean_max_p": 0.6359851141873526,
+            "share_at_0_9": 0.20909090909090908,
+            "normalized_entropy": 0.7358129257673091
           },
           "covered_only_confidence_metrics": {
-            "mean_max_p": 0.6485130260679773,
-            "share_at_0_9": 0.21818181818181817,
-            "normalized_entropy": 0.7184893503275872
+            "mean_max_p": 0.6359851141873526,
+            "share_at_0_9": 0.20909090909090908,
+            "normalized_entropy": 0.7358129257673091
           },
           "controls": {
             "requested": 110,
@@ -8271,32 +8271,32 @@
             "accuracy": 0.8636363636363636,
             "full_probability_metrics": {
               "n": 110,
-              "nll": 0.31120323585634946,
+              "nll": 0.30890154536131303,
               "acc": 0.8636363636363636,
-              "ece": 0.055639968292740274,
-              "brier": 0.1918081945957669,
-              "mean_conf": 0.9060614972263522,
+              "ece": 0.055763619767470476,
+              "brier": 0.18922444613084882,
+              "mean_conf": 0.9019016682678792,
               "confident_error_rate": 0.01818181818181818,
-              "coverage_at_0_9": 0.6909090909090909,
-              "accuracy_at_0_9": 0.9736842105263158,
-              "coverage_at_5pct_error": 0.7727272727272727,
-              "coverage_at_1pct_error": 0.5454545454545454,
-              "aurc": 0.029014791714251358,
-              "error_rate_at_0_9": 0.02631578947368421,
-              "confidence_bias": 0.04242513358998856,
+              "coverage_at_0_9": 0.6818181818181818,
+              "accuracy_at_0_9": 0.9733333333333334,
+              "coverage_at_5pct_error": 0.7545454545454545,
+              "coverage_at_1pct_error": 0.5363636363636364,
+              "aurc": 0.029053269688156625,
+              "error_rate_at_0_9": 0.02666666666666667,
+              "confidence_bias": 0.03826530463151556,
               "top_bins": {
                 "0.9": {
-                  "n": 76,
+                  "n": 75,
                   "errors": 2,
-                  "error_rate": 0.02631578947368421
+                  "error_rate": 0.02666666666666667
                 },
                 "0.95": {
-                  "n": 68,
-                  "errors": 2,
-                  "error_rate": 0.029411764705882353
+                  "n": 66,
+                  "errors": 1,
+                  "error_rate": 0.015151515151515152
                 },
                 "0.99": {
-                  "n": 40,
+                  "n": 37,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -8305,45 +8305,45 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9818909453374407
+                  "confidence_cutoff": 0.9777806624356902
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.9431818181818182,
-                  "confidence_cutoff": 0.8415699130530643
+                  "accuracy": 0.9318181818181818,
+                  "confidence_cutoff": 0.8305143644338201
                 }
               },
-              "score_mae": 0.19343208241221987,
-              "ranked_probability_score": 0.05902836137908972
+              "score_mae": 0.19476583885361418,
+              "ranked_probability_score": 0.05797069848515713
             },
             "covered_only_probability_metrics": {
               "n": 110,
-              "nll": 0.31120323585634946,
+              "nll": 0.30890154536131303,
               "acc": 0.8636363636363636,
-              "ece": 0.055639968292740274,
-              "brier": 0.1918081945957669,
-              "mean_conf": 0.9060614972263522,
+              "ece": 0.055763619767470476,
+              "brier": 0.18922444613084882,
+              "mean_conf": 0.9019016682678792,
               "confident_error_rate": 0.01818181818181818,
-              "coverage_at_0_9": 0.6909090909090909,
-              "accuracy_at_0_9": 0.9736842105263158,
-              "coverage_at_5pct_error": 0.7727272727272727,
-              "coverage_at_1pct_error": 0.5454545454545454,
-              "aurc": 0.029014791714251358,
-              "error_rate_at_0_9": 0.02631578947368421,
-              "confidence_bias": 0.04242513358998856,
+              "coverage_at_0_9": 0.6818181818181818,
+              "accuracy_at_0_9": 0.9733333333333334,
+              "coverage_at_5pct_error": 0.7545454545454545,
+              "coverage_at_1pct_error": 0.5363636363636364,
+              "aurc": 0.029053269688156625,
+              "error_rate_at_0_9": 0.02666666666666667,
+              "confidence_bias": 0.03826530463151556,
               "top_bins": {
                 "0.9": {
-                  "n": 76,
+                  "n": 75,
                   "errors": 2,
-                  "error_rate": 0.02631578947368421
+                  "error_rate": 0.02666666666666667
                 },
                 "0.95": {
-                  "n": 68,
-                  "errors": 2,
-                  "error_rate": 0.029411764705882353
+                  "n": 66,
+                  "errors": 1,
+                  "error_rate": 0.015151515151515152
                 },
                 "0.99": {
-                  "n": 40,
+                  "n": 37,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -8352,22 +8352,22 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9818909453374407
+                  "confidence_cutoff": 0.9777806624356902
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.9431818181818182,
-                  "confidence_cutoff": 0.8415699130530643
+                  "accuracy": 0.9318181818181818,
+                  "confidence_cutoff": 0.8305143644338201
                 }
               },
-              "score_mae": 0.19343208241221987,
-              "ranked_probability_score": 0.05902836137908972
+              "score_mae": 0.19476583885361418,
+              "ranked_probability_score": 0.05797069848515713
             },
             "coverage": 1.0
           },
           "requested_pairs": 110,
           "valid_pairs": 110,
-          "paired_confidence_drop": 0.2575484711583747
+          "paired_confidence_drop": 0.2659165540805263
         },
         "coverage": {
           "questions": 1264,
@@ -8376,7 +8376,7 @@
           "truncated_questions": 0
         },
         "complete_unchanged_upstream_report": {
-          "objective": -0.42681700298384834,
+          "objective": -0.4212918162126965,
           "paired_flip": {
             "pairs": 64,
             "flip_rate": 0.953125,
@@ -8387,69 +8387,69 @@
           },
           "unknowable": {
             "n": 110,
-            "mean_max_p": 0.6485130260679773,
-            "share_at_0_9": 0.21818181818181817,
-            "control_mean_max_p": 0.9060614972263522,
-            "control_share_at_0_9": 0.6909090909090909,
+            "mean_max_p": 0.6359851141873526,
+            "share_at_0_9": 0.20909090909090908,
+            "control_mean_max_p": 0.9019016682678792,
+            "control_share_at_0_9": 0.6818181818181818,
             "control_acc": 0.8636363636363636,
-            "paired_confidence_drop": 0.2575484711583747,
-            "share_less_confident_than_control": 0.9
+            "paired_confidence_drop": 0.2659165540805263,
+            "share_less_confident_than_control": 0.9090909090909091
           },
           "clean": {
             "n": 1046,
-            "nll": 0.5940603022446113,
-            "acc": 0.7762906309751434,
-            "ece": 0.05558262070294892,
-            "brier": 0.2937157813621403,
-            "mean_conf": 0.8318732516780923,
-            "confident_error_rate": 0.02390057361376673,
-            "coverage_at_0_9": 0.5860420650095602,
-            "accuracy_at_0_9": 0.9592169657422512,
-            "coverage_at_5pct_error": 0.6347992351816444,
-            "coverage_at_1pct_error": 0.3030592734225621,
-            "aurc": 0.05816033320114623,
-            "error_rate_at_0_9": 0.040783034257748776,
-            "confidence_bias": 0.05558262070294895,
+            "nll": 0.5870477618581573,
+            "acc": 0.777246653919694,
+            "ece": 0.04794282897297868,
+            "brier": 0.2910675481135292,
+            "mean_conf": 0.8251894828926727,
+            "confident_error_rate": 0.02198852772466539,
+            "coverage_at_0_9": 0.5717017208413002,
+            "accuracy_at_0_9": 0.9615384615384616,
+            "coverage_at_5pct_error": 0.6338432122370937,
+            "coverage_at_1pct_error": 0.29923518164435947,
+            "aurc": 0.057865847250986556,
+            "error_rate_at_0_9": 0.038461538461538464,
+            "confidence_bias": 0.047942828972978635,
             "top_bins": {
               "0.9": {
-                "n": 613,
-                "errors": 25,
-                "error_rate": 0.040783034257748776
+                "n": 598,
+                "errors": 23,
+                "error_rate": 0.038461538461538464
               },
               "0.95": {
-                "n": 514,
-                "errors": 12,
-                "error_rate": 0.023346303501945526
+                "n": 491,
+                "errors": 9,
+                "error_rate": 0.018329938900203666
               },
               "0.99": {
-                "n": 207,
+                "n": 174,
                 "errors": 3,
-                "error_rate": 0.014492753623188406
+                "error_rate": 0.017241379310344827
               }
             },
             "selective": {
               "0.5": {
-                "coverage": 0.5009560229445507,
-                "accuracy": 0.9770992366412213,
-                "confidence_cutoff": 0.947606086730957
+                "coverage": 0.5,
+                "accuracy": 0.9751434034416826,
+                "confidence_cutoff": 0.9408829808235168
               },
               "0.8": {
                 "coverage": 0.8001912045889101,
-                "accuracy": 0.8781362007168458,
-                "confidence_cutoff": 0.647434909565297
+                "accuracy": 0.8793309438470729,
+                "confidence_cutoff": 0.6510422047867755
               }
             },
-            "score_mae": 0.18149603870298436,
-            "ranked_probability_score": 0.051063424067515496
+            "score_mae": 0.18602619774583048,
+            "ranked_probability_score": 0.050882127676710784
           },
           "tasks": {
             "buried_emotion": {
               "n": 20,
-              "nll": 1.4899537928777824,
+              "nll": 1.4638562023564279,
               "acc": 0.45,
-              "ece": 0.3017003229117818,
-              "brier": 0.6913230901539468,
-              "mean_conf": 0.6857035964205832,
+              "ece": 0.22400769818984456,
+              "brier": 0.685438069283081,
+              "mean_conf": 0.6594526435269211,
               "confident_error_rate": 0.05,
               "coverage_at_0_9": 0.2,
               "accuracy_at_0_9": 0.75,
@@ -8457,7 +8457,7 @@
               "coverage_at_1pct_error": 0.05,
               "aurc": 0.3734132508358514,
               "error_rate_at_0_9": 0.25,
-              "confidence_bias": 0.2357035964205832,
+              "confidence_bias": 0.2094526435269211,
               "top_bins": {
                 "0.9": {
                   "n": 4,
@@ -8479,35 +8479,35 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.6,
-                  "confidence_cutoff": 0.720178043538323
+                  "confidence_cutoff": 0.6973127263072573
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.49892304084621675
+                  "confidence_cutoff": 0.483508165065625
                 }
               }
             },
             "buried_paws": {
               "n": 20,
-              "nll": 0.391154446854581,
+              "nll": 0.36403248618468526,
               "acc": 0.8,
-              "ece": 0.0909007219853577,
-              "brier": 0.24269024392468377,
-              "mean_conf": 0.8773054444371697,
+              "ece": 0.1295361487173242,
+              "brier": 0.2257044216488208,
+              "mean_conf": 0.8831790381288052,
               "confident_error_rate": 0.05,
-              "coverage_at_0_9": 0.7,
-              "accuracy_at_0_9": 0.9285714285714286,
+              "coverage_at_0_9": 0.65,
+              "accuracy_at_0_9": 0.9230769230769231,
               "coverage_at_5pct_error": 0.45,
               "coverage_at_1pct_error": 0.45,
-              "aurc": 0.06282346007965203,
-              "error_rate_at_0_9": 0.07142857142857142,
-              "confidence_bias": 0.07730544443716969,
+              "aurc": 0.05397950583128602,
+              "error_rate_at_0_9": 0.07692307692307693,
+              "confidence_bias": 0.08317903812880512,
               "top_bins": {
                 "0.9": {
-                  "n": 14,
+                  "n": 13,
                   "errors": 1,
-                  "error_rate": 0.07142857142857142
+                  "error_rate": 0.07692307692307693
                 },
                 "0.95": {
                   "n": 12,
@@ -8524,43 +8524,43 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.9,
-                  "confidence_cutoff": 0.9649345170900093
+                  "confidence_cutoff": 0.9546051514751633
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.875,
-                  "confidence_cutoff": 0.7085450713959379
+                  "accuracy": 0.9375,
+                  "confidence_cutoff": 0.7867595084452922
                 }
               }
             },
             "buried_qnli": {
               "n": 20,
-              "nll": 0.173920037838875,
+              "nll": 0.18031139864262885,
               "acc": 0.9,
-              "ece": 0.08884237104933625,
-              "brier": 0.11334577453222641,
-              "mean_conf": 0.9453143823740051,
+              "ece": 0.0817012190955263,
+              "brier": 0.11604628935827062,
+              "mean_conf": 0.9409710011124263,
               "confident_error_rate": 0.0,
-              "coverage_at_0_9": 0.85,
+              "coverage_at_0_9": 0.8,
               "accuracy_at_0_9": 1.0,
               "coverage_at_5pct_error": 0.9,
               "coverage_at_1pct_error": 0.9,
               "aurc": 0.0076315789473684215,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": 0.04531438237400509,
+              "confidence_bias": 0.04097100111242624,
               "top_bins": {
                 "0.9": {
-                  "n": 17,
+                  "n": 16,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.95": {
-                  "n": 16,
+                  "n": 15,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 2,
+                  "n": 1,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -8569,30 +8569,30 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9793051012887353
+                  "confidence_cutoff": 0.9736818072730682
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9588658106453812
+                  "confidence_cutoff": 0.9485171543268541
                 }
               }
             },
             "buried_tweet_offensive": {
               "n": 20,
-              "nll": 0.963421060535776,
+              "nll": 0.9361317272289519,
               "acc": 0.4,
-              "ece": 0.3598356256038071,
-              "brier": 0.7060594827987038,
-              "mean_conf": 0.7594032993432073,
+              "ece": 0.3467800262809543,
+              "brier": 0.6890225027879284,
+              "mean_conf": 0.7467800262809542,
               "confident_error_rate": 0.05,
               "coverage_at_0_9": 0.1,
               "accuracy_at_0_9": 0.5,
               "coverage_at_5pct_error": 0.05,
               "coverage_at_1pct_error": 0.05,
-              "aurc": 0.45470466259746445,
+              "aurc": 0.46809751974032154,
               "error_rate_at_0_9": 0.5,
-              "confidence_bias": 0.35940329934320725,
+              "confidence_bias": 0.34678002628095417,
               "top_bins": {
                 "0.9": {
                   "n": 2,
@@ -8600,9 +8600,9 @@
                   "error_rate": 0.5
                 },
                 "0.95": {
-                  "n": 2,
-                  "errors": 1,
-                  "error_rate": 0.5
+                  "n": 1,
+                  "errors": 0,
+                  "error_rate": 0.0
                 },
                 "0.99": {
                   "n": 0,
@@ -8614,30 +8614,30 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.7584096306325441
+                  "confidence_cutoff": 0.7481542379764177
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.4375,
-                  "confidence_cutoff": 0.6582031053840188
+                  "confidence_cutoff": 0.6360844184135074
                 }
               }
             },
             "composition_final_combination": {
               "n": 32,
-              "nll": 0.13034778836964267,
+              "nll": 0.12979322912048136,
               "acc": 0.96875,
-              "ece": 0.07387770926294471,
-              "brier": 0.06436468242012063,
-              "mean_conf": 0.9252340474017305,
+              "ece": 0.05865632527071586,
+              "brier": 0.06221598573997203,
+              "mean_conf": 0.9181713626206109,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.75,
               "accuracy_at_0_9": 1.0,
               "coverage_at_5pct_error": 1.0,
-              "coverage_at_1pct_error": 0.84375,
-              "aurc": 0.005186349167726045,
+              "coverage_at_1pct_error": 0.875,
+              "aurc": 0.004070277739154617,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.04351595259826946,
+              "confidence_bias": -0.05057863737938906,
               "top_bins": {
                 "0.9": {
                   "n": 24,
@@ -8645,7 +8645,7 @@
                   "error_rate": 0.0
                 },
                 "0.95": {
-                  "n": 17,
+                  "n": 11,
                   "errors": 0,
                   "error_rate": 0.0
                 },
@@ -8657,32 +8657,32 @@
               },
               "selective": {
                 "0.5": {
-                  "coverage": 0.53125,
+                  "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9517537497245334
+                  "confidence_cutoff": 0.9449616248177756
                 },
                 "0.8": {
                   "coverage": 0.8125,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.8932077222078463
+                  "confidence_cutoff": 0.8779250187145542
                 }
               }
             },
             "composition_final_exception": {
               "n": 32,
-              "nll": 0.24646259109043608,
+              "nll": 0.24455502641087398,
               "acc": 0.90625,
-              "ece": 0.08165686202673392,
-              "brier": 0.13219691386769145,
-              "mean_conf": 0.9344170796872766,
+              "ece": 0.079792014916701,
+              "brier": 0.13577654408819648,
+              "mean_conf": 0.9357865299742769,
               "confident_error_rate": 0.03125,
               "coverage_at_0_9": 0.75,
               "accuracy_at_0_9": 0.9583333333333334,
               "coverage_at_5pct_error": 0.9375,
               "coverage_at_1pct_error": 0.375,
-              "aurc": 0.033345170986070205,
+              "aurc": 0.033363772176546395,
               "error_rate_at_0_9": 0.041666666666666664,
-              "confidence_bias": 0.02816707968727661,
+              "confidence_bias": 0.029536529974276915,
               "top_bins": {
                 "0.9": {
                   "n": 24,
@@ -8695,7 +8695,7 @@
                   "error_rate": 0.05555555555555555
                 },
                 "0.99": {
-                  "n": 6,
+                  "n": 5,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -8704,22 +8704,22 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.9375,
-                  "confidence_cutoff": 0.9769813800539988
+                  "confidence_cutoff": 0.9711615479408262
                 },
                 "0.8": {
                   "coverage": 0.8125,
                   "accuracy": 0.9615384615384616,
-                  "confidence_cutoff": 0.8745144680431802
+                  "confidence_cutoff": 0.8868939148352457
                 }
               }
             },
             "composition_final_negation": {
               "n": 32,
-              "nll": 0.17366660442192522,
+              "nll": 0.1799321903230246,
               "acc": 0.96875,
-              "ece": 0.139280642091471,
-              "brier": 0.09349301051833991,
-              "mean_conf": 0.8684549516140397,
+              "ece": 0.10182879137977313,
+              "brier": 0.09774414276889377,
+              "mean_conf": 0.8675840743934221,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.5625,
               "accuracy_at_0_9": 1.0,
@@ -8727,7 +8727,7 @@
               "coverage_at_1pct_error": 0.875,
               "aurc": 0.004072377873563219,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.10029504838596026,
+              "confidence_bias": -0.10116592560657789,
               "top_bins": {
                 "0.9": {
                   "n": 18,
@@ -8740,7 +8740,7 @@
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 5,
+                  "n": 4,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -8749,22 +8749,22 @@
                 "0.5": {
                   "coverage": 0.53125,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9398574478142043
+                  "confidence_cutoff": 0.9276751556299673
                 },
                 "0.8": {
                   "coverage": 0.8125,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.7353225569523143
+                  "confidence_cutoff": 0.7040075273516038
                 }
               }
             },
             "contrastive_authorization": {
               "n": 40,
-              "nll": 0.010156044928884202,
+              "nll": 0.012461149245409892,
               "acc": 1.0,
-              "ece": 0.010082175827252016,
-              "brier": 0.0002922128455344832,
-              "mean_conf": 0.989917824172748,
+              "ece": 0.012352988030602807,
+              "brier": 0.0004269628768902182,
+              "mean_conf": 0.9876470119693972,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -8772,7 +8772,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.010082175827252016,
+              "confidence_bias": -0.012352988030602807,
               "top_bins": {
                 "0.9": {
                   "n": 40,
@@ -8794,43 +8794,43 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.994282735900115
+                  "confidence_cutoff": 0.9926770167945137
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9830883529872735
+                  "confidence_cutoff": 0.9794385287249308
                 }
               }
             },
             "contrastive_deadline": {
               "n": 40,
-              "nll": 0.2578333370423695,
+              "nll": 0.26407278866821826,
               "acc": 0.9,
-              "ece": 0.08339734125029008,
-              "brier": 0.1539668312012304,
-              "mean_conf": 0.8928125003966263,
+              "ece": 0.05245202539688848,
+              "brier": 0.15778553734614284,
+              "mean_conf": 0.8829211192449822,
               "confident_error_rate": 0.0,
-              "coverage_at_0_9": 0.6,
+              "coverage_at_0_9": 0.575,
               "accuracy_at_0_9": 1.0,
-              "coverage_at_5pct_error": 0.7,
+              "coverage_at_5pct_error": 0.725,
               "coverage_at_1pct_error": 0.65,
-              "aurc": 0.02394744920629697,
+              "aurc": 0.02308538024077973,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.007187499603373726,
+              "confidence_bias": -0.017078880755017845,
               "top_bins": {
                 "0.9": {
-                  "n": 24,
+                  "n": 23,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.95": {
-                  "n": 16,
+                  "n": 15,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 8,
+                  "n": 5,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -8839,77 +8839,77 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9320149317531109
+                  "confidence_cutoff": 0.9200535815570202
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.9375,
-                  "confidence_cutoff": 0.8366438894497547
+                  "confidence_cutoff": 0.8251958593609416
                 }
               },
-              "score_mae": 0.16359197313913096,
-              "ranked_probability_score": 0.03911601810015416
+              "score_mae": 0.17291673608415475,
+              "ranked_probability_score": 0.04024927146404126
             },
             "emotion": {
               "n": 80,
-              "nll": 1.3250366616414988,
-              "acc": 0.6125,
-              "ece": 0.2033677770446442,
-              "brier": 0.578098657406022,
-              "mean_conf": 0.7837558603474908,
-              "confident_error_rate": 0.075,
-              "coverage_at_0_9": 0.45,
-              "accuracy_at_0_9": 0.8333333333333334,
+              "nll": 1.2865095689254165,
+              "acc": 0.6,
+              "ece": 0.19167859824830136,
+              "brier": 0.5660599536581878,
+              "mean_conf": 0.7759389293425664,
+              "confident_error_rate": 0.0625,
+              "coverage_at_0_9": 0.4,
+              "accuracy_at_0_9": 0.84375,
               "coverage_at_5pct_error": 0.025,
               "coverage_at_1pct_error": 0.025,
-              "aurc": 0.2465828605574525,
-              "error_rate_at_0_9": 0.16666666666666666,
-              "confidence_bias": 0.17125586034749074,
+              "aurc": 0.24689075981896816,
+              "error_rate_at_0_9": 0.15625,
+              "confidence_bias": 0.17593892934256639,
               "top_bins": {
                 "0.9": {
-                  "n": 36,
-                  "errors": 6,
-                  "error_rate": 0.16666666666666666
+                  "n": 32,
+                  "errors": 5,
+                  "error_rate": 0.15625
                 },
                 "0.95": {
-                  "n": 24,
+                  "n": 22,
                   "errors": 3,
-                  "error_rate": 0.125
+                  "error_rate": 0.13636363636363635
                 },
                 "0.99": {
-                  "n": 7,
+                  "n": 5,
                   "errors": 2,
-                  "error_rate": 0.2857142857142857
+                  "error_rate": 0.4
                 }
               },
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.8,
-                  "confidence_cutoff": 0.8534444506954815
+                  "confidence_cutoff": 0.8358595503583666
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.65625,
-                  "confidence_cutoff": 0.5796189437721434
+                  "accuracy": 0.671875,
+                  "confidence_cutoff": 0.5660683528503451
                 }
               }
             },
             "mmlu": {
               "n": 80,
-              "nll": 0.6466641422955599,
-              "acc": 0.7375,
-              "ece": 0.10575227809040644,
-              "brier": 0.3381993704155934,
-              "mean_conf": 0.8239887221865458,
+              "nll": 0.64319962297488,
+              "acc": 0.75,
+              "ece": 0.07583444934027661,
+              "brier": 0.336846797840859,
+              "mean_conf": 0.815404454455512,
               "confident_error_rate": 0.025,
               "coverage_at_0_9": 0.5125,
               "accuracy_at_0_9": 0.9512195121951219,
-              "coverage_at_5pct_error": 0.5875,
-              "coverage_at_1pct_error": 0.4125,
-              "aurc": 0.07075415004878534,
+              "coverage_at_5pct_error": 0.575,
+              "coverage_at_1pct_error": 0.425,
+              "aurc": 0.06990146624747551,
               "error_rate_at_0_9": 0.04878048780487805,
-              "confidence_bias": 0.0864887221865458,
+              "confidence_bias": 0.06540445445551202,
               "top_bins": {
                 "0.9": {
                   "n": 41,
@@ -8917,12 +8917,12 @@
                   "error_rate": 0.04878048780487805
                 },
                 "0.95": {
-                  "n": 31,
+                  "n": 29,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 9,
+                  "n": 6,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -8931,43 +8931,43 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.95,
-                  "confidence_cutoff": 0.9179506124738418
+                  "confidence_cutoff": 0.9107307477881781
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.859375,
-                  "confidence_cutoff": 0.6428546144492293
+                  "confidence_cutoff": 0.631165847975779
                 }
               }
             },
             "mmlu_pro": {
               "n": 200,
-              "nll": 1.3111522675312284,
+              "nll": 1.3020212799561357,
               "acc": 0.53,
-              "ece": 0.08739805659958974,
-              "brier": 0.5728987396461885,
-              "mean_conf": 0.5899912710608897,
+              "ece": 0.057886156442029546,
+              "brier": 0.5699198585015737,
+              "mean_conf": 0.5796763608829093,
               "confident_error_rate": 0.005,
-              "coverage_at_0_9": 0.23,
-              "accuracy_at_0_9": 0.9782608695652174,
-              "coverage_at_5pct_error": 0.25,
-              "coverage_at_1pct_error": 0.19,
-              "aurc": 0.21980746104696547,
-              "error_rate_at_0_9": 0.021739130434782608,
-              "confidence_bias": 0.05999127106088964,
+              "coverage_at_0_9": 0.21,
+              "accuracy_at_0_9": 0.9761904761904762,
+              "coverage_at_5pct_error": 0.255,
+              "coverage_at_1pct_error": 0.185,
+              "aurc": 0.22101453807685856,
+              "error_rate_at_0_9": 0.023809523809523808,
+              "confidence_bias": 0.049676360882909276,
               "top_bins": {
                 "0.9": {
-                  "n": 46,
+                  "n": 42,
                   "errors": 1,
-                  "error_rate": 0.021739130434782608
+                  "error_rate": 0.023809523809523808
                 },
                 "0.95": {
-                  "n": 29,
+                  "n": 28,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 6,
+                  "n": 5,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -8975,113 +8975,113 @@
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
-                  "accuracy": 0.75,
-                  "confidence_cutoff": 0.57725688415424
+                  "accuracy": 0.76,
+                  "confidence_cutoff": 0.5622010178659016
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.625,
-                  "confidence_cutoff": 0.26950159367730925
+                  "accuracy": 0.6125,
+                  "confidence_cutoff": 0.28172954890715934
                 }
               }
             },
             "paws": {
               "n": 80,
-              "nll": 0.28314251837212323,
-              "acc": 0.9,
-              "ece": 0.06247439765758463,
-              "brier": 0.15885879977500503,
-              "mean_conf": 0.9412949163036171,
-              "confident_error_rate": 0.0625,
-              "coverage_at_0_9": 0.8375,
-              "accuracy_at_0_9": 0.9253731343283582,
-              "coverage_at_5pct_error": 0.775,
-              "coverage_at_1pct_error": 0.2875,
-              "aurc": 0.030912620564059478,
-              "error_rate_at_0_9": 0.07462686567164178,
-              "confidence_bias": 0.041294916303617035,
+              "nll": 0.2779227201992138,
+              "acc": 0.9125,
+              "ece": 0.05493591159180497,
+              "brier": 0.15753943063635653,
+              "mean_conf": 0.9357409581036927,
+              "confident_error_rate": 0.05,
+              "coverage_at_0_9": 0.8125,
+              "accuracy_at_0_9": 0.9384615384615385,
+              "coverage_at_5pct_error": 0.7625,
+              "coverage_at_1pct_error": 0.3,
+              "aurc": 0.0308044058290765,
+              "error_rate_at_0_9": 0.06153846153846154,
+              "confidence_bias": 0.023240958103692755,
               "top_bins": {
                 "0.9": {
-                  "n": 67,
-                  "errors": 5,
-                  "error_rate": 0.07462686567164178
+                  "n": 65,
+                  "errors": 4,
+                  "error_rate": 0.06153846153846154
                 },
                 "0.95": {
-                  "n": 60,
+                  "n": 58,
                   "errors": 2,
-                  "error_rate": 0.03333333333333333
+                  "error_rate": 0.034482758620689655
                 },
                 "0.99": {
-                  "n": 2,
+                  "n": 0,
                   "errors": 0,
-                  "error_rate": 0.0
+                  "error_rate": null
                 }
               },
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.975,
-                  "confidence_cutoff": 0.9773598813644135
+                  "confidence_cutoff": 0.9736310261741858
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.9375,
-                  "confidence_cutoff": 0.9395250222016129
+                  "confidence_cutoff": 0.9301026006058246
                 }
               }
             },
             "qnli": {
               "n": 80,
-              "nll": 0.2161076588398619,
+              "nll": 0.21231177025039338,
               "acc": 0.9125,
-              "ece": 0.038010106954782505,
-              "brier": 0.11745397306383834,
-              "mean_conf": 0.9376866478309165,
+              "ece": 0.026039786769557725,
+              "brier": 0.11561259420716938,
+              "mean_conf": 0.9322130681411511,
               "confident_error_rate": 0.025,
-              "coverage_at_0_9": 0.8375,
-              "accuracy_at_0_9": 0.9701492537313433,
+              "coverage_at_0_9": 0.825,
+              "accuracy_at_0_9": 0.9696969696969697,
               "coverage_at_5pct_error": 0.925,
-              "coverage_at_1pct_error": 0.075,
-              "aurc": 0.039172223012800614,
-              "error_rate_at_0_9": 0.029850746268656716,
-              "confidence_bias": 0.02518664783091651,
+              "coverage_at_1pct_error": 0.1,
+              "aurc": 0.03569781457299058,
+              "error_rate_at_0_9": 0.030303030303030304,
+              "confidence_bias": 0.019713068141151147,
               "top_bins": {
                 "0.9": {
-                  "n": 67,
+                  "n": 66,
                   "errors": 2,
-                  "error_rate": 0.029850746268656716
+                  "error_rate": 0.030303030303030304
                 },
                 "0.95": {
-                  "n": 62,
-                  "errors": 2,
-                  "error_rate": 0.03225806451612903
+                  "n": 61,
+                  "errors": 1,
+                  "error_rate": 0.01639344262295082
                 },
                 "0.99": {
-                  "n": 26,
+                  "n": 14,
                   "errors": 1,
-                  "error_rate": 0.038461538461538464
+                  "error_rate": 0.07142857142857142
                 }
               },
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.975,
-                  "confidence_cutoff": 0.9851611733370504
+                  "confidence_cutoff": 0.9814442414444764
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.96875,
-                  "confidence_cutoff": 0.9458952863255798
+                  "confidence_cutoff": 0.9395185848998813
                 }
               }
             },
             "sciq": {
               "n": 80,
-              "nll": 0.006236348098711358,
+              "nll": 0.007801897547958209,
               "acc": 1.0,
-              "ece": 0.006149372309216639,
-              "brier": 0.00030747770792780715,
-              "mean_conf": 0.9938506276907834,
+              "ece": 0.007676312134330843,
+              "brier": 0.0004369807984377848,
+              "mean_conf": 0.9923236878656692,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -9089,7 +9089,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.006149372309216639,
+              "confidence_bias": -0.007676312134330843,
               "top_bins": {
                 "0.9": {
                   "n": 80,
@@ -9097,12 +9097,12 @@
                   "error_rate": 0.0
                 },
                 "0.95": {
-                  "n": 79,
+                  "n": 78,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 69,
+                  "n": 65,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -9111,30 +9111,30 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9978451209130044
+                  "confidence_cutoff": 0.9965300683221596
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9938674921230647
+                  "confidence_cutoff": 0.9921908745945206
                 }
               }
             },
             "tweet_offensive": {
               "n": 80,
-              "nll": 0.47556818540480367,
+              "nll": 0.4720489807336611,
               "acc": 0.75,
-              "ece": 0.07643617030229378,
-              "brier": 0.3199264336587134,
-              "mean_conf": 0.8258139805835978,
+              "ece": 0.0761936963725907,
+              "brier": 0.3177754348396661,
+              "mean_conf": 0.8169291651727064,
               "confident_error_rate": 0.0375,
               "coverage_at_0_9": 0.4125,
               "accuracy_at_0_9": 0.9090909090909091,
               "coverage_at_5pct_error": 0.3625,
-              "coverage_at_1pct_error": 0.35,
-              "aurc": 0.09222008593150528,
+              "coverage_at_1pct_error": 0.3375,
+              "aurc": 0.09282282377572246,
               "error_rate_at_0_9": 0.09090909090909091,
-              "confidence_bias": 0.07581398058359778,
+              "confidence_bias": 0.06692916517270642,
               "top_bins": {
                 "0.9": {
                   "n": 33,
@@ -9142,7 +9142,7 @@
                   "error_rate": 0.09090909090909091
                 },
                 "0.95": {
-                  "n": 24,
+                  "n": 21,
                   "errors": 0,
                   "error_rate": 0.0
                 },
@@ -9155,23 +9155,23 @@
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
-                  "accuracy": 0.9,
-                  "confidence_cutoff": 0.8609209785002616
+                  "accuracy": 0.925,
+                  "confidence_cutoff": 0.8560822733652931
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.828125,
-                  "confidence_cutoff": 0.6948713064193726
+                  "confidence_cutoff": 0.6797098123940769
                 }
               }
             },
             "unknowable_age_eligibility": {
               "n": 10,
-              "nll": 0.8382027962956439,
+              "nll": 0.8159140816405721,
               "acc": 0.5,
-              "ece": 0.23136963355477053,
-              "brier": 0.6189159675554958,
-              "mean_conf": 0.7313696335547706,
+              "ece": 0.21421574323774287,
+              "brier": 0.6032897524261119,
+              "mean_conf": 0.7142157432377428,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.0,
               "accuracy_at_0_9": null,
@@ -9179,7 +9179,7 @@
               "coverage_at_1pct_error": 0.0,
               "aurc": 0.5,
               "error_rate_at_0_9": null,
-              "confidence_bias": 0.23136963355477058,
+              "confidence_bias": 0.21421574323774284,
               "top_bins": {
                 "0.9": {
                   "n": 0,
@@ -9201,35 +9201,35 @@
                 "0.5": {
                   "coverage": 0.6,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.7283886075019836
+                  "confidence_cutoff": 0.7100501272097268
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.6479771137237549
+                  "confidence_cutoff": 0.6407661437988281
                 }
               }
             },
             "unknowable_authorization": {
               "n": 10,
-              "nll": 1.1053561068085782,
+              "nll": 1.0676248278306069,
               "acc": 0.5,
-              "ece": 0.3656208440260864,
-              "brier": 0.7581604276839656,
-              "mean_conf": 0.8656208440260865,
-              "confident_error_rate": 0.3,
-              "coverage_at_0_9": 0.6,
-              "accuracy_at_0_9": 0.5,
+              "ece": 0.35543382596706624,
+              "brier": 0.7452622091449328,
+              "mean_conf": 0.8554338259670662,
+              "confident_error_rate": 0.2,
+              "coverage_at_0_9": 0.5,
+              "accuracy_at_0_9": 0.6,
               "coverage_at_5pct_error": 0.2,
               "coverage_at_1pct_error": 0.2,
-              "aurc": 0.32313492063492066,
-              "error_rate_at_0_9": 0.5,
-              "confidence_bias": 0.3656208440260865,
+              "aurc": 0.3481349206349207,
+              "error_rate_at_0_9": 0.4,
+              "confidence_bias": 0.3554338259670662,
               "top_bins": {
                 "0.9": {
-                  "n": 6,
-                  "errors": 3,
-                  "error_rate": 0.5
+                  "n": 5,
+                  "errors": 2,
+                  "error_rate": 0.4
                 },
                 "0.95": {
                   "n": 0,
@@ -9246,22 +9246,22 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.6,
-                  "confidence_cutoff": 0.9161812663078308
+                  "confidence_cutoff": 0.9051045381673695
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.625,
-                  "confidence_cutoff": 0.7894018645317196
+                  "confidence_cutoff": 0.7762378934586363
                 }
               }
             },
             "unknowable_control_age_eligibility": {
               "n": 10,
-              "nll": 0.007841155789564139,
+              "nll": 0.01010806429581184,
               "acc": 1.0,
-              "ece": 0.007806247293806301,
-              "brier": 0.00013872505147745316,
-              "mean_conf": 0.9921937527061937,
+              "ece": 0.010050520182894429,
+              "brier": 0.00022828427851469854,
+              "mean_conf": 0.9899494798171056,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -9269,7 +9269,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.007806247293806301,
+              "confidence_bias": -0.010050520182894429,
               "top_bins": {
                 "0.9": {
                   "n": 10,
@@ -9282,7 +9282,7 @@
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 6,
+                  "n": 5,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -9291,22 +9291,22 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9939010793872055
+                  "confidence_cutoff": 0.9920052497980926
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9896593136903282
+                  "confidence_cutoff": 0.9867894292404178
                 }
               }
             },
             "unknowable_control_authorization": {
               "n": 10,
-              "nll": 0.0107929267557592,
+              "nll": 0.013336190464884707,
               "acc": 1.0,
-              "ece": 0.010701462639340464,
-              "brier": 0.00036120660545170313,
-              "mean_conf": 0.9892985373606595,
+              "ece": 0.013199482511126015,
+              "brier": 0.0005383885458560314,
+              "mean_conf": 0.986800517488874,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -9314,7 +9314,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.010701462639340464,
+              "confidence_bias": -0.013199482511126015,
               "top_bins": {
                 "0.9": {
                   "n": 10,
@@ -9336,22 +9336,22 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9969034000449554
+                  "confidence_cutoff": 0.9957628189842604
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9816944573491596
+                  "confidence_cutoff": 0.9774085096482822
                 }
               }
             },
             "unknowable_control_deadline": {
               "n": 10,
-              "nll": 0.2138260506584651,
+              "nll": 0.22694873211843464,
               "acc": 0.8,
-              "ece": 0.15741611133328517,
-              "brier": 0.1276425479499806,
-              "mean_conf": 0.856580016184909,
+              "ece": 0.16555386795989538,
+              "brier": 0.13550345474895487,
+              "mean_conf": 0.8535695538437231,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.6,
               "accuracy_at_0_9": 1.0,
@@ -9359,7 +9359,7 @@
               "coverage_at_1pct_error": 0.8,
               "aurc": 0.03111111111111111,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": 0.05658001618490893,
+              "confidence_bias": 0.05356955384372308,
               "top_bins": {
                 "0.9": {
                   "n": 6,
@@ -9367,12 +9367,12 @@
                   "error_rate": 0.0
                 },
                 "0.95": {
-                  "n": 4,
+                  "n": 3,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 2,
+                  "n": 1,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -9381,24 +9381,24 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9426117959877142
+                  "confidence_cutoff": 0.9367303412067545
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.8415699130530643
+                  "confidence_cutoff": 0.8305143644338201
                 }
               },
-              "score_mae": 0.16071754616245987,
-              "ranked_probability_score": 0.03275971187271286
+              "score_mae": 0.16955440723819443,
+              "ranked_probability_score": 0.034895014874681045
             },
             "unknowable_control_late_fee": {
               "n": 10,
-              "nll": 0.07005694097555375,
+              "nll": 0.061294700563837144,
               "acc": 1.0,
-              "ece": 0.0540813816797954,
-              "brier": 0.04110395968042234,
-              "mean_conf": 0.9459186183202046,
+              "ece": 0.04980564296050656,
+              "brier": 0.03092495951104666,
+              "mean_conf": 0.9501943570394934,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.9,
               "accuracy_at_0_9": 1.0,
@@ -9406,7 +9406,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.05408138167979537,
+              "confidence_bias": -0.0498056429605066,
               "top_bins": {
                 "0.9": {
                   "n": 9,
@@ -9419,7 +9419,7 @@
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 6,
+                  "n": 5,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -9428,35 +9428,35 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.993819476814992
+                  "confidence_cutoff": 0.9916381797053619
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9798565648382314
+                  "confidence_cutoff": 0.9777806624356902
                 }
               },
-              "score_mae": 0.0487152458265909,
-              "ranked_probability_score": 0.00975869637261514
+              "score_mae": 0.04333117262198813,
+              "ranked_probability_score": 0.007196645402378658
             },
             "unknowable_control_quantity_limit": {
               "n": 10,
-              "nll": 0.10819644087102691,
+              "nll": 0.12423624782442941,
               "acc": 1.0,
-              "ece": 0.0954442527194344,
-              "brier": 0.039610537719924295,
-              "mean_conf": 0.9045557472805654,
+              "ece": 0.10880820844844666,
+              "brier": 0.04779494768609947,
+              "mean_conf": 0.8911917915515533,
               "confident_error_rate": 0.0,
-              "coverage_at_0_9": 0.7,
+              "coverage_at_0_9": 0.6,
               "accuracy_at_0_9": 1.0,
               "coverage_at_5pct_error": 1.0,
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.09544425271943457,
+              "confidence_bias": -0.10880820844844674,
               "top_bins": {
                 "0.9": {
-                  "n": 7,
+                  "n": 6,
                   "errors": 0,
                   "error_rate": 0.0
                 },
@@ -9475,22 +9475,22 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9656626177441163
+                  "confidence_cutoff": 0.9558210808739257
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.881241046212756
+                  "confidence_cutoff": 0.8866819137686461
                 }
               }
             },
             "unknowable_control_return_window": {
               "n": 10,
-              "nll": 0.8939128624245644,
+              "nll": 0.8828182838634889,
               "acc": 0.6,
-              "ece": 0.39838756334240605,
-              "brier": 0.6386225586932958,
-              "mean_conf": 0.8386086717206818,
+              "ece": 0.3920235500266451,
+              "brier": 0.6280874258674091,
+              "mean_conf": 0.8410658551406985,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.2,
               "accuracy_at_0_9": 1.0,
@@ -9498,7 +9498,7 @@
               "coverage_at_1pct_error": 0.2,
               "aurc": 0.288968253968254,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": 0.2386086717206818,
+              "confidence_bias": 0.24106585514069856,
               "top_bins": {
                 "0.9": {
                   "n": 2,
@@ -9520,22 +9520,22 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.8,
-                  "confidence_cutoff": 0.8853410746258817
+                  "confidence_cutoff": 0.8732116352500265
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.8282266608409089
+                  "confidence_cutoff": 0.8179686895920922
                 }
               }
             },
             "unknowable_control_shipping_delay": {
               "n": 10,
-              "nll": 0.3878790410817737,
+              "nll": 0.38549807963539046,
               "acc": 0.9,
-              "ece": 0.20366310890419706,
-              "brier": 0.23894788485153878,
-              "mean_conf": 0.8019263309455292,
+              "ece": 0.20288794178646427,
+              "brier": 0.23489325574634798,
+              "mean_conf": 0.8015680467880623,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.3,
               "accuracy_at_0_9": 1.0,
@@ -9543,7 +9543,7 @@
               "coverage_at_1pct_error": 0.5,
               "aurc": 0.06456349206349207,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.09807366905447079,
+              "confidence_bias": -0.09843195321193776,
               "top_bins": {
                 "0.9": {
                   "n": 3,
@@ -9565,24 +9565,24 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.8513163369222139
+                  "confidence_cutoff": 0.8456577690156466
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.875,
-                  "confidence_cutoff": 0.7305872644461858
+                  "confidence_cutoff": 0.6947201852727033
                 }
               },
-              "score_mae": 0.2577528085044144,
-              "ranked_probability_score": 0.05968224629533503
+              "score_mae": 0.2564750859760038,
+              "ranked_probability_score": 0.05882371330201126
             },
             "unknowable_control_sla_response": {
               "n": 10,
-              "nll": 0.7470384623310196,
+              "nll": 0.7292834955292916,
               "acc": 0.7,
-              "ece": 0.22029693644423792,
-              "brier": 0.4309736445646065,
-              "mean_conf": 0.9202969364442379,
+              "ece": 0.21088673885292936,
+              "brier": 0.4247531105085889,
+              "mean_conf": 0.9108867388529293,
               "confident_error_rate": 0.1,
               "coverage_at_0_9": 0.7,
               "accuracy_at_0_9": 0.8571428571428571,
@@ -9590,7 +9590,7 @@
               "coverage_at_1pct_error": 0.4,
               "aurc": 0.12817460317460316,
               "error_rate_at_0_9": 0.14285714285714285,
-              "confidence_bias": 0.22029693644423798,
+              "confidence_bias": 0.2108867388529293,
               "top_bins": {
                 "0.9": {
                   "n": 7,
@@ -9612,24 +9612,24 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.8,
-                  "confidence_cutoff": 0.9773512719149
+                  "confidence_cutoff": 0.9757163386049921
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.75,
-                  "confidence_cutoff": 0.8938853735717587
+                  "confidence_cutoff": 0.8745092806607159
                 }
               },
-              "score_mae": 0.2732574139836782,
-              "ranked_probability_score": 0.10706854764238136
+              "score_mae": 0.27669455495743667,
+              "ranked_probability_score": 0.1053871341351588
             },
             "unknowable_control_spend_threshold": {
               "n": 10,
-              "nll": 0.004226216609183238,
+              "nll": 0.005872511904691692,
               "acc": 1.0,
-              "ece": 0.004215976897349094,
-              "brier": 3.4649667131696104e-05,
-              "mean_conf": 0.9957840231026509,
+              "ece": 0.005852885976918243,
+              "brier": 6.553204955281063e-05,
+              "mean_conf": 0.9941471140230818,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -9637,7 +9637,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.004215976897349094,
+              "confidence_bias": -0.005852885976918243,
               "top_bins": {
                 "0.9": {
                   "n": 10,
@@ -9659,22 +9659,22 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9969052065210677
+                  "confidence_cutoff": 0.9956017537902697
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9942030038674581
+                  "confidence_cutoff": 0.992077102727452
                 }
               }
             },
             "unknowable_control_volume_discount": {
               "n": 10,
-              "nll": 0.007053870282788843,
+              "nll": 0.009442501422597043,
               "acc": 1.0,
-              "ece": 0.007024441570579243,
-              "brier": 9.776258511132903e-05,
-              "mean_conf": 0.9929755584294208,
+              "ece": 0.009390689401267194,
+              "brier": 0.00017066799723336928,
+              "mean_conf": 0.9906093105987328,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -9682,7 +9682,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.007024441570579243,
+              "confidence_bias": -0.009390689401267194,
               "top_bins": {
                 "0.9": {
                   "n": 10,
@@ -9704,24 +9704,24 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9932447768419814
+                  "confidence_cutoff": 0.9912854619430778
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9898739137651561
+                  "confidence_cutoff": 0.9857135990468268
                 }
               },
-              "score_mae": 0.007660911551517352,
-              "ranked_probability_score": 2.757223086398964e-05
+              "score_mae": 0.010231745432055298,
+              "ranked_probability_score": 4.863796909247521e-05
             },
             "unknowable_control_warranty_claim": {
               "n": 10,
-              "nll": 0.9724116266401464,
+              "nll": 0.9490781913515873,
               "acc": 0.5,
-              "ece": 0.37317330379992186,
-              "brier": 0.592356663184495,
-              "mean_conf": 0.7285382769948201,
+              "ece": 0.36923739486967955,
+              "brier": 0.5785088804997336,
+              "mean_conf": 0.710935585802415,
               "confident_error_rate": 0.1,
               "coverage_at_0_9": 0.2,
               "accuracy_at_0_9": 0.5,
@@ -9729,7 +9729,7 @@
               "coverage_at_1pct_error": 0.1,
               "aurc": 0.356468253968254,
               "error_rate_at_0_9": 0.5,
-              "confidence_bias": 0.2285382769948201,
+              "confidence_bias": 0.210935585802415,
               "top_bins": {
                 "0.9": {
                   "n": 2,
@@ -9737,9 +9737,9 @@
                   "error_rate": 0.5
                 },
                 "0.95": {
-                  "n": 2,
-                  "errors": 1,
-                  "error_rate": 0.5
+                  "n": 1,
+                  "errors": 0,
+                  "error_rate": 0.0
                 },
                 "0.99": {
                   "n": 0,
@@ -9751,32 +9751,32 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.6,
-                  "confidence_cutoff": 0.7507937755021719
+                  "confidence_cutoff": 0.6925948236413493
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.625,
-                  "confidence_cutoff": 0.597999206215155
+                  "confidence_cutoff": 0.586704671382904
                 }
               },
-              "score_mae": 0.41248856844465875,
-              "ranked_probability_score": 0.14487339386062986
+              "score_mae": 0.41230806689600685,
+              "ranked_probability_score": 0.14147304522762058
             },
             "unknowable_deadline": {
               "n": 10,
-              "nll": 1.2106218946046243,
+              "nll": 1.1936892937872634,
               "acc": 0.3,
-              "ece": 0.17861993312941266,
-              "brier": 0.7343701548653422,
-              "mean_conf": 0.47861993312941264,
+              "ece": 0.17490025011213214,
+              "brier": 0.7253276456261111,
+              "mean_conf": 0.4749002501121321,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.0,
               "accuracy_at_0_9": null,
               "coverage_at_5pct_error": 0.0,
               "coverage_at_1pct_error": 0.0,
-              "aurc": 0.7779761904761905,
+              "aurc": 0.7589285714285715,
               "error_rate_at_0_9": null,
-              "confidence_bias": 0.17861993312941266,
+              "confidence_bias": 0.17490025011213212,
               "top_bins": {
                 "0.9": {
                   "n": 0,
@@ -9798,32 +9798,32 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.2,
-                  "confidence_cutoff": 0.49840795241629865
+                  "confidence_cutoff": 0.4989907294211009
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.375,
-                  "confidence_cutoff": 0.4385092854499817
+                  "confidence_cutoff": 0.4357915260916039
                 }
               },
-              "score_mae": 0.7449380524877681,
-              "ranked_probability_score": 0.24553973235698762
+              "score_mae": 0.7352555113184743,
+              "ranked_probability_score": 0.2420866592785548
             },
             "unknowable_late_fee": {
               "n": 10,
-              "nll": 1.3276710177178344,
+              "nll": 1.2923453860166598,
               "acc": 0.3,
-              "ece": 0.27106129862072126,
-              "brier": 0.8180838268143074,
-              "mean_conf": 0.5710612986207213,
+              "ece": 0.23038840334765942,
+              "brier": 0.7937115661256419,
+              "mean_conf": 0.5303884033476594,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.0,
               "accuracy_at_0_9": null,
               "coverage_at_5pct_error": 0.0,
               "coverage_at_1pct_error": 0.0,
-              "aurc": 0.7483333333333333,
+              "aurc": 0.7733333333333332,
               "error_rate_at_0_9": null,
-              "confidence_bias": 0.2710612986207213,
+              "confidence_bias": 0.23038840334765937,
               "top_bins": {
                 "0.9": {
                   "n": 0,
@@ -9845,24 +9845,24 @@
                 "0.5": {
                   "coverage": 0.6,
                   "accuracy": 0.3333333333333333,
-                  "confidence_cutoff": 0.550855944974189
+                  "confidence_cutoff": 0.5093778740147706
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.375,
-                  "confidence_cutoff": 0.4580442284371989
+                  "accuracy": 0.25,
+                  "confidence_cutoff": 0.44214211890736754
                 }
               },
-              "score_mae": 0.7681052591130838,
-              "ranked_probability_score": 0.2644686148936013
+              "score_mae": 0.7422305550581385,
+              "ranked_probability_score": 0.25270907837522644
             },
             "unknowable_quantity_limit": {
               "n": 10,
-              "nll": 2.4681683362881595,
+              "nll": 2.364100794867005,
               "acc": 0.5,
-              "ece": 0.48573829809507685,
-              "brier": 0.9783875577934342,
-              "mean_conf": 0.9857382980950768,
+              "ece": 0.4827107892011071,
+              "brier": 0.9737439215308642,
+              "mean_conf": 0.9827107892011071,
               "confident_error_rate": 0.5,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 0.5,
@@ -9870,7 +9870,7 @@
               "coverage_at_1pct_error": 0.0,
               "aurc": 0.5,
               "error_rate_at_0_9": 0.5,
-              "confidence_bias": 0.48573829809507685,
+              "confidence_bias": 0.4827107892011071,
               "top_bins": {
                 "0.9": {
                   "n": 10,
@@ -9892,30 +9892,30 @@
                 "0.5": {
                   "coverage": 0.6,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.9864682188832252
+                  "confidence_cutoff": 0.9826534113523934
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.9834588178510609
+                  "confidence_cutoff": 0.98013697658372
                 }
               }
             },
             "unknowable_return_window": {
               "n": 10,
-              "nll": 0.7972145494828772,
-              "acc": 0.6,
-              "ece": 0.18620925667194407,
-              "brier": 0.5676903556290133,
-              "mean_conf": 0.5894333514724946,
+              "nll": 0.7914668987099229,
+              "acc": 0.7,
+              "ece": 0.286885017017804,
+              "brier": 0.5690036879176795,
+              "mean_conf": 0.585196890317801,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.0,
               "accuracy_at_0_9": null,
               "coverage_at_5pct_error": 0.0,
               "coverage_at_1pct_error": 0.0,
-              "aurc": 0.48813492063492064,
+              "aurc": 0.5203571428571429,
               "error_rate_at_0_9": null,
-              "confidence_bias": -0.010566648527505329,
+              "confidence_bias": -0.11480310968219898,
               "top_bins": {
                 "0.9": {
                   "n": 0,
@@ -9936,23 +9936,23 @@
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
-                  "accuracy": 0.6,
-                  "confidence_cutoff": 0.5594538450241089
+                  "accuracy": 0.4,
+                  "confidence_cutoff": 0.5657328367233276
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.625,
-                  "confidence_cutoff": 0.5314476331637015
+                  "confidence_cutoff": 0.5336925983428955
                 }
               }
             },
             "unknowable_shipping_delay": {
               "n": 10,
-              "nll": 1.3036386445451955,
+              "nll": 1.2682712288665416,
               "acc": 0.2,
-              "ece": 0.30095968800211725,
-              "brier": 0.8031605478655157,
-              "mean_conf": 0.4567572303779661,
+              "ece": 0.29121892684207396,
+              "brier": 0.781447600806538,
+              "mean_conf": 0.4462795720835988,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.0,
               "accuracy_at_0_9": null,
@@ -9960,7 +9960,7 @@
               "coverage_at_1pct_error": 0.0,
               "aurc": 0.9309920634920636,
               "error_rate_at_0_9": null,
-              "confidence_bias": 0.2567572303779661,
+              "confidence_bias": 0.2462795720835988,
               "top_bins": {
                 "0.9": {
                   "n": 0,
@@ -9982,24 +9982,24 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.0,
-                  "confidence_cutoff": 0.4595779615514863
+                  "confidence_cutoff": 0.4459783434867859
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.125,
-                  "confidence_cutoff": 0.41920896115276335
+                  "confidence_cutoff": 0.4014878571033478
                 }
               },
-              "score_mae": 0.8655138350919522,
-              "ranked_probability_score": 0.28876623510920263
+              "score_mae": 0.8435188873044972,
+              "ranked_probability_score": 0.27948214430957724
             },
             "unknowable_sla_response": {
               "n": 10,
-              "nll": 1.2124338307190397,
+              "nll": 1.200518862016904,
               "acc": 0.2,
-              "ece": 0.2761916374516639,
-              "brier": 0.746030154235981,
-              "mean_conf": 0.47619163745166393,
+              "ece": 0.25713645324930595,
+              "brier": 0.7363382258751557,
+              "mean_conf": 0.45713645324930585,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.0,
               "accuracy_at_0_9": null,
@@ -10007,7 +10007,7 @@
               "coverage_at_1pct_error": 0.0,
               "aurc": 0.7266666666666668,
               "error_rate_at_0_9": null,
-              "confidence_bias": 0.2761916374516639,
+              "confidence_bias": 0.25713645324930584,
               "top_bins": {
                 "0.9": {
                   "n": 0,
@@ -10029,24 +10029,24 @@
                 "0.5": {
                   "coverage": 0.6,
                   "accuracy": 0.16666666666666666,
-                  "confidence_cutoff": 0.4481450452847792
+                  "confidence_cutoff": 0.4304346442222595
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.25,
-                  "confidence_cutoff": 0.4357074330957959
+                  "confidence_cutoff": 0.4151159699226627
                 }
               },
-              "score_mae": 0.6120337496191068,
-              "ranked_probability_score": 0.21905875267992797
+              "score_mae": 0.5909151001699179,
+              "ranked_probability_score": 0.21421444884056046
             },
             "unknowable_spend_threshold": {
               "n": 10,
-              "nll": 0.736875114520118,
+              "nll": 0.7380288693535819,
               "acc": 0.5,
-              "ece": 0.07921755937455173,
-              "brier": 0.5233519615225514,
-              "mean_conf": 0.5792175593745517,
+              "ece": 0.07006097788326784,
+              "brier": 0.520722008501961,
+              "mean_conf": 0.5699825910724938,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.0,
               "accuracy_at_0_9": null,
@@ -10054,7 +10054,7 @@
               "coverage_at_1pct_error": 0.0,
               "aurc": 0.5,
               "error_rate_at_0_9": null,
-              "confidence_bias": 0.07921755937455166,
+              "confidence_bias": 0.06998259107249383,
               "top_bins": {
                 "0.9": {
                   "n": 0,
@@ -10076,22 +10076,22 @@
                 "0.5": {
                   "coverage": 0.6,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.5943543182085235
+                  "confidence_cutoff": 0.5894361470891768
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.5139663834645092
+                  "confidence_cutoff": 0.502282899723621
                 }
               }
             },
             "unknowable_volume_discount": {
               "n": 10,
-              "nll": 2.5691700417991905,
+              "nll": 2.475799282190841,
               "acc": 0.4,
-              "ece": 0.5382559019399908,
-              "brier": 1.121603982108248,
-              "mean_conf": 0.9382559019399908,
+              "ece": 0.5247692963711266,
+              "brier": 1.1101745871995399,
+              "mean_conf": 0.9247692963711266,
               "confident_error_rate": 0.5,
               "coverage_at_0_9": 0.8,
               "accuracy_at_0_9": 0.375,
@@ -10099,7 +10099,7 @@
               "coverage_at_1pct_error": 0.0,
               "aurc": 0.5783333333333334,
               "error_rate_at_0_9": 0.625,
-              "confidence_bias": 0.5382559019399907,
+              "confidence_bias": 0.5247692963711266,
               "top_bins": {
                 "0.9": {
                   "n": 8,
@@ -10121,32 +10121,32 @@
                 "0.5": {
                   "coverage": 0.6,
                   "accuracy": 0.3333333333333333,
-                  "confidence_cutoff": 0.9634562251390246
+                  "confidence_cutoff": 0.9584822208396057
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.375,
-                  "confidence_cutoff": 0.9614389381469302
+                  "confidence_cutoff": 0.9566530548759113
                 }
               },
-              "score_mae": 0.9905057090116424,
-              "ranked_probability_score": 0.4635245683115593
+              "score_mae": 0.9890203760461705,
+              "ranked_probability_score": 0.456901425188147
             },
             "unknowable_warranty_claim": {
               "n": 10,
-              "nll": 0.9921422537254998,
-              "acc": 0.5,
-              "ece": 0.2020352656125068,
-              "brier": 0.6100410016954086,
-              "mean_conf": 0.46137759870501566,
+              "nll": 0.9999682429218097,
+              "acc": 0.4,
+              "ece": 0.16837300755606652,
+              "brier": 0.6134949708297965,
+              "mean_conf": 0.4548224411008467,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.0,
               "accuracy_at_0_9": null,
               "coverage_at_5pct_error": 0.1,
               "coverage_at_1pct_error": 0.1,
-              "aurc": 0.4818650793650794,
+              "aurc": 0.5029761904761905,
               "error_rate_at_0_9": null,
-              "confidence_bias": -0.03862240129498434,
+              "confidence_bias": 0.054822441100846664,
               "top_bins": {
                 "0.9": {
                   "n": 0,
@@ -10168,86 +10168,86 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.6,
-                  "confidence_cutoff": 0.42317827905448857
+                  "confidence_cutoff": 0.42227191991424323
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.375,
-                  "confidence_cutoff": 0.40588767546080573
+                  "confidence_cutoff": 0.4032192648416538
                 }
               },
-              "score_mae": 0.5401415858390292,
-              "ranked_probability_score": 0.17205966045392337
+              "score_mae": 0.5399159280722049,
+              "ranked_probability_score": 0.17294328544089552
             }
           },
           "variants": {
             "clean": {
               "n": 1156,
-              "nll": 0.6634965588347153,
-              "acc": 0.7413494809688581,
-              "ece": 0.07307599837609173,
-              "brier": 0.3373915801751656,
-              "mean_conf": 0.8144254793449499,
-              "confident_error_rate": 0.0328719723183391,
-              "coverage_at_0_9": 0.551038062283737,
-              "accuracy_at_0_9": 0.9403453689167975,
-              "coverage_at_5pct_error": 0.5077854671280276,
-              "coverage_at_1pct_error": 0.1185121107266436,
-              "aurc": 0.08049307232208963,
-              "error_rate_at_0_9": 0.059654631083202514,
-              "confidence_bias": 0.0730759983760918,
-              "top_bins": {
-                "0.9": {
-                  "n": 637,
-                  "errors": 38,
-                  "error_rate": 0.059654631083202514
-                },
-                "0.95": {
-                  "n": 532,
-                  "errors": 22,
-                  "error_rate": 0.041353383458646614
+              "nll": 0.6540910351086936,
+              "acc": 0.7422145328719724,
+              "ece": 0.06497107410583432,
+              "brier": 0.33406731581885374,
+              "mean_conf": 0.8071856069778066,
+              "confident_error_rate": 0.03027681660899654,
+              "coverage_at_0_9": 0.53719723183391,
+              "accuracy_at_0_9": 0.9436392914653784,
+              "coverage_at_5pct_error": 0.5086505190311419,
+              "coverage_at_1pct_error": 0.11245674740484429,
+              "aurc": 0.08035729090618272,
+              "error_rate_at_0_9": 0.05636070853462158,
+              "confidence_bias": 0.06497107410583425,
+              "top_bins": {
+                "0.9": {
+                  "n": 621,
+                  "errors": 35,
+                  "error_rate": 0.05636070853462158
+                },
+                "0.95": {
+                  "n": 509,
+                  "errors": 19,
+                  "error_rate": 0.03732809430255403
                 },
                 "0.99": {
-                  "n": 207,
+                  "n": 174,
                   "errors": 3,
-                  "error_rate": 0.014492753623188406
+                  "error_rate": 0.017241379310344827
                 }
               },
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.9515570934256056,
-                  "confidence_cutoff": 0.9346480717751411
+                  "confidence_cutoff": 0.9274184747088865
                 },
                 "0.8": {
                   "coverage": 0.801038062283737,
-                  "accuracy": 0.8412526997840173,
-                  "confidence_cutoff": 0.5943543182085235
+                  "accuracy": 0.8390928725701944,
+                  "confidence_cutoff": 0.5894361470891768
                 }
               },
-              "score_mae": 0.3960124111370266,
-              "ranked_probability_score": 0.13525323778002232
+              "score_mae": 0.3938198959642317,
+              "ranked_probability_score": 0.13294739488750432
             },
             "none_absent": {
               "n": 36,
-              "nll": 1.2780587311745955,
+              "nll": 1.2677850992926212,
               "acc": 0.5,
-              "ece": 0.28787280132015003,
-              "brier": 0.7144050226937334,
-              "mean_conf": 0.6869218961640411,
+              "ece": 0.278053484436123,
+              "brier": 0.7154228122314299,
+              "mean_conf": 0.6718423502744483,
               "confident_error_rate": 0.08333333333333333,
-              "coverage_at_0_9": 0.1111111111111111,
-              "accuracy_at_0_9": 0.25,
+              "coverage_at_0_9": 0.08333333333333333,
+              "accuracy_at_0_9": 0.0,
               "coverage_at_5pct_error": 0.0,
               "coverage_at_1pct_error": 0.0,
-              "aurc": 0.5376174183815399,
-              "error_rate_at_0_9": 0.75,
-              "confidence_bias": 0.1869218961640411,
+              "aurc": 0.5441880853688735,
+              "error_rate_at_0_9": 1.0,
+              "confidence_bias": 0.17184235027444827,
               "top_bins": {
                 "0.9": {
-                  "n": 4,
+                  "n": 3,
                   "errors": 3,
-                  "error_rate": 0.75
+                  "error_rate": 1.0
                 },
                 "0.95": {
                   "n": 2,
@@ -10264,33 +10264,33 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.6111111111111112,
-                  "confidence_cutoff": 0.696060664667048
+                  "confidence_cutoff": 0.6822161181306214
                 },
                 "0.8": {
                   "coverage": 0.8055555555555556,
                   "accuracy": 0.5172413793103449,
-                  "confidence_cutoff": 0.4846763048647645
+                  "confidence_cutoff": 0.4968951628630185
                 }
               }
             },
             "none_present": {
               "n": 36,
-              "nll": 0.7460177433221598,
+              "nll": 0.7499526141523112,
               "acc": 0.7777777777777778,
-              "ece": 0.14627996115067443,
-              "brier": 0.28709997650834473,
-              "mean_conf": 0.8017359444765791,
+              "ece": 0.10875199149059422,
+              "brier": 0.2872316962408804,
+              "mean_conf": 0.7911516003199925,
               "confident_error_rate": 0.0,
-              "coverage_at_0_9": 0.5555555555555556,
+              "coverage_at_0_9": 0.5277777777777778,
               "accuracy_at_0_9": 1.0,
               "coverage_at_5pct_error": 0.7222222222222222,
               "coverage_at_1pct_error": 0.6666666666666666,
-              "aurc": 0.04264397382032627,
+              "aurc": 0.0406940559221325,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": 0.023958166698801353,
+              "confidence_bias": 0.013373822542214753,
               "top_bins": {
                 "0.9": {
-                  "n": 20,
+                  "n": 19,
                   "errors": 0,
                   "error_rate": 0.0
                 },
@@ -10300,7 +10300,7 @@
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 6,
+                  "n": 5,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -10309,43 +10309,43 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9382298345728617
+                  "confidence_cutoff": 0.9345858612432604
                 },
                 "0.8": {
                   "coverage": 0.8055555555555556,
-                  "accuracy": 0.8620689655172413,
-                  "confidence_cutoff": 0.5789058197682615
+                  "accuracy": 0.896551724137931,
+                  "confidence_cutoff": 0.5753147944995638
                 }
               }
             },
             "permuted": {
               "n": 36,
-              "nll": 0.7203281270997002,
+              "nll": 0.7075736943294805,
               "acc": 0.7777777777777778,
-              "ece": 0.09527377914540269,
-              "brier": 0.27961577244776725,
-              "mean_conf": 0.836032418535974,
+              "ece": 0.11496218985382063,
+              "brier": 0.2750490327493021,
+              "mean_conf": 0.8292829467749867,
               "confident_error_rate": 0.0,
-              "coverage_at_0_9": 0.6388888888888888,
+              "coverage_at_0_9": 0.6111111111111112,
               "accuracy_at_0_9": 1.0,
               "coverage_at_5pct_error": 0.7777777777777778,
               "coverage_at_1pct_error": 0.6666666666666666,
-              "aurc": 0.03678940551365068,
+              "aurc": 0.037657461069206234,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": 0.0582546407581962,
+              "confidence_bias": 0.05150516899720892,
               "top_bins": {
                 "0.9": {
-                  "n": 23,
+                  "n": 22,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.95": {
-                  "n": 18,
+                  "n": 16,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 9,
+                  "n": 7,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -10354,12 +10354,12 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9544422568229554
+                  "confidence_cutoff": 0.9415673543396044
                 },
                 "0.8": {
                   "coverage": 0.8055555555555556,
                   "accuracy": 0.9310344827586207,
-                  "confidence_cutoff": 0.6258627504206388
+                  "confidence_cutoff": 0.6202919952139035
                 }
               }
             }
@@ -10367,56 +10367,56 @@
           "heldout_tasks": {},
           "permutation": {
             "n": 36,
-            "mean_max_delta": 0.08279943035271198,
+            "mean_max_delta": 0.08021796939627156,
             "flip_rate": 0.08333333333333333
           },
           "temperature": 1.0,
           "calibrated_clean": {
             "n": 1046,
-            "nll": 0.5940603022446113,
-            "acc": 0.7762906309751434,
-            "ece": 0.05558262070294892,
-            "brier": 0.2937157813621403,
-            "mean_conf": 0.8318732516780923,
-            "confident_error_rate": 0.02390057361376673,
-            "coverage_at_0_9": 0.5860420650095602,
-            "accuracy_at_0_9": 0.9592169657422512,
-            "coverage_at_5pct_error": 0.6347992351816444,
-            "coverage_at_1pct_error": 0.3030592734225621,
-            "aurc": 0.05816033320114623,
-            "error_rate_at_0_9": 0.040783034257748776,
-            "confidence_bias": 0.05558262070294895,
+            "nll": 0.5870477618581573,
+            "acc": 0.777246653919694,
+            "ece": 0.04794282897297868,
+            "brier": 0.2910675481135292,
+            "mean_conf": 0.8251894828926727,
+            "confident_error_rate": 0.02198852772466539,
+            "coverage_at_0_9": 0.5717017208413002,
+            "accuracy_at_0_9": 0.9615384615384616,
+            "coverage_at_5pct_error": 0.6338432122370937,
+            "coverage_at_1pct_error": 0.29923518164435947,
+            "aurc": 0.057865847250986556,
+            "error_rate_at_0_9": 0.038461538461538464,
+            "confidence_bias": 0.047942828972978635,
             "top_bins": {
               "0.9": {
-                "n": 613,
-                "errors": 25,
-                "error_rate": 0.040783034257748776
+                "n": 598,
+                "errors": 23,
+                "error_rate": 0.038461538461538464
               },
               "0.95": {
-                "n": 514,
-                "errors": 12,
-                "error_rate": 0.023346303501945526
+                "n": 491,
+                "errors": 9,
+                "error_rate": 0.018329938900203666
               },
               "0.99": {
-                "n": 207,
+                "n": 174,
                 "errors": 3,
-                "error_rate": 0.014492753623188406
+                "error_rate": 0.017241379310344827
               }
             },
             "selective": {
               "0.5": {
-                "coverage": 0.5009560229445507,
-                "accuracy": 0.9770992366412213,
-                "confidence_cutoff": 0.947606086730957
+                "coverage": 0.5,
+                "accuracy": 0.9751434034416826,
+                "confidence_cutoff": 0.9408829808235168
               },
               "0.8": {
                 "coverage": 0.8001912045889101,
-                "accuracy": 0.8781362007168458,
-                "confidence_cutoff": 0.647434909565297
+                "accuracy": 0.8793309438470729,
+                "confidence_cutoff": 0.6510422047867755
               }
             },
-            "score_mae": 0.18149603870298436,
-            "ranked_probability_score": 0.051063424067515496
+            "score_mae": 0.18602619774583048,
+            "ranked_probability_score": 0.050882127676710784
           },
           "metric_policy": {
             "version": 2,