diff --git "a/metrics/benchmark.json" "b/metrics/benchmark.json"
--- "a/metrics/benchmark.json"
+++ "b/metrics/benchmark.json"
@@ -8,7 +8,7 @@
     "transfer_v9_test": 0.15
   },
   "protocol_sha256": "ab97f652ab890eaefdd5373dd2deeb1e26588815f1f339eee4bcaa610846b0af",
-  "statistics_sha256": "169bdbb413aa3302369d56f814d35641a5b4e1de82ef3ae688c28ca740b3b0dc",
+  "statistics_sha256": "c73006e701434d215264f2d623160f213cebf2a8ff9b4f8c0d3402b3b99436d3",
   "source_per_model": {
     "Jev": {
       "statistics_sha256": "b319a993ac569ea1fa8da027ccb2195868557e4efe232a5d06091db2884086be",
@@ -119,9 +119,9 @@
       }
     },
     "Lux": {
-      "statistics_sha256": "bfc75a9080fb0fa22a45ac21c23048d5c8e5deac81dcc202792254565951d8e7",
-      "source_model_key": "Lux-retention-candidate",
-      "manifest_sha256": "469d9e05ff0283cb50efe45307c7e7bcefa9cb124cc5103b9aa319e2537801a0",
+      "statistics_sha256": "49c1195fcd88860620fa7d5eb48bb77d6bc759cbac6db89ab11a6faf485644fa",
+      "source_model_key": "Lux-semantic-format-candidate",
+      "manifest_sha256": "485e1f7505d390fb6e84c34edca5812c6803f4d7a48b3f752f7784decdf58559",
       "qualified_runtime": {
         "python": "3.12.13",
         "numpy": "2.3.5"
@@ -5296,19 +5296,19 @@
     "Lux": {
       "panels": {
         "old_core": {
-          "point": 0.8409598214285714,
-          "exact_fraction": "1507/1792",
+          "point": 0.8417410714285715,
+          "exact_fraction": "3771/4480",
           "ci95": [
-            0.8149553571428573,
-            0.866630394345238
+            0.8145452008928572,
+            0.8688244047619047
           ]
         },
         "v3_core": {
-          "point": 0.5175,
-          "exact_fraction": "207/400",
+          "point": 0.5275,
+          "exact_fraction": "211/400",
           "ci95": [
-            0.48583333333333345,
-            0.5491666666666666
+            0.49416666666666675,
+            0.5608333333333333
           ]
         },
         "v4": {
@@ -5320,35 +5320,35 @@
           ]
         },
         "v5": {
-          "point": 0.9145833333333333,
-          "exact_fraction": "439/480",
+          "point": 0.9125,
+          "exact_fraction": "73/80",
           "ci95": [
-            0.8888980313342782,
-            0.9377058522747549
+            0.8866503163407948,
+            0.9366505343021726
           ]
         },
         "transfer_v9_test": {
-          "point": 0.7734225621414914,
-          "exact_fraction": "809/1046",
+          "point": 0.7762906309751434,
+          "exact_fraction": "406/523",
           "ci95": [
-            0.7450787401574803,
-            0.8005593780665484
+            0.7482989865877792,
+            0.8032481563248626
           ]
         },
         "original_four_panel_mean": {
-          "point": 0.7924795386904762,
-          "exact_fraction": "426037/537600",
+          "point": 0.7946540178571428,
+          "exact_fraction": "71201/89600",
           "ci95": [
-            0.7777779110823996,
-            0.8066691389450736
+            0.7794567024393497,
+            0.8092736519074929
           ]
         },
         "weighted_mean": {
-          "point": 0.7693950807497951,
-          "exact_fraction": "36054469/46860800",
+          "point": 0.7722471660748429,
+          "exact_fraction": "904703/1171520",
           "ci95": [
-            0.7559702781267388,
-            0.7824389898408891
+            0.7583454502909792,
+            0.785859954609448
           ]
         }
       },
@@ -5359,7 +5359,7 @@
           "valid_probability_rows": 880,
           "families": {
             "ag_news": {
-              "correct": 113,
+              "correct": 114,
               "requested": 128
             },
             "boolean_constraints": {
@@ -5375,7 +5375,7 @@
               "requested": 64
             },
             "option_carrier": {
-              "correct": 62,
+              "correct": 70,
               "requested": 96
             },
             "ordinal_rubric": {
@@ -5383,15 +5383,15 @@
               "requested": 64
             },
             "relational_composition": {
-              "correct": 59,
+              "correct": 57,
               "requested": 96
             },
             "scoped_evidence": {
-              "correct": 94,
+              "correct": 91,
               "requested": 96
             },
             "state_tracking": {
-              "correct": 31,
+              "correct": 28,
               "requested": 96
             },
             "unknown_rejection": {
@@ -5406,15 +5406,15 @@
           "valid_probability_rows": 880,
           "families": {
             "canonical_record_identity": {
-              "correct": 51,
+              "correct": 49,
               "requested": 80
             },
             "capacitated_assignment": {
-              "correct": 50,
+              "correct": 58,
               "requested": 80
             },
             "constraint_assignment": {
-              "correct": 30,
+              "correct": 26,
               "requested": 80
             },
             "massive_en": {
@@ -5426,23 +5426,23 @@
               "requested": 120
             },
             "multiset_reconciliation": {
-              "correct": 21,
+              "correct": 23,
               "requested": 80
             },
             "ordinal_service_loss": {
-              "correct": 19,
+              "correct": 22,
               "requested": 80
             },
             "paraconsistent_rule_closure": {
-              "correct": 25,
+              "correct": 29,
               "requested": 80
             },
             "temporal_exclusion": {
-              "correct": 36,
+              "correct": 31,
               "requested": 80
             },
             "transaction_recovery": {
-              "correct": 40,
+              "correct": 42,
               "requested": 80
             }
           }
@@ -5472,7 +5472,7 @@
           "valid_probability_rows": 480,
           "families": {
             "cosmos_qa": {
-              "correct": 100,
+              "correct": 99,
               "requested": 120
             },
             "squad2_answerability": {
@@ -5494,101 +5494,101 @@
         "clean": {
           "requested": 1046,
           "valid_probability_rows": 1046,
-          "correct": 809,
-          "accuracy": 0.7734225621414914,
+          "correct": 812,
+          "accuracy": 0.7762906309751434,
           "full_probability_metrics": {
             "n": 1046,
-            "nll": 0.6086212459300988,
-            "acc": 0.7734225621414914,
-            "ece": 0.05889322575332617,
-            "brier": 0.3004372452589081,
-            "mean_conf": 0.8323157878948174,
+            "nll": 0.5940603022446113,
+            "acc": 0.7762906309751434,
+            "ece": 0.05558262070294892,
+            "brier": 0.2937157813621403,
+            "mean_conf": 0.8318732516780923,
             "confident_error_rate": 0.02390057361376673,
-            "coverage_at_0_9": 0.5774378585086042,
-            "accuracy_at_0_9": 0.9586092715231788,
-            "coverage_at_5pct_error": 0.6338432122370937,
-            "coverage_at_1pct_error": 0.38336520076481834,
-            "aurc": 0.061361186701696054,
-            "error_rate_at_0_9": 0.041390728476821195,
-            "confidence_bias": 0.058893225753326006,
+            "coverage_at_0_9": 0.5860420650095602,
+            "accuracy_at_0_9": 0.9592169657422512,
+            "coverage_at_5pct_error": 0.6347992351816444,
+            "coverage_at_1pct_error": 0.3030592734225621,
+            "aurc": 0.05816033320114623,
+            "error_rate_at_0_9": 0.040783034257748776,
+            "confidence_bias": 0.05558262070294895,
             "top_bins": {
               "0.9": {
-                "n": 604,
+                "n": 613,
                 "errors": 25,
-                "error_rate": 0.041390728476821195
+                "error_rate": 0.040783034257748776
               },
               "0.95": {
-                "n": 525,
-                "errors": 15,
-                "error_rate": 0.02857142857142857
+                "n": 514,
+                "errors": 12,
+                "error_rate": 0.023346303501945526
               },
               "0.99": {
-                "n": 226,
+                "n": 207,
                 "errors": 3,
-                "error_rate": 0.01327433628318584
+                "error_rate": 0.014492753623188406
               }
             },
             "selective": {
               "0.5": {
-                "coverage": 0.5,
-                "accuracy": 0.97131931166348,
-                "confidence_cutoff": 0.9503957614513975
+                "coverage": 0.5009560229445507,
+                "accuracy": 0.9770992366412213,
+                "confidence_cutoff": 0.947606086730957
               },
               "0.8": {
                 "coverage": 0.8001912045889101,
-                "accuracy": 0.8745519713261649,
-                "confidence_cutoff": 0.6527601881992761
+                "accuracy": 0.8781362007168458,
+                "confidence_cutoff": 0.647434909565297
               }
             },
-            "score_mae": 0.19182332300936422,
-            "ranked_probability_score": 0.05030124964901672
+            "score_mae": 0.18149603870298436,
+            "ranked_probability_score": 0.051063424067515496
           },
           "covered_only_probability_metrics": {
             "n": 1046,
-            "nll": 0.6086212459300988,
-            "acc": 0.7734225621414914,
-            "ece": 0.05889322575332617,
-            "brier": 0.3004372452589081,
-            "mean_conf": 0.8323157878948174,
+            "nll": 0.5940603022446113,
+            "acc": 0.7762906309751434,
+            "ece": 0.05558262070294892,
+            "brier": 0.2937157813621403,
+            "mean_conf": 0.8318732516780923,
             "confident_error_rate": 0.02390057361376673,
-            "coverage_at_0_9": 0.5774378585086042,
-            "accuracy_at_0_9": 0.9586092715231788,
-            "coverage_at_5pct_error": 0.6338432122370937,
-            "coverage_at_1pct_error": 0.38336520076481834,
-            "aurc": 0.061361186701696054,
-            "error_rate_at_0_9": 0.041390728476821195,
-            "confidence_bias": 0.058893225753326006,
+            "coverage_at_0_9": 0.5860420650095602,
+            "accuracy_at_0_9": 0.9592169657422512,
+            "coverage_at_5pct_error": 0.6347992351816444,
+            "coverage_at_1pct_error": 0.3030592734225621,
+            "aurc": 0.05816033320114623,
+            "error_rate_at_0_9": 0.040783034257748776,
+            "confidence_bias": 0.05558262070294895,
             "top_bins": {
               "0.9": {
-                "n": 604,
+                "n": 613,
                 "errors": 25,
-                "error_rate": 0.041390728476821195
+                "error_rate": 0.040783034257748776
               },
               "0.95": {
-                "n": 525,
-                "errors": 15,
-                "error_rate": 0.02857142857142857
+                "n": 514,
+                "errors": 12,
+                "error_rate": 0.023346303501945526
               },
               "0.99": {
-                "n": 226,
+                "n": 207,
                 "errors": 3,
-                "error_rate": 0.01327433628318584
+                "error_rate": 0.014492753623188406
               }
             },
             "selective": {
               "0.5": {
-                "coverage": 0.5,
-                "accuracy": 0.97131931166348,
-                "confidence_cutoff": 0.9503957614513975
+                "coverage": 0.5009560229445507,
+                "accuracy": 0.9770992366412213,
+                "confidence_cutoff": 0.947606086730957
               },
               "0.8": {
                 "coverage": 0.8001912045889101,
-                "accuracy": 0.8745519713261649,
-                "confidence_cutoff": 0.6527601881992761
+                "accuracy": 0.8781362007168458,
+                "confidence_cutoff": 0.647434909565297
               }
             },
-            "score_mae": 0.19182332300936422,
-            "ranked_probability_score": 0.05030124964901672
+            "score_mae": 0.18149603870298436,
+            "ranked_probability_score": 0.051063424067515496
           },
           "coverage": 1.0
         },
@@ -5600,29 +5600,29 @@
             "accuracy": 0.45,
             "full_probability_metrics": {
               "n": 20,
-              "nll": 1.5449426114821578,
+              "nll": 1.4899537928777824,
               "acc": 0.45,
-              "ece": 0.24636374641400255,
-              "brier": 0.7048268473893603,
-              "mean_conf": 0.6963637464140026,
+              "ece": 0.3017003229117818,
+              "brier": 0.6913230901539468,
+              "mean_conf": 0.6857035964205832,
               "confident_error_rate": 0.05,
-              "coverage_at_0_9": 0.25,
-              "accuracy_at_0_9": 0.8,
+              "coverage_at_0_9": 0.2,
+              "accuracy_at_0_9": 0.75,
               "coverage_at_5pct_error": 0.05,
               "coverage_at_1pct_error": 0.05,
-              "aurc": 0.3654720743652632,
-              "error_rate_at_0_9": 0.2,
-              "confidence_bias": 0.24636374641400255,
+              "aurc": 0.3734132508358514,
+              "error_rate_at_0_9": 0.25,
+              "confidence_bias": 0.2357035964205832,
               "top_bins": {
                 "0.9": {
-                  "n": 5,
+                  "n": 4,
                   "errors": 1,
-                  "error_rate": 0.2
+                  "error_rate": 0.25
                 },
                 "0.95": {
-                  "n": 2,
-                  "errors": 1,
-                  "error_rate": 0.5
+                  "n": 1,
+                  "errors": 0,
+                  "error_rate": 0.0
                 },
                 "0.99": {
                   "n": 0,
@@ -5633,41 +5633,41 @@
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
-                  "accuracy": 0.7,
-                  "confidence_cutoff": 0.7313402914926098
+                  "accuracy": 0.6,
+                  "confidence_cutoff": 0.720178043538323
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.5043779386113015
+                  "confidence_cutoff": 0.49892304084621675
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 20,
-              "nll": 1.5449426114821578,
+              "nll": 1.4899537928777824,
               "acc": 0.45,
-              "ece": 0.24636374641400255,
-              "brier": 0.7048268473893603,
-              "mean_conf": 0.6963637464140026,
+              "ece": 0.3017003229117818,
+              "brier": 0.6913230901539468,
+              "mean_conf": 0.6857035964205832,
               "confident_error_rate": 0.05,
-              "coverage_at_0_9": 0.25,
-              "accuracy_at_0_9": 0.8,
+              "coverage_at_0_9": 0.2,
+              "accuracy_at_0_9": 0.75,
               "coverage_at_5pct_error": 0.05,
               "coverage_at_1pct_error": 0.05,
-              "aurc": 0.3654720743652632,
-              "error_rate_at_0_9": 0.2,
-              "confidence_bias": 0.24636374641400255,
+              "aurc": 0.3734132508358514,
+              "error_rate_at_0_9": 0.25,
+              "confidence_bias": 0.2357035964205832,
               "top_bins": {
                 "0.9": {
-                  "n": 5,
+                  "n": 4,
                   "errors": 1,
-                  "error_rate": 0.2
+                  "error_rate": 0.25
                 },
                 "0.95": {
-                  "n": 2,
-                  "errors": 1,
-                  "error_rate": 0.5
+                  "n": 1,
+                  "errors": 0,
+                  "error_rate": 0.0
                 },
                 "0.99": {
                   "n": 0,
@@ -5678,13 +5678,13 @@
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
-                  "accuracy": 0.7,
-                  "confidence_cutoff": 0.7313402914926098
+                  "accuracy": 0.6,
+                  "confidence_cutoff": 0.720178043538323
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.5043779386113015
+                  "confidence_cutoff": 0.49892304084621675
                 }
               }
             },
@@ -5697,19 +5697,19 @@
             "accuracy": 0.8,
             "full_probability_metrics": {
               "n": 20,
-              "nll": 0.3590183753756898,
+              "nll": 0.391154446854581,
               "acc": 0.8,
-              "ece": 0.12098253375100354,
-              "brier": 0.2234753942109866,
-              "mean_conf": 0.8746775461236951,
+              "ece": 0.0909007219853577,
+              "brier": 0.24269024392468377,
+              "mean_conf": 0.8773054444371697,
               "confident_error_rate": 0.05,
               "coverage_at_0_9": 0.7,
               "accuracy_at_0_9": 0.9285714285714286,
-              "coverage_at_5pct_error": 0.5,
-              "coverage_at_1pct_error": 0.5,
-              "aurc": 0.04897950583128602,
+              "coverage_at_5pct_error": 0.45,
+              "coverage_at_1pct_error": 0.45,
+              "aurc": 0.06282346007965203,
               "error_rate_at_0_9": 0.07142857142857142,
-              "confidence_bias": 0.07467754612369504,
+              "confidence_bias": 0.07730544443716969,
               "top_bins": {
                 "0.9": {
                   "n": 14,
@@ -5730,31 +5730,31 @@
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
-                  "accuracy": 1.0,
-                  "confidence_cutoff": 0.9621361601591426
+                  "accuracy": 0.9,
+                  "confidence_cutoff": 0.9649345170900093
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.9375,
-                  "confidence_cutoff": 0.708977460861206
+                  "accuracy": 0.875,
+                  "confidence_cutoff": 0.7085450713959379
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 20,
-              "nll": 0.3590183753756898,
+              "nll": 0.391154446854581,
               "acc": 0.8,
-              "ece": 0.12098253375100354,
-              "brier": 0.2234753942109866,
-              "mean_conf": 0.8746775461236951,
+              "ece": 0.0909007219853577,
+              "brier": 0.24269024392468377,
+              "mean_conf": 0.8773054444371697,
               "confident_error_rate": 0.05,
               "coverage_at_0_9": 0.7,
               "accuracy_at_0_9": 0.9285714285714286,
-              "coverage_at_5pct_error": 0.5,
-              "coverage_at_1pct_error": 0.5,
-              "aurc": 0.04897950583128602,
+              "coverage_at_5pct_error": 0.45,
+              "coverage_at_1pct_error": 0.45,
+              "aurc": 0.06282346007965203,
               "error_rate_at_0_9": 0.07142857142857142,
-              "confidence_bias": 0.07467754612369504,
+              "confidence_bias": 0.07730544443716969,
               "top_bins": {
                 "0.9": {
                   "n": 14,
@@ -5775,13 +5775,13 @@
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
-                  "accuracy": 1.0,
-                  "confidence_cutoff": 0.9621361601591426
+                  "accuracy": 0.9,
+                  "confidence_cutoff": 0.9649345170900093
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.9375,
-                  "confidence_cutoff": 0.708977460861206
+                  "accuracy": 0.875,
+                  "confidence_cutoff": 0.7085450713959379
                 }
               }
             },
@@ -5794,22 +5794,22 @@
             "accuracy": 0.9,
             "full_probability_metrics": {
               "n": 20,
-              "nll": 0.17230822458126488,
+              "nll": 0.173920037838875,
               "acc": 0.9,
-              "ece": 0.07775387997537764,
-              "brier": 0.11106925746781111,
-              "mean_conf": 0.9399342803070365,
+              "ece": 0.08884237104933625,
+              "brier": 0.11334577453222641,
+              "mean_conf": 0.9453143823740051,
               "confident_error_rate": 0.0,
-              "coverage_at_0_9": 0.8,
+              "coverage_at_0_9": 0.85,
               "accuracy_at_0_9": 1.0,
               "coverage_at_5pct_error": 0.9,
               "coverage_at_1pct_error": 0.9,
               "aurc": 0.0076315789473684215,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": 0.039934280307036496,
+              "confidence_bias": 0.04531438237400509,
               "top_bins": {
                 "0.9": {
-                  "n": 16,
+                  "n": 17,
                   "errors": 0,
                   "error_rate": 0.0
                 },
@@ -5828,33 +5828,33 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9794639367227702
+                  "confidence_cutoff": 0.9793051012887353
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9520198042701821
+                  "confidence_cutoff": 0.9588658106453812
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 20,
-              "nll": 0.17230822458126488,
+              "nll": 0.173920037838875,
               "acc": 0.9,
-              "ece": 0.07775387997537764,
-              "brier": 0.11106925746781111,
-              "mean_conf": 0.9399342803070365,
+              "ece": 0.08884237104933625,
+              "brier": 0.11334577453222641,
+              "mean_conf": 0.9453143823740051,
               "confident_error_rate": 0.0,
-              "coverage_at_0_9": 0.8,
+              "coverage_at_0_9": 0.85,
               "accuracy_at_0_9": 1.0,
               "coverage_at_5pct_error": 0.9,
               "coverage_at_1pct_error": 0.9,
               "aurc": 0.0076315789473684215,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": 0.039934280307036496,
+              "confidence_bias": 0.04531438237400509,
               "top_bins": {
                 "0.9": {
-                  "n": 16,
+                  "n": 17,
                   "errors": 0,
                   "error_rate": 0.0
                 },
@@ -5873,12 +5873,12 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9794639367227702
+                  "confidence_cutoff": 0.9793051012887353
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9520198042701821
+                  "confidence_cutoff": 0.9588658106453812
                 }
               }
             },
@@ -5891,19 +5891,19 @@
             "accuracy": 0.4,
             "full_probability_metrics": {
               "n": 20,
-              "nll": 1.0747043813664825,
+              "nll": 0.963421060535776,
               "acc": 0.4,
-              "ece": 0.376769070482061,
-              "brier": 0.7839551359464374,
-              "mean_conf": 0.776769070482061,
+              "ece": 0.3598356256038071,
+              "brier": 0.7060594827987038,
+              "mean_conf": 0.7594032993432073,
               "confident_error_rate": 0.05,
               "coverage_at_0_9": 0.1,
               "accuracy_at_0_9": 0.5,
-              "coverage_at_5pct_error": 0.0,
-              "coverage_at_1pct_error": 0.0,
-              "aurc": 0.5897093311315447,
+              "coverage_at_5pct_error": 0.05,
+              "coverage_at_1pct_error": 0.05,
+              "aurc": 0.45470466259746445,
               "error_rate_at_0_9": 0.5,
-              "confidence_bias": 0.376769070482061,
+              "confidence_bias": 0.35940329934320725,
               "top_bins": {
                 "0.9": {
                   "n": 2,
@@ -5925,30 +5925,30 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.7923556442472722
+                  "confidence_cutoff": 0.7584096306325441
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.3125,
-                  "confidence_cutoff": 0.6593232357905366
+                  "accuracy": 0.4375,
+                  "confidence_cutoff": 0.6582031053840188
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 20,
-              "nll": 1.0747043813664825,
+              "nll": 0.963421060535776,
               "acc": 0.4,
-              "ece": 0.376769070482061,
-              "brier": 0.7839551359464374,
-              "mean_conf": 0.776769070482061,
+              "ece": 0.3598356256038071,
+              "brier": 0.7060594827987038,
+              "mean_conf": 0.7594032993432073,
               "confident_error_rate": 0.05,
               "coverage_at_0_9": 0.1,
               "accuracy_at_0_9": 0.5,
-              "coverage_at_5pct_error": 0.0,
-              "coverage_at_1pct_error": 0.0,
-              "aurc": 0.5897093311315447,
+              "coverage_at_5pct_error": 0.05,
+              "coverage_at_1pct_error": 0.05,
+              "aurc": 0.45470466259746445,
               "error_rate_at_0_9": 0.5,
-              "confidence_bias": 0.376769070482061,
+              "confidence_bias": 0.35940329934320725,
               "top_bins": {
                 "0.9": {
                   "n": 2,
@@ -5970,12 +5970,12 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.7923556442472722
+                  "confidence_cutoff": 0.7584096306325441
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.3125,
-                  "confidence_cutoff": 0.6593232357905366
+                  "accuracy": 0.4375,
+                  "confidence_cutoff": 0.6582031053840188
                 }
               }
             },
@@ -5988,22 +5988,22 @@
             "accuracy": 0.96875,
             "full_probability_metrics": {
               "n": 32,
-              "nll": 0.1488496730625657,
+              "nll": 0.13034778836964267,
               "acc": 0.96875,
-              "ece": 0.0518019411069781,
-              "brier": 0.07608424815707887,
-              "mean_conf": 0.9169480588930219,
+              "ece": 0.07387770926294471,
+              "brier": 0.06436468242012063,
+              "mean_conf": 0.9252340474017305,
               "confident_error_rate": 0.0,
-              "coverage_at_0_9": 0.6875,
+              "coverage_at_0_9": 0.75,
               "accuracy_at_0_9": 1.0,
               "coverage_at_5pct_error": 1.0,
-              "coverage_at_1pct_error": 0.8125,
-              "aurc": 0.006343756575133452,
+              "coverage_at_1pct_error": 0.84375,
+              "aurc": 0.005186349167726045,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.05180194110697811,
+              "confidence_bias": -0.04351595259826946,
               "top_bins": {
                 "0.9": {
-                  "n": 22,
+                  "n": 24,
                   "errors": 0,
                   "error_rate": 0.0
                 },
@@ -6020,35 +6020,35 @@
               },
               "selective": {
                 "0.5": {
-                  "coverage": 0.5,
+                  "coverage": 0.53125,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9526311826156912
+                  "confidence_cutoff": 0.9517537497245334
                 },
                 "0.8": {
                   "coverage": 0.8125,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.8658791104483075
+                  "confidence_cutoff": 0.8932077222078463
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 32,
-              "nll": 0.1488496730625657,
+              "nll": 0.13034778836964267,
               "acc": 0.96875,
-              "ece": 0.0518019411069781,
-              "brier": 0.07608424815707887,
-              "mean_conf": 0.9169480588930219,
+              "ece": 0.07387770926294471,
+              "brier": 0.06436468242012063,
+              "mean_conf": 0.9252340474017305,
               "confident_error_rate": 0.0,
-              "coverage_at_0_9": 0.6875,
+              "coverage_at_0_9": 0.75,
               "accuracy_at_0_9": 1.0,
               "coverage_at_5pct_error": 1.0,
-              "coverage_at_1pct_error": 0.8125,
-              "aurc": 0.006343756575133452,
+              "coverage_at_1pct_error": 0.84375,
+              "aurc": 0.005186349167726045,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.05180194110697811,
+              "confidence_bias": -0.04351595259826946,
               "top_bins": {
                 "0.9": {
-                  "n": 22,
+                  "n": 24,
                   "errors": 0,
                   "error_rate": 0.0
                 },
@@ -6065,14 +6065,14 @@
               },
               "selective": {
                 "0.5": {
-                  "coverage": 0.5,
+                  "coverage": 0.53125,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9526311826156912
+                  "confidence_cutoff": 0.9517537497245334
                 },
                 "0.8": {
                   "coverage": 0.8125,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.8658791104483075
+                  "confidence_cutoff": 0.8932077222078463
                 }
               }
             },
@@ -6085,32 +6085,32 @@
             "accuracy": 0.90625,
             "full_probability_metrics": {
               "n": 32,
-              "nll": 0.2775084906134728,
+              "nll": 0.24646259109043608,
               "acc": 0.90625,
-              "ece": 0.05746431889041253,
-              "brier": 0.15312468872339335,
-              "mean_conf": 0.9174051821778417,
+              "ece": 0.08165686202673392,
+              "brier": 0.13219691386769145,
+              "mean_conf": 0.9344170796872766,
               "confident_error_rate": 0.03125,
-              "coverage_at_0_9": 0.65625,
-              "accuracy_at_0_9": 0.9523809523809523,
-              "coverage_at_5pct_error": 0.84375,
+              "coverage_at_0_9": 0.75,
+              "accuracy_at_0_9": 0.9583333333333334,
+              "coverage_at_5pct_error": 0.9375,
               "coverage_at_1pct_error": 0.375,
-              "aurc": 0.03975357109250195,
-              "error_rate_at_0_9": 0.047619047619047616,
-              "confidence_bias": 0.011155182177841727,
+              "aurc": 0.033345170986070205,
+              "error_rate_at_0_9": 0.041666666666666664,
+              "confidence_bias": 0.02816707968727661,
               "top_bins": {
                 "0.9": {
-                  "n": 21,
+                  "n": 24,
                   "errors": 1,
-                  "error_rate": 0.047619047619047616
+                  "error_rate": 0.041666666666666664
                 },
                 "0.95": {
-                  "n": 16,
+                  "n": 18,
                   "errors": 1,
-                  "error_rate": 0.0625
+                  "error_rate": 0.05555555555555555
                 },
                 "0.99": {
-                  "n": 4,
+                  "n": 6,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -6119,43 +6119,43 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.9375,
-                  "confidence_cutoff": 0.9714136069375261
+                  "confidence_cutoff": 0.9769813800539988
                 },
                 "0.8": {
-                  "coverage": 0.84375,
-                  "accuracy": 0.9629629629629629,
-                  "confidence_cutoff": 0.866120457649231
+                  "coverage": 0.8125,
+                  "accuracy": 0.9615384615384616,
+                  "confidence_cutoff": 0.8745144680431802
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 32,
-              "nll": 0.2775084906134728,
+              "nll": 0.24646259109043608,
               "acc": 0.90625,
-              "ece": 0.05746431889041253,
-              "brier": 0.15312468872339335,
-              "mean_conf": 0.9174051821778417,
+              "ece": 0.08165686202673392,
+              "brier": 0.13219691386769145,
+              "mean_conf": 0.9344170796872766,
               "confident_error_rate": 0.03125,
-              "coverage_at_0_9": 0.65625,
-              "accuracy_at_0_9": 0.9523809523809523,
-              "coverage_at_5pct_error": 0.84375,
+              "coverage_at_0_9": 0.75,
+              "accuracy_at_0_9": 0.9583333333333334,
+              "coverage_at_5pct_error": 0.9375,
               "coverage_at_1pct_error": 0.375,
-              "aurc": 0.03975357109250195,
-              "error_rate_at_0_9": 0.047619047619047616,
-              "confidence_bias": 0.011155182177841727,
+              "aurc": 0.033345170986070205,
+              "error_rate_at_0_9": 0.041666666666666664,
+              "confidence_bias": 0.02816707968727661,
               "top_bins": {
                 "0.9": {
-                  "n": 21,
+                  "n": 24,
                   "errors": 1,
-                  "error_rate": 0.047619047619047616
+                  "error_rate": 0.041666666666666664
                 },
                 "0.95": {
-                  "n": 16,
+                  "n": 18,
                   "errors": 1,
-                  "error_rate": 0.0625
+                  "error_rate": 0.05555555555555555
                 },
                 "0.99": {
-                  "n": 4,
+                  "n": 6,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -6164,12 +6164,12 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.9375,
-                  "confidence_cutoff": 0.9714136069375261
+                  "confidence_cutoff": 0.9769813800539988
                 },
                 "0.8": {
-                  "coverage": 0.84375,
-                  "accuracy": 0.9629629629629629,
-                  "confidence_cutoff": 0.866120457649231
+                  "coverage": 0.8125,
+                  "accuracy": 0.9615384615384616,
+                  "confidence_cutoff": 0.8745144680431802
                 }
               }
             },
@@ -6178,26 +6178,26 @@
           "composition_final_negation": {
             "requested": 32,
             "valid_probability_rows": 32,
-            "correct": 28,
-            "accuracy": 0.875,
+            "correct": 31,
+            "accuracy": 0.96875,
             "full_probability_metrics": {
               "n": 32,
-              "nll": 0.23890017515673415,
-              "acc": 0.875,
-              "ece": 0.14167053791272208,
-              "brier": 0.14524192276307057,
-              "mean_conf": 0.8435546575020636,
+              "nll": 0.17366660442192522,
+              "acc": 0.96875,
+              "ece": 0.139280642091471,
+              "brier": 0.09349301051833991,
+              "mean_conf": 0.8684549516140397,
               "confident_error_rate": 0.0,
-              "coverage_at_0_9": 0.46875,
+              "coverage_at_0_9": 0.5625,
               "accuracy_at_0_9": 1.0,
-              "coverage_at_5pct_error": 0.90625,
-              "coverage_at_1pct_error": 0.84375,
-              "aurc": 0.012148294732242174,
+              "coverage_at_5pct_error": 1.0,
+              "coverage_at_1pct_error": 0.875,
+              "aurc": 0.004072377873563219,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.0314453424979364,
+              "confidence_bias": -0.10029504838596026,
               "top_bins": {
                 "0.9": {
-                  "n": 15,
+                  "n": 18,
                   "errors": 0,
                   "error_rate": 0.0
                 },
@@ -6216,33 +6216,33 @@
                 "0.5": {
                   "coverage": 0.53125,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.8950825933794926
+                  "confidence_cutoff": 0.9398574478142043
                 },
                 "0.8": {
-                  "coverage": 0.84375,
+                  "coverage": 0.8125,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.6886792778968811
+                  "confidence_cutoff": 0.7353225569523143
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 32,
-              "nll": 0.23890017515673415,
-              "acc": 0.875,
-              "ece": 0.14167053791272208,
-              "brier": 0.14524192276307057,
-              "mean_conf": 0.8435546575020636,
+              "nll": 0.17366660442192522,
+              "acc": 0.96875,
+              "ece": 0.139280642091471,
+              "brier": 0.09349301051833991,
+              "mean_conf": 0.8684549516140397,
               "confident_error_rate": 0.0,
-              "coverage_at_0_9": 0.46875,
+              "coverage_at_0_9": 0.5625,
               "accuracy_at_0_9": 1.0,
-              "coverage_at_5pct_error": 0.90625,
-              "coverage_at_1pct_error": 0.84375,
-              "aurc": 0.012148294732242174,
+              "coverage_at_5pct_error": 1.0,
+              "coverage_at_1pct_error": 0.875,
+              "aurc": 0.004072377873563219,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.0314453424979364,
+              "confidence_bias": -0.10029504838596026,
               "top_bins": {
                 "0.9": {
-                  "n": 15,
+                  "n": 18,
                   "errors": 0,
                   "error_rate": 0.0
                 },
@@ -6261,12 +6261,12 @@
                 "0.5": {
                   "coverage": 0.53125,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.8950825933794926
+                  "confidence_cutoff": 0.9398574478142043
                 },
                 "0.8": {
-                  "coverage": 0.84375,
+                  "coverage": 0.8125,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.6886792778968811
+                  "confidence_cutoff": 0.7353225569523143
                 }
               }
             },
@@ -6279,11 +6279,11 @@
             "accuracy": 1.0,
             "full_probability_metrics": {
               "n": 40,
-              "nll": 0.009534104113565645,
+              "nll": 0.010156044928884202,
               "acc": 1.0,
-              "ece": 0.009472102695785245,
-              "brier": 0.0002455458386976324,
-              "mean_conf": 0.9905278973042148,
+              "ece": 0.010082175827252016,
+              "brier": 0.0002922128455344832,
+              "mean_conf": 0.989917824172748,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -6291,7 +6291,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.009472102695785245,
+              "confidence_bias": -0.010082175827252016,
               "top_bins": {
                 "0.9": {
                   "n": 40,
@@ -6313,22 +6313,22 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9938308764165708
+                  "confidence_cutoff": 0.994282735900115
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9842986932208933
+                  "confidence_cutoff": 0.9830883529872735
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 40,
-              "nll": 0.009534104113565645,
+              "nll": 0.010156044928884202,
               "acc": 1.0,
-              "ece": 0.009472102695785245,
-              "brier": 0.0002455458386976324,
-              "mean_conf": 0.9905278973042148,
+              "ece": 0.010082175827252016,
+              "brier": 0.0002922128455344832,
+              "mean_conf": 0.989917824172748,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -6336,7 +6336,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.009472102695785245,
+              "confidence_bias": -0.010082175827252016,
               "top_bins": {
                 "0.9": {
                   "n": 40,
@@ -6358,12 +6358,12 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9938308764165708
+                  "confidence_cutoff": 0.994282735900115
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9842986932208933
+                  "confidence_cutoff": 0.9830883529872735
                 }
               }
             },
@@ -6376,27 +6376,27 @@
             "accuracy": 0.9,
             "full_probability_metrics": {
               "n": 40,
-              "nll": 0.24068925250396264,
+              "nll": 0.2578333370423695,
               "acc": 0.9,
-              "ece": 0.08521130467101881,
-              "brier": 0.13743249520879042,
-              "mean_conf": 0.869609319103461,
+              "ece": 0.08339734125029008,
+              "brier": 0.1539668312012304,
+              "mean_conf": 0.8928125003966263,
               "confident_error_rate": 0.0,
-              "coverage_at_0_9": 0.5,
+              "coverage_at_0_9": 0.6,
               "accuracy_at_0_9": 1.0,
               "coverage_at_5pct_error": 0.7,
-              "coverage_at_1pct_error": 0.675,
-              "aurc": 0.020352482782383177,
+              "coverage_at_1pct_error": 0.65,
+              "aurc": 0.02394744920629697,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.03039068089653907,
+              "confidence_bias": -0.007187499603373726,
               "top_bins": {
                 "0.9": {
-                  "n": 20,
+                  "n": 24,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.95": {
-                  "n": 14,
+                  "n": 16,
                   "errors": 0,
                   "error_rate": 0.0
                 },
@@ -6410,40 +6410,40 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9031725006679073
+                  "confidence_cutoff": 0.9320149317531109
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.9375,
-                  "confidence_cutoff": 0.773653912414944
+                  "confidence_cutoff": 0.8366438894497547
                 }
               },
-              "score_mae": 0.17163726291792777,
-              "ranked_probability_score": 0.03497160888154245
+              "score_mae": 0.16359197313913096,
+              "ranked_probability_score": 0.03911601810015416
             },
             "covered_only_probability_metrics": {
               "n": 40,
-              "nll": 0.24068925250396264,
+              "nll": 0.2578333370423695,
               "acc": 0.9,
-              "ece": 0.08521130467101881,
-              "brier": 0.13743249520879042,
-              "mean_conf": 0.869609319103461,
+              "ece": 0.08339734125029008,
+              "brier": 0.1539668312012304,
+              "mean_conf": 0.8928125003966263,
               "confident_error_rate": 0.0,
-              "coverage_at_0_9": 0.5,
+              "coverage_at_0_9": 0.6,
               "accuracy_at_0_9": 1.0,
               "coverage_at_5pct_error": 0.7,
-              "coverage_at_1pct_error": 0.675,
-              "aurc": 0.020352482782383177,
+              "coverage_at_1pct_error": 0.65,
+              "aurc": 0.02394744920629697,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.03039068089653907,
+              "confidence_bias": -0.007187499603373726,
               "top_bins": {
                 "0.9": {
-                  "n": 20,
+                  "n": 24,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.95": {
-                  "n": 14,
+                  "n": 16,
                   "errors": 0,
                   "error_rate": 0.0
                 },
@@ -6457,111 +6457,111 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9031725006679073
+                  "confidence_cutoff": 0.9320149317531109
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.9375,
-                  "confidence_cutoff": 0.773653912414944
+                  "confidence_cutoff": 0.8366438894497547
                 }
               },
-              "score_mae": 0.17163726291792777,
-              "ranked_probability_score": 0.03497160888154245
+              "score_mae": 0.16359197313913096,
+              "ranked_probability_score": 0.03911601810015416
             },
             "coverage": 1.0
           },
           "emotion": {
             "requested": 80,
             "valid_probability_rows": 80,
-            "correct": 48,
-            "accuracy": 0.6,
+            "correct": 49,
+            "accuracy": 0.6125,
             "full_probability_metrics": {
               "n": 80,
-              "nll": 1.404957749606962,
-              "acc": 0.6,
-              "ece": 0.2239237429633081,
-              "brier": 0.5924528456339091,
-              "mean_conf": 0.8068140896390107,
+              "nll": 1.3250366616414988,
+              "acc": 0.6125,
+              "ece": 0.2033677770446442,
+              "brier": 0.578098657406022,
+              "mean_conf": 0.7837558603474908,
               "confident_error_rate": 0.075,
-              "coverage_at_0_9": 0.4625,
-              "accuracy_at_0_9": 0.8378378378378378,
+              "coverage_at_0_9": 0.45,
+              "accuracy_at_0_9": 0.8333333333333334,
               "coverage_at_5pct_error": 0.025,
               "coverage_at_1pct_error": 0.025,
-              "aurc": 0.249645290423344,
-              "error_rate_at_0_9": 0.16216216216216217,
-              "confidence_bias": 0.2068140896390107,
+              "aurc": 0.2465828605574525,
+              "error_rate_at_0_9": 0.16666666666666666,
+              "confidence_bias": 0.17125586034749074,
               "top_bins": {
                 "0.9": {
-                  "n": 37,
+                  "n": 36,
                   "errors": 6,
-                  "error_rate": 0.16216216216216217
+                  "error_rate": 0.16666666666666666
                 },
                 "0.95": {
-                  "n": 31,
-                  "errors": 5,
-                  "error_rate": 0.16129032258064516
+                  "n": 24,
+                  "errors": 3,
+                  "error_rate": 0.125
                 },
                 "0.99": {
-                  "n": 10,
+                  "n": 7,
                   "errors": 2,
-                  "error_rate": 0.2
+                  "error_rate": 0.2857142857142857
                 }
               },
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
-                  "accuracy": 0.825,
-                  "confidence_cutoff": 0.8734302110272708
+                  "accuracy": 0.8,
+                  "confidence_cutoff": 0.8534444506954815
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.625,
-                  "confidence_cutoff": 0.612044678035727
+                  "accuracy": 0.65625,
+                  "confidence_cutoff": 0.5796189437721434
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 80,
-              "nll": 1.404957749606962,
-              "acc": 0.6,
-              "ece": 0.2239237429633081,
-              "brier": 0.5924528456339091,
-              "mean_conf": 0.8068140896390107,
+              "nll": 1.3250366616414988,
+              "acc": 0.6125,
+              "ece": 0.2033677770446442,
+              "brier": 0.578098657406022,
+              "mean_conf": 0.7837558603474908,
               "confident_error_rate": 0.075,
-              "coverage_at_0_9": 0.4625,
-              "accuracy_at_0_9": 0.8378378378378378,
+              "coverage_at_0_9": 0.45,
+              "accuracy_at_0_9": 0.8333333333333334,
               "coverage_at_5pct_error": 0.025,
               "coverage_at_1pct_error": 0.025,
-              "aurc": 0.249645290423344,
-              "error_rate_at_0_9": 0.16216216216216217,
-              "confidence_bias": 0.2068140896390107,
+              "aurc": 0.2465828605574525,
+              "error_rate_at_0_9": 0.16666666666666666,
+              "confidence_bias": 0.17125586034749074,
               "top_bins": {
                 "0.9": {
-                  "n": 37,
+                  "n": 36,
                   "errors": 6,
-                  "error_rate": 0.16216216216216217
+                  "error_rate": 0.16666666666666666
                 },
                 "0.95": {
-                  "n": 31,
-                  "errors": 5,
-                  "error_rate": 0.16129032258064516
+                  "n": 24,
+                  "errors": 3,
+                  "error_rate": 0.125
                 },
                 "0.99": {
-                  "n": 10,
+                  "n": 7,
                   "errors": 2,
-                  "error_rate": 0.2
+                  "error_rate": 0.2857142857142857
                 }
               },
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
-                  "accuracy": 0.825,
-                  "confidence_cutoff": 0.8734302110272708
+                  "accuracy": 0.8,
+                  "confidence_cutoff": 0.8534444506954815
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.625,
-                  "confidence_cutoff": 0.612044678035727
+                  "accuracy": 0.65625,
+                  "confidence_cutoff": 0.5796189437721434
                 }
               }
             },
@@ -6574,32 +6574,32 @@
             "accuracy": 0.7375,
             "full_probability_metrics": {
               "n": 80,
-              "nll": 0.6291131826963371,
+              "nll": 0.6466641422955599,
               "acc": 0.7375,
-              "ece": 0.09166398596467382,
-              "brier": 0.32846647117297145,
-              "mean_conf": 0.8291639859646738,
+              "ece": 0.10575227809040644,
+              "brier": 0.3381993704155934,
+              "mean_conf": 0.8239887221865458,
               "confident_error_rate": 0.025,
-              "coverage_at_0_9": 0.5625,
-              "accuracy_at_0_9": 0.9555555555555556,
-              "coverage_at_5pct_error": 0.6,
-              "coverage_at_1pct_error": 0.475,
-              "aurc": 0.06444573232402091,
-              "error_rate_at_0_9": 0.044444444444444446,
-              "confidence_bias": 0.09166398596467373,
+              "coverage_at_0_9": 0.5125,
+              "accuracy_at_0_9": 0.9512195121951219,
+              "coverage_at_5pct_error": 0.5875,
+              "coverage_at_1pct_error": 0.4125,
+              "aurc": 0.07075415004878534,
+              "error_rate_at_0_9": 0.04878048780487805,
+              "confidence_bias": 0.0864887221865458,
               "top_bins": {
                 "0.9": {
-                  "n": 45,
+                  "n": 41,
                   "errors": 2,
-                  "error_rate": 0.044444444444444446
+                  "error_rate": 0.04878048780487805
                 },
                 "0.95": {
-                  "n": 35,
+                  "n": 31,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 12,
+                  "n": 9,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -6608,43 +6608,43 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.95,
-                  "confidence_cutoff": 0.9337377421761286
+                  "confidence_cutoff": 0.9179506124738418
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.875,
-                  "confidence_cutoff": 0.6497676638791126
+                  "accuracy": 0.859375,
+                  "confidence_cutoff": 0.6428546144492293
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 80,
-              "nll": 0.6291131826963371,
+              "nll": 0.6466641422955599,
               "acc": 0.7375,
-              "ece": 0.09166398596467382,
-              "brier": 0.32846647117297145,
-              "mean_conf": 0.8291639859646738,
+              "ece": 0.10575227809040644,
+              "brier": 0.3381993704155934,
+              "mean_conf": 0.8239887221865458,
               "confident_error_rate": 0.025,
-              "coverage_at_0_9": 0.5625,
-              "accuracy_at_0_9": 0.9555555555555556,
-              "coverage_at_5pct_error": 0.6,
-              "coverage_at_1pct_error": 0.475,
-              "aurc": 0.06444573232402091,
-              "error_rate_at_0_9": 0.044444444444444446,
-              "confidence_bias": 0.09166398596467373,
+              "coverage_at_0_9": 0.5125,
+              "accuracy_at_0_9": 0.9512195121951219,
+              "coverage_at_5pct_error": 0.5875,
+              "coverage_at_1pct_error": 0.4125,
+              "aurc": 0.07075415004878534,
+              "error_rate_at_0_9": 0.04878048780487805,
+              "confidence_bias": 0.0864887221865458,
               "top_bins": {
                 "0.9": {
-                  "n": 45,
+                  "n": 41,
                   "errors": 2,
-                  "error_rate": 0.044444444444444446
+                  "error_rate": 0.04878048780487805
                 },
                 "0.95": {
-                  "n": 35,
+                  "n": 31,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 12,
+                  "n": 9,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -6653,12 +6653,12 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.95,
-                  "confidence_cutoff": 0.9337377421761286
+                  "confidence_cutoff": 0.9179506124738418
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.875,
-                  "confidence_cutoff": 0.6497676638791126
+                  "accuracy": 0.859375,
+                  "confidence_cutoff": 0.6428546144492293
                 }
               }
             },
@@ -6671,32 +6671,32 @@
             "accuracy": 0.53,
             "full_probability_metrics": {
               "n": 200,
-              "nll": 1.3194872971476064,
+              "nll": 1.3111522675312284,
               "acc": 0.53,
-              "ece": 0.0712459960944238,
-              "brier": 0.57596282275509,
-              "mean_conf": 0.5988417766167529,
-              "confident_error_rate": 0.01,
-              "coverage_at_0_9": 0.245,
-              "accuracy_at_0_9": 0.9591836734693877,
-              "coverage_at_5pct_error": 0.265,
-              "coverage_at_1pct_error": 0.185,
-              "aurc": 0.22156004538018903,
-              "error_rate_at_0_9": 0.04081632653061224,
-              "confidence_bias": 0.06884177661675284,
+              "ece": 0.08739805659958974,
+              "brier": 0.5728987396461885,
+              "mean_conf": 0.5899912710608897,
+              "confident_error_rate": 0.005,
+              "coverage_at_0_9": 0.23,
+              "accuracy_at_0_9": 0.9782608695652174,
+              "coverage_at_5pct_error": 0.25,
+              "coverage_at_1pct_error": 0.19,
+              "aurc": 0.21980746104696547,
+              "error_rate_at_0_9": 0.021739130434782608,
+              "confidence_bias": 0.05999127106088964,
               "top_bins": {
                 "0.9": {
-                  "n": 49,
-                  "errors": 2,
-                  "error_rate": 0.04081632653061224
+                  "n": 46,
+                  "errors": 1,
+                  "error_rate": 0.021739130434782608
                 },
                 "0.95": {
-                  "n": 36,
+                  "n": 29,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 9,
+                  "n": 6,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -6704,44 +6704,44 @@
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
-                  "accuracy": 0.73,
-                  "confidence_cutoff": 0.5902113030216032
+                  "accuracy": 0.75,
+                  "confidence_cutoff": 0.57725688415424
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.625,
-                  "confidence_cutoff": 0.2801405568637054
+                  "confidence_cutoff": 0.26950159367730925
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 200,
-              "nll": 1.3194872971476064,
+              "nll": 1.3111522675312284,
               "acc": 0.53,
-              "ece": 0.0712459960944238,
-              "brier": 0.57596282275509,
-              "mean_conf": 0.5988417766167529,
-              "confident_error_rate": 0.01,
-              "coverage_at_0_9": 0.245,
-              "accuracy_at_0_9": 0.9591836734693877,
-              "coverage_at_5pct_error": 0.265,
-              "coverage_at_1pct_error": 0.185,
-              "aurc": 0.22156004538018903,
-              "error_rate_at_0_9": 0.04081632653061224,
-              "confidence_bias": 0.06884177661675284,
+              "ece": 0.08739805659958974,
+              "brier": 0.5728987396461885,
+              "mean_conf": 0.5899912710608897,
+              "confident_error_rate": 0.005,
+              "coverage_at_0_9": 0.23,
+              "accuracy_at_0_9": 0.9782608695652174,
+              "coverage_at_5pct_error": 0.25,
+              "coverage_at_1pct_error": 0.19,
+              "aurc": 0.21980746104696547,
+              "error_rate_at_0_9": 0.021739130434782608,
+              "confidence_bias": 0.05999127106088964,
               "top_bins": {
                 "0.9": {
-                  "n": 49,
-                  "errors": 2,
-                  "error_rate": 0.04081632653061224
+                  "n": 46,
+                  "errors": 1,
+                  "error_rate": 0.021739130434782608
                 },
                 "0.95": {
-                  "n": 36,
+                  "n": 29,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 9,
+                  "n": 6,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -6749,13 +6749,13 @@
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
-                  "accuracy": 0.73,
-                  "confidence_cutoff": 0.5902113030216032
+                  "accuracy": 0.75,
+                  "confidence_cutoff": 0.57725688415424
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.625,
-                  "confidence_cutoff": 0.2801405568637054
+                  "confidence_cutoff": 0.26950159367730925
                 }
               }
             },
@@ -6768,29 +6768,29 @@
             "accuracy": 0.9,
             "full_probability_metrics": {
               "n": 80,
-              "nll": 0.2774054133452753,
+              "nll": 0.28314251837212323,
               "acc": 0.9,
-              "ece": 0.04500883219029569,
-              "brier": 0.1577162228589053,
-              "mean_conf": 0.9337538243395758,
-              "confident_error_rate": 0.05,
-              "coverage_at_0_9": 0.8125,
-              "accuracy_at_0_9": 0.9384615384615385,
-              "coverage_at_5pct_error": 0.7875,
-              "coverage_at_1pct_error": 0.35,
-              "aurc": 0.027700444825567955,
-              "error_rate_at_0_9": 0.06153846153846154,
-              "confidence_bias": 0.033753824339575744,
+              "ece": 0.06247439765758463,
+              "brier": 0.15885879977500503,
+              "mean_conf": 0.9412949163036171,
+              "confident_error_rate": 0.0625,
+              "coverage_at_0_9": 0.8375,
+              "accuracy_at_0_9": 0.9253731343283582,
+              "coverage_at_5pct_error": 0.775,
+              "coverage_at_1pct_error": 0.2875,
+              "aurc": 0.030912620564059478,
+              "error_rate_at_0_9": 0.07462686567164178,
+              "confidence_bias": 0.041294916303617035,
               "top_bins": {
                 "0.9": {
-                  "n": 65,
-                  "errors": 4,
-                  "error_rate": 0.06153846153846154
+                  "n": 67,
+                  "errors": 5,
+                  "error_rate": 0.07462686567164178
                 },
                 "0.95": {
-                  "n": 59,
+                  "n": 60,
                   "errors": 2,
-                  "error_rate": 0.03389830508474576
+                  "error_rate": 0.03333333333333333
                 },
                 "0.99": {
                   "n": 2,
@@ -6802,40 +6802,40 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.975,
-                  "confidence_cutoff": 0.9777894020080566
+                  "confidence_cutoff": 0.9773598813644135
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.9375,
-                  "confidence_cutoff": 0.9301801601105976
+                  "confidence_cutoff": 0.9395250222016129
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 80,
-              "nll": 0.2774054133452753,
+              "nll": 0.28314251837212323,
               "acc": 0.9,
-              "ece": 0.04500883219029569,
-              "brier": 0.1577162228589053,
-              "mean_conf": 0.9337538243395758,
-              "confident_error_rate": 0.05,
-              "coverage_at_0_9": 0.8125,
-              "accuracy_at_0_9": 0.9384615384615385,
-              "coverage_at_5pct_error": 0.7875,
-              "coverage_at_1pct_error": 0.35,
-              "aurc": 0.027700444825567955,
-              "error_rate_at_0_9": 0.06153846153846154,
-              "confidence_bias": 0.033753824339575744,
+              "ece": 0.06247439765758463,
+              "brier": 0.15885879977500503,
+              "mean_conf": 0.9412949163036171,
+              "confident_error_rate": 0.0625,
+              "coverage_at_0_9": 0.8375,
+              "accuracy_at_0_9": 0.9253731343283582,
+              "coverage_at_5pct_error": 0.775,
+              "coverage_at_1pct_error": 0.2875,
+              "aurc": 0.030912620564059478,
+              "error_rate_at_0_9": 0.07462686567164178,
+              "confidence_bias": 0.041294916303617035,
               "top_bins": {
                 "0.9": {
-                  "n": 65,
-                  "errors": 4,
-                  "error_rate": 0.06153846153846154
+                  "n": 67,
+                  "errors": 5,
+                  "error_rate": 0.07462686567164178
                 },
                 "0.95": {
-                  "n": 59,
+                  "n": 60,
                   "errors": 2,
-                  "error_rate": 0.03389830508474576
+                  "error_rate": 0.03333333333333333
                 },
                 "0.99": {
                   "n": 2,
@@ -6847,12 +6847,12 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.975,
-                  "confidence_cutoff": 0.9777894020080566
+                  "confidence_cutoff": 0.9773598813644135
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.9375,
-                  "confidence_cutoff": 0.9301801601105976
+                  "confidence_cutoff": 0.9395250222016129
                 }
               }
             },
@@ -6865,91 +6865,91 @@
             "accuracy": 0.9125,
             "full_probability_metrics": {
               "n": 80,
-              "nll": 0.21752679625446542,
+              "nll": 0.2161076588398619,
               "acc": 0.9125,
-              "ece": 0.03901376668147833,
-              "brier": 0.11730868195084412,
-              "mean_conf": 0.9352517996137696,
+              "ece": 0.038010106954782505,
+              "brier": 0.11745397306383834,
+              "mean_conf": 0.9376866478309165,
               "confident_error_rate": 0.025,
-              "coverage_at_0_9": 0.8125,
-              "accuracy_at_0_9": 0.9692307692307692,
-              "coverage_at_5pct_error": 0.9125,
-              "coverage_at_1pct_error": 0.0625,
-              "aurc": 0.040943653344357525,
-              "error_rate_at_0_9": 0.03076923076923077,
-              "confidence_bias": 0.022751799613769608,
+              "coverage_at_0_9": 0.8375,
+              "accuracy_at_0_9": 0.9701492537313433,
+              "coverage_at_5pct_error": 0.925,
+              "coverage_at_1pct_error": 0.075,
+              "aurc": 0.039172223012800614,
+              "error_rate_at_0_9": 0.029850746268656716,
+              "confidence_bias": 0.02518664783091651,
               "top_bins": {
                 "0.9": {
-                  "n": 65,
+                  "n": 67,
                   "errors": 2,
-                  "error_rate": 0.03076923076923077
+                  "error_rate": 0.029850746268656716
                 },
                 "0.95": {
-                  "n": 63,
+                  "n": 62,
                   "errors": 2,
-                  "error_rate": 0.031746031746031744
+                  "error_rate": 0.03225806451612903
                 },
                 "0.99": {
-                  "n": 28,
+                  "n": 26,
                   "errors": 1,
-                  "error_rate": 0.03571428571428571
+                  "error_rate": 0.038461538461538464
                 }
               },
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.975,
-                  "confidence_cutoff": 0.9841503702581936
+                  "confidence_cutoff": 0.9851611733370504
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.96875,
-                  "confidence_cutoff": 0.9355465034314021
+                  "confidence_cutoff": 0.9458952863255798
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 80,
-              "nll": 0.21752679625446542,
+              "nll": 0.2161076588398619,
               "acc": 0.9125,
-              "ece": 0.03901376668147833,
-              "brier": 0.11730868195084412,
-              "mean_conf": 0.9352517996137696,
+              "ece": 0.038010106954782505,
+              "brier": 0.11745397306383834,
+              "mean_conf": 0.9376866478309165,
               "confident_error_rate": 0.025,
-              "coverage_at_0_9": 0.8125,
-              "accuracy_at_0_9": 0.9692307692307692,
-              "coverage_at_5pct_error": 0.9125,
-              "coverage_at_1pct_error": 0.0625,
-              "aurc": 0.040943653344357525,
-              "error_rate_at_0_9": 0.03076923076923077,
-              "confidence_bias": 0.022751799613769608,
+              "coverage_at_0_9": 0.8375,
+              "accuracy_at_0_9": 0.9701492537313433,
+              "coverage_at_5pct_error": 0.925,
+              "coverage_at_1pct_error": 0.075,
+              "aurc": 0.039172223012800614,
+              "error_rate_at_0_9": 0.029850746268656716,
+              "confidence_bias": 0.02518664783091651,
               "top_bins": {
                 "0.9": {
-                  "n": 65,
+                  "n": 67,
                   "errors": 2,
-                  "error_rate": 0.03076923076923077
+                  "error_rate": 0.029850746268656716
                 },
                 "0.95": {
-                  "n": 63,
+                  "n": 62,
                   "errors": 2,
-                  "error_rate": 0.031746031746031744
+                  "error_rate": 0.03225806451612903
                 },
                 "0.99": {
-                  "n": 28,
+                  "n": 26,
                   "errors": 1,
-                  "error_rate": 0.03571428571428571
+                  "error_rate": 0.038461538461538464
                 }
               },
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.975,
-                  "confidence_cutoff": 0.9841503702581936
+                  "confidence_cutoff": 0.9851611733370504
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.96875,
-                  "confidence_cutoff": 0.9355465034314021
+                  "confidence_cutoff": 0.9458952863255798
                 }
               }
             },
@@ -6962,11 +6962,11 @@
             "accuracy": 1.0,
             "full_probability_metrics": {
               "n": 80,
-              "nll": 0.004514713334100866,
+              "nll": 0.006236348098711358,
               "acc": 1.0,
-              "ece": 0.004458343565561407,
-              "brier": 0.00020475202026520086,
-              "mean_conf": 0.9955416564344386,
+              "ece": 0.006149372309216639,
+              "brier": 0.00030747770792780715,
+              "mean_conf": 0.9938506276907834,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -6974,7 +6974,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.004458343565561407,
+              "confidence_bias": -0.006149372309216639,
               "top_bins": {
                 "0.9": {
                   "n": 80,
@@ -6982,12 +6982,12 @@
                   "error_rate": 0.0
                 },
                 "0.95": {
-                  "n": 78,
+                  "n": 79,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 70,
+                  "n": 69,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -6996,22 +6996,22 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9987595494699697
+                  "confidence_cutoff": 0.9978451209130044
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9963549793899044
+                  "confidence_cutoff": 0.9938674921230647
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 80,
-              "nll": 0.004514713334100866,
+              "nll": 0.006236348098711358,
               "acc": 1.0,
-              "ece": 0.004458343565561407,
-              "brier": 0.00020475202026520086,
-              "mean_conf": 0.9955416564344386,
+              "ece": 0.006149372309216639,
+              "brier": 0.00030747770792780715,
+              "mean_conf": 0.9938506276907834,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -7019,7 +7019,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.004458343565561407,
+              "confidence_bias": -0.006149372309216639,
               "top_bins": {
                 "0.9": {
                   "n": 80,
@@ -7027,12 +7027,12 @@
                   "error_rate": 0.0
                 },
                 "0.95": {
-                  "n": 78,
+                  "n": 79,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 70,
+                  "n": 69,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -7041,12 +7041,12 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9987595494699697
+                  "confidence_cutoff": 0.9978451209130044
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9963549793899044
+                  "confidence_cutoff": 0.9938674921230647
                 }
               }
             },
@@ -7055,31 +7055,31 @@
           "tweet_offensive": {
             "requested": 80,
             "valid_probability_rows": 80,
-            "correct": 59,
-            "accuracy": 0.7375,
+            "correct": 60,
+            "accuracy": 0.75,
             "full_probability_metrics": {
               "n": 80,
-              "nll": 0.5266044697812845,
-              "acc": 0.7375,
-              "ece": 0.10983391602437403,
-              "brier": 0.3587274567128259,
-              "mean_conf": 0.8256957228540391,
+              "nll": 0.47556818540480367,
+              "acc": 0.75,
+              "ece": 0.07643617030229378,
+              "brier": 0.3199264336587134,
+              "mean_conf": 0.8258139805835978,
               "confident_error_rate": 0.0375,
-              "coverage_at_0_9": 0.425,
-              "accuracy_at_0_9": 0.9117647058823529,
-              "coverage_at_5pct_error": 0.35,
-              "coverage_at_1pct_error": 0.3,
-              "aurc": 0.11516294025516034,
-              "error_rate_at_0_9": 0.08823529411764706,
-              "confidence_bias": 0.0881957228540391,
+              "coverage_at_0_9": 0.4125,
+              "accuracy_at_0_9": 0.9090909090909091,
+              "coverage_at_5pct_error": 0.3625,
+              "coverage_at_1pct_error": 0.35,
+              "aurc": 0.09222008593150528,
+              "error_rate_at_0_9": 0.09090909090909091,
+              "confidence_bias": 0.07581398058359778,
               "top_bins": {
                 "0.9": {
-                  "n": 34,
+                  "n": 33,
                   "errors": 3,
-                  "error_rate": 0.08823529411764706
+                  "error_rate": 0.09090909090909091
                 },
                 "0.95": {
-                  "n": 23,
+                  "n": 24,
                   "errors": 0,
                   "error_rate": 0.0
                 },
@@ -7093,38 +7093,38 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.9,
-                  "confidence_cutoff": 0.8725079970057013
+                  "confidence_cutoff": 0.8609209785002616
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.765625,
-                  "confidence_cutoff": 0.6975352375964892
+                  "accuracy": 0.828125,
+                  "confidence_cutoff": 0.6948713064193726
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 80,
-              "nll": 0.5266044697812845,
-              "acc": 0.7375,
-              "ece": 0.10983391602437403,
-              "brier": 0.3587274567128259,
-              "mean_conf": 0.8256957228540391,
+              "nll": 0.47556818540480367,
+              "acc": 0.75,
+              "ece": 0.07643617030229378,
+              "brier": 0.3199264336587134,
+              "mean_conf": 0.8258139805835978,
               "confident_error_rate": 0.0375,
-              "coverage_at_0_9": 0.425,
-              "accuracy_at_0_9": 0.9117647058823529,
-              "coverage_at_5pct_error": 0.35,
-              "coverage_at_1pct_error": 0.3,
-              "aurc": 0.11516294025516034,
-              "error_rate_at_0_9": 0.08823529411764706,
-              "confidence_bias": 0.0881957228540391,
+              "coverage_at_0_9": 0.4125,
+              "accuracy_at_0_9": 0.9090909090909091,
+              "coverage_at_5pct_error": 0.3625,
+              "coverage_at_1pct_error": 0.35,
+              "aurc": 0.09222008593150528,
+              "error_rate_at_0_9": 0.09090909090909091,
+              "confidence_bias": 0.07581398058359778,
               "top_bins": {
                 "0.9": {
-                  "n": 34,
+                  "n": 33,
                   "errors": 3,
-                  "error_rate": 0.08823529411764706
+                  "error_rate": 0.09090909090909091
                 },
                 "0.95": {
-                  "n": 23,
+                  "n": 24,
                   "errors": 0,
                   "error_rate": 0.0
                 },
@@ -7138,12 +7138,12 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.9,
-                  "confidence_cutoff": 0.8725079970057013
+                  "confidence_cutoff": 0.8609209785002616
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.765625,
-                  "confidence_cutoff": 0.6975352375964892
+                  "accuracy": 0.828125,
+                  "confidence_cutoff": 0.6948713064193726
                 }
               }
             },
@@ -7156,11 +7156,11 @@
             "accuracy": 1.0,
             "full_probability_metrics": {
               "n": 10,
-              "nll": 0.006876414004309999,
+              "nll": 0.007841155789564139,
               "acc": 1.0,
-              "ece": 0.0068486883882732386,
-              "brier": 0.00011023848598835957,
-              "mean_conf": 0.9931513116117268,
+              "ece": 0.007806247293806301,
+              "brier": 0.00013872505147745316,
+              "mean_conf": 0.9921937527061937,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -7168,7 +7168,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.0068486883882732386,
+              "confidence_bias": -0.007806247293806301,
               "top_bins": {
                 "0.9": {
                   "n": 10,
@@ -7181,7 +7181,7 @@
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 9,
+                  "n": 6,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -7190,22 +7190,22 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9946927960913962
+                  "confidence_cutoff": 0.9939010793872055
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9907611704497484
+                  "confidence_cutoff": 0.9896593136903282
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 10,
-              "nll": 0.006876414004309999,
+              "nll": 0.007841155789564139,
               "acc": 1.0,
-              "ece": 0.0068486883882732386,
-              "brier": 0.00011023848598835957,
-              "mean_conf": 0.9931513116117268,
+              "ece": 0.007806247293806301,
+              "brier": 0.00013872505147745316,
+              "mean_conf": 0.9921937527061937,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -7213,7 +7213,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.0068486883882732386,
+              "confidence_bias": -0.007806247293806301,
               "top_bins": {
                 "0.9": {
                   "n": 10,
@@ -7226,7 +7226,7 @@
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 9,
+                  "n": 6,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -7235,12 +7235,12 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9946927960913962
+                  "confidence_cutoff": 0.9939010793872055
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9907611704497484
+                  "confidence_cutoff": 0.9896593136903282
                 }
               }
             },
@@ -7253,11 +7253,11 @@
             "accuracy": 1.0,
             "full_probability_metrics": {
               "n": 10,
-              "nll": 0.010054244756134334,
+              "nll": 0.0107929267557592,
               "acc": 1.0,
-              "ece": 0.009977028457380022,
-              "brier": 0.0003052461514864493,
-              "mean_conf": 0.99002297154262,
+              "ece": 0.010701462639340464,
+              "brier": 0.00036120660545170313,
+              "mean_conf": 0.9892985373606595,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -7265,7 +7265,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.009977028457380022,
+              "confidence_bias": -0.010701462639340464,
               "top_bins": {
                 "0.9": {
                   "n": 10,
@@ -7287,22 +7287,22 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9966472206636919
+                  "confidence_cutoff": 0.9969034000449554
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9836865232385298
+                  "confidence_cutoff": 0.9816944573491596
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 10,
-              "nll": 0.010054244756134334,
+              "nll": 0.0107929267557592,
               "acc": 1.0,
-              "ece": 0.009977028457380022,
-              "brier": 0.0003052461514864493,
-              "mean_conf": 0.99002297154262,
+              "ece": 0.010701462639340464,
+              "brier": 0.00036120660545170313,
+              "mean_conf": 0.9892985373606595,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -7310,7 +7310,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.009977028457380022,
+              "confidence_bias": -0.010701462639340464,
               "top_bins": {
                 "0.9": {
                   "n": 10,
@@ -7332,12 +7332,12 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9966472206636919
+                  "confidence_cutoff": 0.9969034000449554
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9836865232385298
+                  "confidence_cutoff": 0.9816944573491596
                 }
               }
             },
@@ -7346,31 +7346,31 @@
           "unknowable_control_deadline": {
             "requested": 10,
             "valid_probability_rows": 10,
-            "correct": 10,
-            "accuracy": 1.0,
+            "correct": 8,
+            "accuracy": 0.8,
             "full_probability_metrics": {
               "n": 10,
-              "nll": 0.20875349707626656,
-              "acc": 1.0,
-              "ece": 0.16950983158479752,
-              "brier": 0.10697640035089204,
-              "mean_conf": 0.8304901684152025,
+              "nll": 0.2138260506584651,
+              "acc": 0.8,
+              "ece": 0.15741611133328517,
+              "brier": 0.1276425479499806,
+              "mean_conf": 0.856580016184909,
               "confident_error_rate": 0.0,
-              "coverage_at_0_9": 0.5,
+              "coverage_at_0_9": 0.6,
               "accuracy_at_0_9": 1.0,
-              "coverage_at_5pct_error": 1.0,
-              "coverage_at_1pct_error": 1.0,
-              "aurc": 0.0,
+              "coverage_at_5pct_error": 0.8,
+              "coverage_at_1pct_error": 0.8,
+              "aurc": 0.03111111111111111,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.1695098315847975,
+              "confidence_bias": 0.05658001618490893,
               "top_bins": {
                 "0.9": {
-                  "n": 5,
+                  "n": 6,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.95": {
-                  "n": 3,
+                  "n": 4,
                   "errors": 0,
                   "error_rate": 0.0
                 },
@@ -7384,40 +7384,40 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9212612921563369
+                  "confidence_cutoff": 0.9426117959877142
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.7564743954513996
+                  "confidence_cutoff": 0.8415699130530643
                 }
               },
-              "score_mae": 0.1696031781535504,
-              "ranked_probability_score": 0.027307549723045547
+              "score_mae": 0.16071754616245987,
+              "ranked_probability_score": 0.03275971187271286
             },
             "covered_only_probability_metrics": {
               "n": 10,
-              "nll": 0.20875349707626656,
-              "acc": 1.0,
-              "ece": 0.16950983158479752,
-              "brier": 0.10697640035089204,
-              "mean_conf": 0.8304901684152025,
+              "nll": 0.2138260506584651,
+              "acc": 0.8,
+              "ece": 0.15741611133328517,
+              "brier": 0.1276425479499806,
+              "mean_conf": 0.856580016184909,
               "confident_error_rate": 0.0,
-              "coverage_at_0_9": 0.5,
+              "coverage_at_0_9": 0.6,
               "accuracy_at_0_9": 1.0,
-              "coverage_at_5pct_error": 1.0,
-              "coverage_at_1pct_error": 1.0,
-              "aurc": 0.0,
+              "coverage_at_5pct_error": 0.8,
+              "coverage_at_1pct_error": 0.8,
+              "aurc": 0.03111111111111111,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.1695098315847975,
+              "confidence_bias": 0.05658001618490893,
               "top_bins": {
                 "0.9": {
-                  "n": 5,
+                  "n": 6,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.95": {
-                  "n": 3,
+                  "n": 4,
                   "errors": 0,
                   "error_rate": 0.0
                 },
@@ -7431,16 +7431,16 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9212612921563369
+                  "confidence_cutoff": 0.9426117959877142
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.7564743954513996
+                  "confidence_cutoff": 0.8415699130530643
                 }
               },
-              "score_mae": 0.1696031781535504,
-              "ranked_probability_score": 0.027307549723045547
+              "score_mae": 0.16071754616245987,
+              "ranked_probability_score": 0.03275971187271286
             },
             "coverage": 1.0
           },
@@ -7451,11 +7451,11 @@
             "accuracy": 1.0,
             "full_probability_metrics": {
               "n": 10,
-              "nll": 0.07820821810037346,
+              "nll": 0.07005694097555375,
               "acc": 1.0,
-              "ece": 0.05979035227629538,
-              "brier": 0.04652092623743477,
-              "mean_conf": 0.9402096477237045,
+              "ece": 0.0540813816797954,
+              "brier": 0.04110395968042234,
+              "mean_conf": 0.9459186183202046,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.9,
               "accuracy_at_0_9": 1.0,
@@ -7463,7 +7463,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.05979035227629548,
+              "confidence_bias": -0.05408138167979537,
               "top_bins": {
                 "0.9": {
                   "n": 9,
@@ -7476,7 +7476,7 @@
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 7,
+                  "n": 6,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -7485,24 +7485,24 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9956810330664218
+                  "confidence_cutoff": 0.993819476814992
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9593766863298722
+                  "confidence_cutoff": 0.9798565648382314
                 }
               },
-              "score_mae": 0.0541174703260209,
-              "ranked_probability_score": 0.011129613417488322
+              "score_mae": 0.0487152458265909,
+              "ranked_probability_score": 0.00975869637261514
             },
             "covered_only_probability_metrics": {
               "n": 10,
-              "nll": 0.07820821810037346,
+              "nll": 0.07005694097555375,
               "acc": 1.0,
-              "ece": 0.05979035227629538,
-              "brier": 0.04652092623743477,
-              "mean_conf": 0.9402096477237045,
+              "ece": 0.0540813816797954,
+              "brier": 0.04110395968042234,
+              "mean_conf": 0.9459186183202046,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.9,
               "accuracy_at_0_9": 1.0,
@@ -7510,7 +7510,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.05979035227629548,
+              "confidence_bias": -0.05408138167979537,
               "top_bins": {
                 "0.9": {
                   "n": 9,
@@ -7523,7 +7523,7 @@
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 7,
+                  "n": 6,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -7532,16 +7532,16 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9956810330664218
+                  "confidence_cutoff": 0.993819476814992
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9593766863298722
+                  "confidence_cutoff": 0.9798565648382314
                 }
               },
-              "score_mae": 0.0541174703260209,
-              "ranked_probability_score": 0.011129613417488322
+              "score_mae": 0.0487152458265909,
+              "ranked_probability_score": 0.00975869637261514
             },
             "coverage": 1.0
           },
@@ -7552,11 +7552,11 @@
             "accuracy": 1.0,
             "full_probability_metrics": {
               "n": 10,
-              "nll": 0.10634568298556582,
+              "nll": 0.10819644087102691,
               "acc": 1.0,
-              "ece": 0.09157236085416048,
-              "brier": 0.043689973951603296,
-              "mean_conf": 0.9084276391458393,
+              "ece": 0.0954442527194344,
+              "brier": 0.039610537719924295,
+              "mean_conf": 0.9045557472805654,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.7,
               "accuracy_at_0_9": 1.0,
@@ -7564,7 +7564,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.09157236085416065,
+              "confidence_bias": -0.09544425271943457,
               "top_bins": {
                 "0.9": {
                   "n": 7,
@@ -7577,31 +7577,31 @@
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 2,
+                  "n": 0,
                   "errors": 0,
-                  "error_rate": 0.0
+                  "error_rate": null
                 }
               },
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9648931116321414
+                  "confidence_cutoff": 0.9656626177441163
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.8787499248169464
+                  "confidence_cutoff": 0.881241046212756
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 10,
-              "nll": 0.10634568298556582,
+              "nll": 0.10819644087102691,
               "acc": 1.0,
-              "ece": 0.09157236085416048,
-              "brier": 0.043689973951603296,
-              "mean_conf": 0.9084276391458393,
+              "ece": 0.0954442527194344,
+              "brier": 0.039610537719924295,
+              "mean_conf": 0.9045557472805654,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.7,
               "accuracy_at_0_9": 1.0,
@@ -7609,7 +7609,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.09157236085416065,
+              "confidence_bias": -0.09544425271943457,
               "top_bins": {
                 "0.9": {
                   "n": 7,
@@ -7622,21 +7622,21 @@
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 2,
+                  "n": 0,
                   "errors": 0,
-                  "error_rate": 0.0
+                  "error_rate": null
                 }
               },
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9648931116321414
+                  "confidence_cutoff": 0.9656626177441163
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.8787499248169464
+                  "confidence_cutoff": 0.881241046212756
                 }
               }
             },
@@ -7649,19 +7649,19 @@
             "accuracy": 0.6,
             "full_probability_metrics": {
               "n": 10,
-              "nll": 0.7842587765663349,
+              "nll": 0.8939128624245644,
               "acc": 0.6,
-              "ece": 0.3758565683544252,
-              "brier": 0.5730022276631428,
-              "mean_conf": 0.8042052849775017,
+              "ece": 0.39838756334240605,
+              "brier": 0.6386225586932958,
+              "mean_conf": 0.8386086717206818,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.2,
               "accuracy_at_0_9": 1.0,
-              "coverage_at_5pct_error": 0.3,
-              "coverage_at_1pct_error": 0.3,
-              "aurc": 0.25563492063492066,
+              "coverage_at_5pct_error": 0.2,
+              "coverage_at_1pct_error": 0.2,
+              "aurc": 0.288968253968254,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": 0.20420528497750168,
+              "confidence_bias": 0.2386086717206818,
               "top_bins": {
                 "0.9": {
                   "n": 2,
@@ -7683,30 +7683,30 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.8,
-                  "confidence_cutoff": 0.8447330845812883
+                  "confidence_cutoff": 0.8853410746258817
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.7434658255258695
+                  "confidence_cutoff": 0.8282266608409089
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 10,
-              "nll": 0.7842587765663349,
+              "nll": 0.8939128624245644,
               "acc": 0.6,
-              "ece": 0.3758565683544252,
-              "brier": 0.5730022276631428,
-              "mean_conf": 0.8042052849775017,
+              "ece": 0.39838756334240605,
+              "brier": 0.6386225586932958,
+              "mean_conf": 0.8386086717206818,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.2,
               "accuracy_at_0_9": 1.0,
-              "coverage_at_5pct_error": 0.3,
-              "coverage_at_1pct_error": 0.3,
-              "aurc": 0.25563492063492066,
+              "coverage_at_5pct_error": 0.2,
+              "coverage_at_1pct_error": 0.2,
+              "aurc": 0.288968253968254,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": 0.20420528497750168,
+              "confidence_bias": 0.2386086717206818,
               "top_bins": {
                 "0.9": {
                   "n": 2,
@@ -7728,12 +7728,12 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.8,
-                  "confidence_cutoff": 0.8447330845812883
+                  "confidence_cutoff": 0.8853410746258817
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.7434658255258695
+                  "confidence_cutoff": 0.8282266608409089
                 }
               }
             },
@@ -7746,11 +7746,11 @@
             "accuracy": 0.9,
             "full_probability_metrics": {
               "n": 10,
-              "nll": 0.3877484850113062,
+              "nll": 0.3878790410817737,
               "acc": 0.9,
-              "ece": 0.1957660562559332,
-              "brier": 0.23322717018981343,
-              "mean_conf": 0.8058850082032224,
+              "ece": 0.20366310890419706,
+              "brier": 0.23894788485153878,
+              "mean_conf": 0.8019263309455292,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.3,
               "accuracy_at_0_9": 1.0,
@@ -7758,7 +7758,7 @@
               "coverage_at_1pct_error": 0.5,
               "aurc": 0.06456349206349207,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.09411499179677763,
+              "confidence_bias": -0.09807366905447079,
               "top_bins": {
                 "0.9": {
                   "n": 3,
@@ -7780,24 +7780,24 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.8255768464444774
+                  "confidence_cutoff": 0.8513163369222139
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.875,
-                  "confidence_cutoff": 0.734157871218888
+                  "confidence_cutoff": 0.7305872644461858
                 }
               },
-              "score_mae": 0.2481150404394402,
-              "ranked_probability_score": 0.05778942095101965
+              "score_mae": 0.2577528085044144,
+              "ranked_probability_score": 0.05968224629533503
             },
             "covered_only_probability_metrics": {
               "n": 10,
-              "nll": 0.3877484850113062,
+              "nll": 0.3878790410817737,
               "acc": 0.9,
-              "ece": 0.1957660562559332,
-              "brier": 0.23322717018981343,
-              "mean_conf": 0.8058850082032224,
+              "ece": 0.20366310890419706,
+              "brier": 0.23894788485153878,
+              "mean_conf": 0.8019263309455292,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.3,
               "accuracy_at_0_9": 1.0,
@@ -7805,7 +7805,7 @@
               "coverage_at_1pct_error": 0.5,
               "aurc": 0.06456349206349207,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.09411499179677763,
+              "confidence_bias": -0.09807366905447079,
               "top_bins": {
                 "0.9": {
                   "n": 3,
@@ -7827,16 +7827,16 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.8255768464444774
+                  "confidence_cutoff": 0.8513163369222139
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.875,
-                  "confidence_cutoff": 0.734157871218888
+                  "confidence_cutoff": 0.7305872644461858
                 }
               },
-              "score_mae": 0.2481150404394402,
-              "ranked_probability_score": 0.05778942095101965
+              "score_mae": 0.2577528085044144,
+              "ranked_probability_score": 0.05968224629533503
             },
             "coverage": 1.0
           },
@@ -7847,29 +7847,29 @@
             "accuracy": 0.7,
             "full_probability_metrics": {
               "n": 10,
-              "nll": 0.753852089972006,
+              "nll": 0.7470384623310196,
               "acc": 0.7,
-              "ece": 0.21322026889904205,
-              "brier": 0.44008690936983347,
-              "mean_conf": 0.9132202688990422,
+              "ece": 0.22029693644423792,
+              "brier": 0.4309736445646065,
+              "mean_conf": 0.9202969364442379,
               "confident_error_rate": 0.1,
-              "coverage_at_0_9": 0.6,
-              "accuracy_at_0_9": 0.8333333333333334,
+              "coverage_at_0_9": 0.7,
+              "accuracy_at_0_9": 0.8571428571428571,
               "coverage_at_5pct_error": 0.4,
               "coverage_at_1pct_error": 0.4,
               "aurc": 0.12817460317460316,
-              "error_rate_at_0_9": 0.16666666666666666,
-              "confidence_bias": 0.21322026889904222,
+              "error_rate_at_0_9": 0.14285714285714285,
+              "confidence_bias": 0.22029693644423798,
               "top_bins": {
                 "0.9": {
-                  "n": 6,
+                  "n": 7,
                   "errors": 1,
-                  "error_rate": 0.16666666666666666
+                  "error_rate": 0.14285714285714285
                 },
                 "0.95": {
-                  "n": 5,
+                  "n": 6,
                   "errors": 1,
-                  "error_rate": 0.2
+                  "error_rate": 0.16666666666666666
                 },
                 "0.99": {
                   "n": 3,
@@ -7881,42 +7881,42 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.8,
-                  "confidence_cutoff": 0.9756218860055605
+                  "confidence_cutoff": 0.9773512719149
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.75,
-                  "confidence_cutoff": 0.8950916416154385
+                  "confidence_cutoff": 0.8938853735717587
                 }
               },
-              "score_mae": 0.28450960978161083,
-              "ranked_probability_score": 0.10937006220858798
+              "score_mae": 0.2732574139836782,
+              "ranked_probability_score": 0.10706854764238136
             },
             "covered_only_probability_metrics": {
               "n": 10,
-              "nll": 0.753852089972006,
+              "nll": 0.7470384623310196,
               "acc": 0.7,
-              "ece": 0.21322026889904205,
-              "brier": 0.44008690936983347,
-              "mean_conf": 0.9132202688990422,
+              "ece": 0.22029693644423792,
+              "brier": 0.4309736445646065,
+              "mean_conf": 0.9202969364442379,
               "confident_error_rate": 0.1,
-              "coverage_at_0_9": 0.6,
-              "accuracy_at_0_9": 0.8333333333333334,
+              "coverage_at_0_9": 0.7,
+              "accuracy_at_0_9": 0.8571428571428571,
               "coverage_at_5pct_error": 0.4,
               "coverage_at_1pct_error": 0.4,
               "aurc": 0.12817460317460316,
-              "error_rate_at_0_9": 0.16666666666666666,
-              "confidence_bias": 0.21322026889904222,
+              "error_rate_at_0_9": 0.14285714285714285,
+              "confidence_bias": 0.22029693644423798,
               "top_bins": {
                 "0.9": {
-                  "n": 6,
+                  "n": 7,
                   "errors": 1,
-                  "error_rate": 0.16666666666666666
+                  "error_rate": 0.14285714285714285
                 },
                 "0.95": {
-                  "n": 5,
+                  "n": 6,
                   "errors": 1,
-                  "error_rate": 0.2
+                  "error_rate": 0.16666666666666666
                 },
                 "0.99": {
                   "n": 3,
@@ -7928,16 +7928,16 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.8,
-                  "confidence_cutoff": 0.9756218860055605
+                  "confidence_cutoff": 0.9773512719149
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.75,
-                  "confidence_cutoff": 0.8950916416154385
+                  "confidence_cutoff": 0.8938853735717587
                 }
               },
-              "score_mae": 0.28450960978161083,
-              "ranked_probability_score": 0.10937006220858798
+              "score_mae": 0.2732574139836782,
+              "ranked_probability_score": 0.10706854764238136
             },
             "coverage": 1.0
           },
@@ -7948,11 +7948,11 @@
             "accuracy": 1.0,
             "full_probability_metrics": {
               "n": 10,
-              "nll": 0.0029451718587351072,
+              "nll": 0.004226216609183238,
               "acc": 1.0,
-              "ece": 0.0029393785852619425,
-              "brier": 2.0483928291246792e-05,
-              "mean_conf": 0.9970606214147381,
+              "ece": 0.004215976897349094,
+              "brier": 3.4649667131696104e-05,
+              "mean_conf": 0.9957840231026509,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -7960,7 +7960,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.0029393785852619425,
+              "confidence_bias": -0.004215976897349094,
               "top_bins": {
                 "0.9": {
                   "n": 10,
@@ -7982,22 +7982,22 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9985005648070538
+                  "confidence_cutoff": 0.9969052065210677
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9953866143850587
+                  "confidence_cutoff": 0.9942030038674581
                 }
               }
             },
             "covered_only_probability_metrics": {
               "n": 10,
-              "nll": 0.0029451718587351072,
+              "nll": 0.004226216609183238,
               "acc": 1.0,
-              "ece": 0.0029393785852619425,
-              "brier": 2.0483928291246792e-05,
-              "mean_conf": 0.9970606214147381,
+              "ece": 0.004215976897349094,
+              "brier": 3.4649667131696104e-05,
+              "mean_conf": 0.9957840231026509,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -8005,7 +8005,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.0029393785852619425,
+              "confidence_bias": -0.004215976897349094,
               "top_bins": {
                 "0.9": {
                   "n": 10,
@@ -8027,12 +8027,12 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9985005648070538
+                  "confidence_cutoff": 0.9969052065210677
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9953866143850587
+                  "confidence_cutoff": 0.9942030038674581
                 }
               }
             },
@@ -8045,11 +8045,11 @@
             "accuracy": 1.0,
             "full_probability_metrics": {
               "n": 10,
-              "nll": 0.00544297332787101,
+              "nll": 0.007053870282788843,
               "acc": 1.0,
-              "ece": 0.005423762486261863,
-              "brier": 6.639927679627542e-05,
-              "mean_conf": 0.9945762375137381,
+              "ece": 0.007024441570579243,
+              "brier": 9.776258511132903e-05,
+              "mean_conf": 0.9929755584294208,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -8057,7 +8057,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.005423762486261863,
+              "confidence_bias": -0.007024441570579243,
               "top_bins": {
                 "0.9": {
                   "n": 10,
@@ -8070,7 +8070,7 @@
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 10,
+                  "n": 7,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -8079,24 +8079,24 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9954970691885048
+                  "confidence_cutoff": 0.9932447768419814
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9907873254440961
+                  "confidence_cutoff": 0.9898739137651561
                 }
               },
-              "score_mae": 0.005819174097637415,
-              "ranked_probability_score": 1.8048948557090246e-05
+              "score_mae": 0.007660911551517352,
+              "ranked_probability_score": 2.757223086398964e-05
             },
             "covered_only_probability_metrics": {
               "n": 10,
-              "nll": 0.00544297332787101,
+              "nll": 0.007053870282788843,
               "acc": 1.0,
-              "ece": 0.005423762486261863,
-              "brier": 6.639927679627542e-05,
-              "mean_conf": 0.9945762375137381,
+              "ece": 0.007024441570579243,
+              "brier": 9.776258511132903e-05,
+              "mean_conf": 0.9929755584294208,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -8104,7 +8104,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.005423762486261863,
+              "confidence_bias": -0.007024441570579243,
               "top_bins": {
                 "0.9": {
                   "n": 10,
@@ -8117,7 +8117,7 @@
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 10,
+                  "n": 7,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -8126,16 +8126,16 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9954970691885048
+                  "confidence_cutoff": 0.9932447768419814
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9907873254440961
+                  "confidence_cutoff": 0.9898739137651561
                 }
               },
-              "score_mae": 0.005819174097637415,
-              "ranked_probability_score": 1.8048948557090246e-05
+              "score_mae": 0.007660911551517352,
+              "ranked_probability_score": 2.757223086398964e-05
             },
             "coverage": 1.0
           },
@@ -8146,19 +8146,19 @@
             "accuracy": 0.5,
             "full_probability_metrics": {
               "n": 10,
-              "nll": 1.0149049311837293,
+              "nll": 0.9724116266401464,
               "acc": 0.5,
-              "ece": 0.42166657051283946,
-              "brier": 0.6278537954984514,
-              "mean_conf": 0.711034663533795,
+              "ece": 0.37317330379992186,
+              "brier": 0.592356663184495,
+              "mean_conf": 0.7285382769948201,
               "confident_error_rate": 0.1,
               "coverage_at_0_9": 0.2,
               "accuracy_at_0_9": 0.5,
               "coverage_at_5pct_error": 0.1,
               "coverage_at_1pct_error": 0.1,
-              "aurc": 0.3800793650793651,
+              "aurc": 0.356468253968254,
               "error_rate_at_0_9": 0.5,
-              "confidence_bias": 0.21103466353379496,
+              "confidence_bias": 0.2285382769948201,
               "top_bins": {
                 "0.9": {
                   "n": 2,
@@ -8180,32 +8180,32 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.6,
-                  "confidence_cutoff": 0.7399423277412458
+                  "confidence_cutoff": 0.7507937755021719
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.5,
-                  "confidence_cutoff": 0.5655117799494201
+                  "accuracy": 0.625,
+                  "confidence_cutoff": 0.597999206215155
                 }
               },
-              "score_mae": 0.46951970562367135,
-              "ranked_probability_score": 0.1575113657152988
+              "score_mae": 0.41248856844465875,
+              "ranked_probability_score": 0.14487339386062986
             },
             "covered_only_probability_metrics": {
               "n": 10,
-              "nll": 1.0149049311837293,
+              "nll": 0.9724116266401464,
               "acc": 0.5,
-              "ece": 0.42166657051283946,
-              "brier": 0.6278537954984514,
-              "mean_conf": 0.711034663533795,
+              "ece": 0.37317330379992186,
+              "brier": 0.592356663184495,
+              "mean_conf": 0.7285382769948201,
               "confident_error_rate": 0.1,
               "coverage_at_0_9": 0.2,
               "accuracy_at_0_9": 0.5,
               "coverage_at_5pct_error": 0.1,
               "coverage_at_1pct_error": 0.1,
-              "aurc": 0.3800793650793651,
+              "aurc": 0.356468253968254,
               "error_rate_at_0_9": 0.5,
-              "confidence_bias": 0.21103466353379496,
+              "confidence_bias": 0.2285382769948201,
               "top_bins": {
                 "0.9": {
                   "n": 2,
@@ -8227,76 +8227,76 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.6,
-                  "confidence_cutoff": 0.7399423277412458
+                  "confidence_cutoff": 0.7507937755021719
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.5,
-                  "confidence_cutoff": 0.5655117799494201
+                  "accuracy": 0.625,
+                  "confidence_cutoff": 0.597999206215155
                 }
               },
-              "score_mae": 0.46951970562367135,
-              "ranked_probability_score": 0.1575113657152988
+              "score_mae": 0.41248856844465875,
+              "ranked_probability_score": 0.14487339386062986
             },
             "coverage": 1.0
           }
         },
-        "clean_task_macro_accuracy": 0.8265740740740741,
+        "clean_task_macro_accuracy": 0.8235648148148148,
         "permutation": {
           "requested_pairs": 36,
           "valid_pairs": 36,
           "both_correct_requested": 0.7777777777777778,
           "flip_rate": 0.08333333333333333,
           "covered_only_flip_rate": 0.08333333333333333,
-          "half_l1": 0.08193039842124604
+          "half_l1": 0.08515840387623191
         },
         "unknowable": {
           "requested": 110,
           "valid": 110,
           "accuracy": null,
           "full_confidence_metrics": {
-            "mean_max_p": 0.6367563362463791,
-            "share_at_0_9": 0.16363636363636364,
-            "normalized_entropy": 0.7292206320388325
+            "mean_max_p": 0.6485130260679773,
+            "share_at_0_9": 0.21818181818181817,
+            "normalized_entropy": 0.7184893503275872
           },
           "covered_only_confidence_metrics": {
-            "mean_max_p": 0.6367563362463791,
-            "share_at_0_9": 0.16363636363636364,
-            "normalized_entropy": 0.7292206320388325
+            "mean_max_p": 0.6485130260679773,
+            "share_at_0_9": 0.21818181818181817,
+            "normalized_entropy": 0.7184893503275872
           },
           "controls": {
             "requested": 110,
             "valid_probability_rows": 110,
-            "correct": 97,
-            "accuracy": 0.8818181818181818,
+            "correct": 95,
+            "accuracy": 0.8636363636363636,
             "full_probability_metrics": {
               "n": 110,
-              "nll": 0.3053991349856939,
-              "acc": 0.8818181818181818,
-              "ece": 0.0526352555914349,
-              "brier": 0.18835088828215762,
-              "mean_conf": 0.8989348929982847,
+              "nll": 0.31120323585634946,
+              "acc": 0.8636363636363636,
+              "ece": 0.055639968292740274,
+              "brier": 0.1918081945957669,
+              "mean_conf": 0.9060614972263522,
               "confident_error_rate": 0.01818181818181818,
-              "coverage_at_0_9": 0.6727272727272727,
-              "accuracy_at_0_9": 0.972972972972973,
-              "coverage_at_5pct_error": 0.7818181818181819,
-              "coverage_at_1pct_error": 0.5272727272727272,
-              "aurc": 0.029171898955925056,
-              "error_rate_at_0_9": 0.02702702702702703,
-              "confidence_bias": 0.017116711180102828,
+              "coverage_at_0_9": 0.6909090909090909,
+              "accuracy_at_0_9": 0.9736842105263158,
+              "coverage_at_5pct_error": 0.7727272727272727,
+              "coverage_at_1pct_error": 0.5454545454545454,
+              "aurc": 0.029014791714251358,
+              "error_rate_at_0_9": 0.02631578947368421,
+              "confidence_bias": 0.04242513358998856,
               "top_bins": {
                 "0.9": {
-                  "n": 74,
+                  "n": 76,
                   "errors": 2,
-                  "error_rate": 0.02702702702702703
+                  "error_rate": 0.02631578947368421
                 },
                 "0.95": {
-                  "n": 66,
+                  "n": 68,
                   "errors": 2,
-                  "error_rate": 0.030303030303030304
+                  "error_rate": 0.029411764705882353
                 },
                 "0.99": {
-                  "n": 49,
+                  "n": 40,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -8305,45 +8305,45 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9837696603934573
+                  "confidence_cutoff": 0.9818909453374407
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.9318181818181818,
-                  "confidence_cutoff": 0.8159752894198687
+                  "accuracy": 0.9431818181818182,
+                  "confidence_cutoff": 0.8415699130530643
                 }
               },
-              "score_mae": 0.20528069640365518,
-              "ranked_probability_score": 0.06052101016066624
+              "score_mae": 0.19343208241221987,
+              "ranked_probability_score": 0.05902836137908972
             },
             "covered_only_probability_metrics": {
               "n": 110,
-              "nll": 0.3053991349856939,
-              "acc": 0.8818181818181818,
-              "ece": 0.0526352555914349,
-              "brier": 0.18835088828215762,
-              "mean_conf": 0.8989348929982847,
+              "nll": 0.31120323585634946,
+              "acc": 0.8636363636363636,
+              "ece": 0.055639968292740274,
+              "brier": 0.1918081945957669,
+              "mean_conf": 0.9060614972263522,
               "confident_error_rate": 0.01818181818181818,
-              "coverage_at_0_9": 0.6727272727272727,
-              "accuracy_at_0_9": 0.972972972972973,
-              "coverage_at_5pct_error": 0.7818181818181819,
-              "coverage_at_1pct_error": 0.5272727272727272,
-              "aurc": 0.029171898955925056,
-              "error_rate_at_0_9": 0.02702702702702703,
-              "confidence_bias": 0.017116711180102828,
+              "coverage_at_0_9": 0.6909090909090909,
+              "accuracy_at_0_9": 0.9736842105263158,
+              "coverage_at_5pct_error": 0.7727272727272727,
+              "coverage_at_1pct_error": 0.5454545454545454,
+              "aurc": 0.029014791714251358,
+              "error_rate_at_0_9": 0.02631578947368421,
+              "confidence_bias": 0.04242513358998856,
               "top_bins": {
                 "0.9": {
-                  "n": 74,
+                  "n": 76,
                   "errors": 2,
-                  "error_rate": 0.02702702702702703
+                  "error_rate": 0.02631578947368421
                 },
                 "0.95": {
-                  "n": 66,
+                  "n": 68,
                   "errors": 2,
-                  "error_rate": 0.030303030303030304
+                  "error_rate": 0.029411764705882353
                 },
                 "0.99": {
-                  "n": 49,
+                  "n": 40,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -8352,22 +8352,22 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9837696603934573
+                  "confidence_cutoff": 0.9818909453374407
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.9318181818181818,
-                  "confidence_cutoff": 0.8159752894198687
+                  "accuracy": 0.9431818181818182,
+                  "confidence_cutoff": 0.8415699130530643
                 }
               },
-              "score_mae": 0.20528069640365518,
-              "ranked_probability_score": 0.06052101016066624
+              "score_mae": 0.19343208241221987,
+              "ranked_probability_score": 0.05902836137908972
             },
             "coverage": 1.0
           },
           "requested_pairs": 110,
           "valid_pairs": 110,
-          "paired_confidence_drop": 0.26217855675190543
+          "paired_confidence_drop": 0.2575484711583747
         },
         "coverage": {
           "questions": 1264,
@@ -8376,98 +8376,98 @@
           "truncated_questions": 0
         },
         "complete_unchanged_upstream_report": {
-          "objective": -0.4372390887135022,
+          "objective": -0.42681700298384834,
           "paired_flip": {
             "pairs": 64,
-            "flip_rate": 0.9375,
-            "both_correct_rate": 0.875,
+            "flip_rate": 0.953125,
+            "both_correct_rate": 0.890625,
             "invariant_pairs": 24,
             "invariance_rate": 1.0,
-            "invariant_both_correct_rate": 0.9166666666666666
+            "invariant_both_correct_rate": 0.9583333333333334
           },
           "unknowable": {
             "n": 110,
-            "mean_max_p": 0.6367563362463791,
-            "share_at_0_9": 0.16363636363636364,
-            "control_mean_max_p": 0.8989348929982847,
-            "control_share_at_0_9": 0.6727272727272727,
-            "control_acc": 0.8818181818181818,
-            "paired_confidence_drop": 0.26217855675190543,
-            "share_less_confident_than_control": 0.8909090909090909
+            "mean_max_p": 0.6485130260679773,
+            "share_at_0_9": 0.21818181818181817,
+            "control_mean_max_p": 0.9060614972263522,
+            "control_share_at_0_9": 0.6909090909090909,
+            "control_acc": 0.8636363636363636,
+            "paired_confidence_drop": 0.2575484711583747,
+            "share_less_confident_than_control": 0.9
           },
           "clean": {
             "n": 1046,
-            "nll": 0.6086212459300988,
-            "acc": 0.7734225621414914,
-            "ece": 0.05889322575332617,
-            "brier": 0.3004372452589081,
-            "mean_conf": 0.8323157878948174,
+            "nll": 0.5940603022446113,
+            "acc": 0.7762906309751434,
+            "ece": 0.05558262070294892,
+            "brier": 0.2937157813621403,
+            "mean_conf": 0.8318732516780923,
             "confident_error_rate": 0.02390057361376673,
-            "coverage_at_0_9": 0.5774378585086042,
-            "accuracy_at_0_9": 0.9586092715231788,
-            "coverage_at_5pct_error": 0.6338432122370937,
-            "coverage_at_1pct_error": 0.38336520076481834,
-            "aurc": 0.061361186701696054,
-            "error_rate_at_0_9": 0.041390728476821195,
-            "confidence_bias": 0.058893225753326006,
+            "coverage_at_0_9": 0.5860420650095602,
+            "accuracy_at_0_9": 0.9592169657422512,
+            "coverage_at_5pct_error": 0.6347992351816444,
+            "coverage_at_1pct_error": 0.3030592734225621,
+            "aurc": 0.05816033320114623,
+            "error_rate_at_0_9": 0.040783034257748776,
+            "confidence_bias": 0.05558262070294895,
             "top_bins": {
               "0.9": {
-                "n": 604,
+                "n": 613,
                 "errors": 25,
-                "error_rate": 0.041390728476821195
+                "error_rate": 0.040783034257748776
               },
               "0.95": {
-                "n": 525,
-                "errors": 15,
-                "error_rate": 0.02857142857142857
+                "n": 514,
+                "errors": 12,
+                "error_rate": 0.023346303501945526
               },
               "0.99": {
-                "n": 226,
+                "n": 207,
                 "errors": 3,
-                "error_rate": 0.01327433628318584
+                "error_rate": 0.014492753623188406
               }
             },
             "selective": {
               "0.5": {
-                "coverage": 0.5,
-                "accuracy": 0.97131931166348,
-                "confidence_cutoff": 0.9503957614513975
+                "coverage": 0.5009560229445507,
+                "accuracy": 0.9770992366412213,
+                "confidence_cutoff": 0.947606086730957
               },
               "0.8": {
                 "coverage": 0.8001912045889101,
-                "accuracy": 0.8745519713261649,
-                "confidence_cutoff": 0.6527601881992761
+                "accuracy": 0.8781362007168458,
+                "confidence_cutoff": 0.647434909565297
               }
             },
-            "score_mae": 0.19182332300936422,
-            "ranked_probability_score": 0.05030124964901672
+            "score_mae": 0.18149603870298436,
+            "ranked_probability_score": 0.051063424067515496
           },
           "tasks": {
             "buried_emotion": {
               "n": 20,
-              "nll": 1.5449426114821578,
+              "nll": 1.4899537928777824,
               "acc": 0.45,
-              "ece": 0.24636374641400255,
-              "brier": 0.7048268473893603,
-              "mean_conf": 0.6963637464140026,
+              "ece": 0.3017003229117818,
+              "brier": 0.6913230901539468,
+              "mean_conf": 0.6857035964205832,
               "confident_error_rate": 0.05,
-              "coverage_at_0_9": 0.25,
-              "accuracy_at_0_9": 0.8,
+              "coverage_at_0_9": 0.2,
+              "accuracy_at_0_9": 0.75,
               "coverage_at_5pct_error": 0.05,
               "coverage_at_1pct_error": 0.05,
-              "aurc": 0.3654720743652632,
-              "error_rate_at_0_9": 0.2,
-              "confidence_bias": 0.24636374641400255,
+              "aurc": 0.3734132508358514,
+              "error_rate_at_0_9": 0.25,
+              "confidence_bias": 0.2357035964205832,
               "top_bins": {
                 "0.9": {
-                  "n": 5,
+                  "n": 4,
                   "errors": 1,
-                  "error_rate": 0.2
+                  "error_rate": 0.25
                 },
                 "0.95": {
-                  "n": 2,
-                  "errors": 1,
-                  "error_rate": 0.5
+                  "n": 1,
+                  "errors": 0,
+                  "error_rate": 0.0
                 },
                 "0.99": {
                   "n": 0,
@@ -8478,31 +8478,31 @@
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
-                  "accuracy": 0.7,
-                  "confidence_cutoff": 0.7313402914926098
+                  "accuracy": 0.6,
+                  "confidence_cutoff": 0.720178043538323
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.5043779386113015
+                  "confidence_cutoff": 0.49892304084621675
                 }
               }
             },
             "buried_paws": {
               "n": 20,
-              "nll": 0.3590183753756898,
+              "nll": 0.391154446854581,
               "acc": 0.8,
-              "ece": 0.12098253375100354,
-              "brier": 0.2234753942109866,
-              "mean_conf": 0.8746775461236951,
+              "ece": 0.0909007219853577,
+              "brier": 0.24269024392468377,
+              "mean_conf": 0.8773054444371697,
               "confident_error_rate": 0.05,
               "coverage_at_0_9": 0.7,
               "accuracy_at_0_9": 0.9285714285714286,
-              "coverage_at_5pct_error": 0.5,
-              "coverage_at_1pct_error": 0.5,
-              "aurc": 0.04897950583128602,
+              "coverage_at_5pct_error": 0.45,
+              "coverage_at_1pct_error": 0.45,
+              "aurc": 0.06282346007965203,
               "error_rate_at_0_9": 0.07142857142857142,
-              "confidence_bias": 0.07467754612369504,
+              "confidence_bias": 0.07730544443716969,
               "top_bins": {
                 "0.9": {
                   "n": 14,
@@ -8523,34 +8523,34 @@
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
-                  "accuracy": 1.0,
-                  "confidence_cutoff": 0.9621361601591426
+                  "accuracy": 0.9,
+                  "confidence_cutoff": 0.9649345170900093
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.9375,
-                  "confidence_cutoff": 0.708977460861206
+                  "accuracy": 0.875,
+                  "confidence_cutoff": 0.7085450713959379
                 }
               }
             },
             "buried_qnli": {
               "n": 20,
-              "nll": 0.17230822458126488,
+              "nll": 0.173920037838875,
               "acc": 0.9,
-              "ece": 0.07775387997537764,
-              "brier": 0.11106925746781111,
-              "mean_conf": 0.9399342803070365,
+              "ece": 0.08884237104933625,
+              "brier": 0.11334577453222641,
+              "mean_conf": 0.9453143823740051,
               "confident_error_rate": 0.0,
-              "coverage_at_0_9": 0.8,
+              "coverage_at_0_9": 0.85,
               "accuracy_at_0_9": 1.0,
               "coverage_at_5pct_error": 0.9,
               "coverage_at_1pct_error": 0.9,
               "aurc": 0.0076315789473684215,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": 0.039934280307036496,
+              "confidence_bias": 0.04531438237400509,
               "top_bins": {
                 "0.9": {
-                  "n": 16,
+                  "n": 17,
                   "errors": 0,
                   "error_rate": 0.0
                 },
@@ -8569,30 +8569,30 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9794639367227702
+                  "confidence_cutoff": 0.9793051012887353
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9520198042701821
+                  "confidence_cutoff": 0.9588658106453812
                 }
               }
             },
             "buried_tweet_offensive": {
               "n": 20,
-              "nll": 1.0747043813664825,
+              "nll": 0.963421060535776,
               "acc": 0.4,
-              "ece": 0.376769070482061,
-              "brier": 0.7839551359464374,
-              "mean_conf": 0.776769070482061,
+              "ece": 0.3598356256038071,
+              "brier": 0.7060594827987038,
+              "mean_conf": 0.7594032993432073,
               "confident_error_rate": 0.05,
               "coverage_at_0_9": 0.1,
               "accuracy_at_0_9": 0.5,
-              "coverage_at_5pct_error": 0.0,
-              "coverage_at_1pct_error": 0.0,
-              "aurc": 0.5897093311315447,
+              "coverage_at_5pct_error": 0.05,
+              "coverage_at_1pct_error": 0.05,
+              "aurc": 0.45470466259746445,
               "error_rate_at_0_9": 0.5,
-              "confidence_bias": 0.376769070482061,
+              "confidence_bias": 0.35940329934320725,
               "top_bins": {
                 "0.9": {
                   "n": 2,
@@ -8614,33 +8614,33 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.7923556442472722
+                  "confidence_cutoff": 0.7584096306325441
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.3125,
-                  "confidence_cutoff": 0.6593232357905366
+                  "accuracy": 0.4375,
+                  "confidence_cutoff": 0.6582031053840188
                 }
               }
             },
             "composition_final_combination": {
               "n": 32,
-              "nll": 0.1488496730625657,
+              "nll": 0.13034778836964267,
               "acc": 0.96875,
-              "ece": 0.0518019411069781,
-              "brier": 0.07608424815707887,
-              "mean_conf": 0.9169480588930219,
+              "ece": 0.07387770926294471,
+              "brier": 0.06436468242012063,
+              "mean_conf": 0.9252340474017305,
               "confident_error_rate": 0.0,
-              "coverage_at_0_9": 0.6875,
+              "coverage_at_0_9": 0.75,
               "accuracy_at_0_9": 1.0,
               "coverage_at_5pct_error": 1.0,
-              "coverage_at_1pct_error": 0.8125,
-              "aurc": 0.006343756575133452,
+              "coverage_at_1pct_error": 0.84375,
+              "aurc": 0.005186349167726045,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.05180194110697811,
+              "confidence_bias": -0.04351595259826946,
               "top_bins": {
                 "0.9": {
-                  "n": 22,
+                  "n": 24,
                   "errors": 0,
                   "error_rate": 0.0
                 },
@@ -8657,45 +8657,45 @@
               },
               "selective": {
                 "0.5": {
-                  "coverage": 0.5,
+                  "coverage": 0.53125,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9526311826156912
+                  "confidence_cutoff": 0.9517537497245334
                 },
                 "0.8": {
                   "coverage": 0.8125,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.8658791104483075
+                  "confidence_cutoff": 0.8932077222078463
                 }
               }
             },
             "composition_final_exception": {
               "n": 32,
-              "nll": 0.2775084906134728,
+              "nll": 0.24646259109043608,
               "acc": 0.90625,
-              "ece": 0.05746431889041253,
-              "brier": 0.15312468872339335,
-              "mean_conf": 0.9174051821778417,
+              "ece": 0.08165686202673392,
+              "brier": 0.13219691386769145,
+              "mean_conf": 0.9344170796872766,
               "confident_error_rate": 0.03125,
-              "coverage_at_0_9": 0.65625,
-              "accuracy_at_0_9": 0.9523809523809523,
-              "coverage_at_5pct_error": 0.84375,
+              "coverage_at_0_9": 0.75,
+              "accuracy_at_0_9": 0.9583333333333334,
+              "coverage_at_5pct_error": 0.9375,
               "coverage_at_1pct_error": 0.375,
-              "aurc": 0.03975357109250195,
-              "error_rate_at_0_9": 0.047619047619047616,
-              "confidence_bias": 0.011155182177841727,
+              "aurc": 0.033345170986070205,
+              "error_rate_at_0_9": 0.041666666666666664,
+              "confidence_bias": 0.02816707968727661,
               "top_bins": {
                 "0.9": {
-                  "n": 21,
+                  "n": 24,
                   "errors": 1,
-                  "error_rate": 0.047619047619047616
+                  "error_rate": 0.041666666666666664
                 },
                 "0.95": {
-                  "n": 16,
+                  "n": 18,
                   "errors": 1,
-                  "error_rate": 0.0625
+                  "error_rate": 0.05555555555555555
                 },
                 "0.99": {
-                  "n": 4,
+                  "n": 6,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -8704,33 +8704,33 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.9375,
-                  "confidence_cutoff": 0.9714136069375261
+                  "confidence_cutoff": 0.9769813800539988
                 },
                 "0.8": {
-                  "coverage": 0.84375,
-                  "accuracy": 0.9629629629629629,
-                  "confidence_cutoff": 0.866120457649231
+                  "coverage": 0.8125,
+                  "accuracy": 0.9615384615384616,
+                  "confidence_cutoff": 0.8745144680431802
                 }
               }
             },
             "composition_final_negation": {
               "n": 32,
-              "nll": 0.23890017515673415,
-              "acc": 0.875,
-              "ece": 0.14167053791272208,
-              "brier": 0.14524192276307057,
-              "mean_conf": 0.8435546575020636,
+              "nll": 0.17366660442192522,
+              "acc": 0.96875,
+              "ece": 0.139280642091471,
+              "brier": 0.09349301051833991,
+              "mean_conf": 0.8684549516140397,
               "confident_error_rate": 0.0,
-              "coverage_at_0_9": 0.46875,
+              "coverage_at_0_9": 0.5625,
               "accuracy_at_0_9": 1.0,
-              "coverage_at_5pct_error": 0.90625,
-              "coverage_at_1pct_error": 0.84375,
-              "aurc": 0.012148294732242174,
+              "coverage_at_5pct_error": 1.0,
+              "coverage_at_1pct_error": 0.875,
+              "aurc": 0.004072377873563219,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.0314453424979364,
+              "confidence_bias": -0.10029504838596026,
               "top_bins": {
                 "0.9": {
-                  "n": 15,
+                  "n": 18,
                   "errors": 0,
                   "error_rate": 0.0
                 },
@@ -8749,22 +8749,22 @@
                 "0.5": {
                   "coverage": 0.53125,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.8950825933794926
+                  "confidence_cutoff": 0.9398574478142043
                 },
                 "0.8": {
-                  "coverage": 0.84375,
+                  "coverage": 0.8125,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.6886792778968811
+                  "confidence_cutoff": 0.7353225569523143
                 }
               }
             },
             "contrastive_authorization": {
               "n": 40,
-              "nll": 0.009534104113565645,
+              "nll": 0.010156044928884202,
               "acc": 1.0,
-              "ece": 0.009472102695785245,
-              "brier": 0.0002455458386976324,
-              "mean_conf": 0.9905278973042148,
+              "ece": 0.010082175827252016,
+              "brier": 0.0002922128455344832,
+              "mean_conf": 0.989917824172748,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -8772,7 +8772,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.009472102695785245,
+              "confidence_bias": -0.010082175827252016,
               "top_bins": {
                 "0.9": {
                   "n": 40,
@@ -8794,38 +8794,38 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9938308764165708
+                  "confidence_cutoff": 0.994282735900115
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9842986932208933
+                  "confidence_cutoff": 0.9830883529872735
                 }
               }
             },
             "contrastive_deadline": {
               "n": 40,
-              "nll": 0.24068925250396264,
+              "nll": 0.2578333370423695,
               "acc": 0.9,
-              "ece": 0.08521130467101881,
-              "brier": 0.13743249520879042,
-              "mean_conf": 0.869609319103461,
+              "ece": 0.08339734125029008,
+              "brier": 0.1539668312012304,
+              "mean_conf": 0.8928125003966263,
               "confident_error_rate": 0.0,
-              "coverage_at_0_9": 0.5,
+              "coverage_at_0_9": 0.6,
               "accuracy_at_0_9": 1.0,
               "coverage_at_5pct_error": 0.7,
-              "coverage_at_1pct_error": 0.675,
-              "aurc": 0.020352482782383177,
+              "coverage_at_1pct_error": 0.65,
+              "aurc": 0.02394744920629697,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.03039068089653907,
+              "confidence_bias": -0.007187499603373726,
               "top_bins": {
                 "0.9": {
-                  "n": 20,
+                  "n": 24,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.95": {
-                  "n": 14,
+                  "n": 16,
                   "errors": 0,
                   "error_rate": 0.0
                 },
@@ -8839,90 +8839,90 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9031725006679073
+                  "confidence_cutoff": 0.9320149317531109
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.9375,
-                  "confidence_cutoff": 0.773653912414944
+                  "confidence_cutoff": 0.8366438894497547
                 }
               },
-              "score_mae": 0.17163726291792777,
-              "ranked_probability_score": 0.03497160888154245
+              "score_mae": 0.16359197313913096,
+              "ranked_probability_score": 0.03911601810015416
             },
             "emotion": {
               "n": 80,
-              "nll": 1.404957749606962,
-              "acc": 0.6,
-              "ece": 0.2239237429633081,
-              "brier": 0.5924528456339091,
-              "mean_conf": 0.8068140896390107,
+              "nll": 1.3250366616414988,
+              "acc": 0.6125,
+              "ece": 0.2033677770446442,
+              "brier": 0.578098657406022,
+              "mean_conf": 0.7837558603474908,
               "confident_error_rate": 0.075,
-              "coverage_at_0_9": 0.4625,
-              "accuracy_at_0_9": 0.8378378378378378,
+              "coverage_at_0_9": 0.45,
+              "accuracy_at_0_9": 0.8333333333333334,
               "coverage_at_5pct_error": 0.025,
               "coverage_at_1pct_error": 0.025,
-              "aurc": 0.249645290423344,
-              "error_rate_at_0_9": 0.16216216216216217,
-              "confidence_bias": 0.2068140896390107,
+              "aurc": 0.2465828605574525,
+              "error_rate_at_0_9": 0.16666666666666666,
+              "confidence_bias": 0.17125586034749074,
               "top_bins": {
                 "0.9": {
-                  "n": 37,
+                  "n": 36,
                   "errors": 6,
-                  "error_rate": 0.16216216216216217
+                  "error_rate": 0.16666666666666666
                 },
                 "0.95": {
-                  "n": 31,
-                  "errors": 5,
-                  "error_rate": 0.16129032258064516
+                  "n": 24,
+                  "errors": 3,
+                  "error_rate": 0.125
                 },
                 "0.99": {
-                  "n": 10,
+                  "n": 7,
                   "errors": 2,
-                  "error_rate": 0.2
+                  "error_rate": 0.2857142857142857
                 }
               },
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
-                  "accuracy": 0.825,
-                  "confidence_cutoff": 0.8734302110272708
+                  "accuracy": 0.8,
+                  "confidence_cutoff": 0.8534444506954815
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.625,
-                  "confidence_cutoff": 0.612044678035727
+                  "accuracy": 0.65625,
+                  "confidence_cutoff": 0.5796189437721434
                 }
               }
             },
             "mmlu": {
               "n": 80,
-              "nll": 0.6291131826963371,
+              "nll": 0.6466641422955599,
               "acc": 0.7375,
-              "ece": 0.09166398596467382,
-              "brier": 0.32846647117297145,
-              "mean_conf": 0.8291639859646738,
+              "ece": 0.10575227809040644,
+              "brier": 0.3381993704155934,
+              "mean_conf": 0.8239887221865458,
               "confident_error_rate": 0.025,
-              "coverage_at_0_9": 0.5625,
-              "accuracy_at_0_9": 0.9555555555555556,
-              "coverage_at_5pct_error": 0.6,
-              "coverage_at_1pct_error": 0.475,
-              "aurc": 0.06444573232402091,
-              "error_rate_at_0_9": 0.044444444444444446,
-              "confidence_bias": 0.09166398596467373,
+              "coverage_at_0_9": 0.5125,
+              "accuracy_at_0_9": 0.9512195121951219,
+              "coverage_at_5pct_error": 0.5875,
+              "coverage_at_1pct_error": 0.4125,
+              "aurc": 0.07075415004878534,
+              "error_rate_at_0_9": 0.04878048780487805,
+              "confidence_bias": 0.0864887221865458,
               "top_bins": {
                 "0.9": {
-                  "n": 45,
+                  "n": 41,
                   "errors": 2,
-                  "error_rate": 0.044444444444444446
+                  "error_rate": 0.04878048780487805
                 },
                 "0.95": {
-                  "n": 35,
+                  "n": 31,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 12,
+                  "n": 9,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -8931,43 +8931,43 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.95,
-                  "confidence_cutoff": 0.9337377421761286
+                  "confidence_cutoff": 0.9179506124738418
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.875,
-                  "confidence_cutoff": 0.6497676638791126
+                  "accuracy": 0.859375,
+                  "confidence_cutoff": 0.6428546144492293
                 }
               }
             },
             "mmlu_pro": {
               "n": 200,
-              "nll": 1.3194872971476064,
+              "nll": 1.3111522675312284,
               "acc": 0.53,
-              "ece": 0.0712459960944238,
-              "brier": 0.57596282275509,
-              "mean_conf": 0.5988417766167529,
-              "confident_error_rate": 0.01,
-              "coverage_at_0_9": 0.245,
-              "accuracy_at_0_9": 0.9591836734693877,
-              "coverage_at_5pct_error": 0.265,
-              "coverage_at_1pct_error": 0.185,
-              "aurc": 0.22156004538018903,
-              "error_rate_at_0_9": 0.04081632653061224,
-              "confidence_bias": 0.06884177661675284,
+              "ece": 0.08739805659958974,
+              "brier": 0.5728987396461885,
+              "mean_conf": 0.5899912710608897,
+              "confident_error_rate": 0.005,
+              "coverage_at_0_9": 0.23,
+              "accuracy_at_0_9": 0.9782608695652174,
+              "coverage_at_5pct_error": 0.25,
+              "coverage_at_1pct_error": 0.19,
+              "aurc": 0.21980746104696547,
+              "error_rate_at_0_9": 0.021739130434782608,
+              "confidence_bias": 0.05999127106088964,
               "top_bins": {
                 "0.9": {
-                  "n": 49,
-                  "errors": 2,
-                  "error_rate": 0.04081632653061224
+                  "n": 46,
+                  "errors": 1,
+                  "error_rate": 0.021739130434782608
                 },
                 "0.95": {
-                  "n": 36,
+                  "n": 29,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 9,
+                  "n": 6,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -8975,41 +8975,41 @@
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
-                  "accuracy": 0.73,
-                  "confidence_cutoff": 0.5902113030216032
+                  "accuracy": 0.75,
+                  "confidence_cutoff": 0.57725688415424
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.625,
-                  "confidence_cutoff": 0.2801405568637054
+                  "confidence_cutoff": 0.26950159367730925
                 }
               }
             },
             "paws": {
               "n": 80,
-              "nll": 0.2774054133452753,
+              "nll": 0.28314251837212323,
               "acc": 0.9,
-              "ece": 0.04500883219029569,
-              "brier": 0.1577162228589053,
-              "mean_conf": 0.9337538243395758,
-              "confident_error_rate": 0.05,
-              "coverage_at_0_9": 0.8125,
-              "accuracy_at_0_9": 0.9384615384615385,
-              "coverage_at_5pct_error": 0.7875,
-              "coverage_at_1pct_error": 0.35,
-              "aurc": 0.027700444825567955,
-              "error_rate_at_0_9": 0.06153846153846154,
-              "confidence_bias": 0.033753824339575744,
+              "ece": 0.06247439765758463,
+              "brier": 0.15885879977500503,
+              "mean_conf": 0.9412949163036171,
+              "confident_error_rate": 0.0625,
+              "coverage_at_0_9": 0.8375,
+              "accuracy_at_0_9": 0.9253731343283582,
+              "coverage_at_5pct_error": 0.775,
+              "coverage_at_1pct_error": 0.2875,
+              "aurc": 0.030912620564059478,
+              "error_rate_at_0_9": 0.07462686567164178,
+              "confidence_bias": 0.041294916303617035,
               "top_bins": {
                 "0.9": {
-                  "n": 65,
-                  "errors": 4,
-                  "error_rate": 0.06153846153846154
+                  "n": 67,
+                  "errors": 5,
+                  "error_rate": 0.07462686567164178
                 },
                 "0.95": {
-                  "n": 59,
+                  "n": 60,
                   "errors": 2,
-                  "error_rate": 0.03389830508474576
+                  "error_rate": 0.03333333333333333
                 },
                 "0.99": {
                   "n": 2,
@@ -9021,67 +9021,67 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.975,
-                  "confidence_cutoff": 0.9777894020080566
+                  "confidence_cutoff": 0.9773598813644135
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.9375,
-                  "confidence_cutoff": 0.9301801601105976
+                  "confidence_cutoff": 0.9395250222016129
                 }
               }
             },
             "qnli": {
               "n": 80,
-              "nll": 0.21752679625446542,
+              "nll": 0.2161076588398619,
               "acc": 0.9125,
-              "ece": 0.03901376668147833,
-              "brier": 0.11730868195084412,
-              "mean_conf": 0.9352517996137696,
+              "ece": 0.038010106954782505,
+              "brier": 0.11745397306383834,
+              "mean_conf": 0.9376866478309165,
               "confident_error_rate": 0.025,
-              "coverage_at_0_9": 0.8125,
-              "accuracy_at_0_9": 0.9692307692307692,
-              "coverage_at_5pct_error": 0.9125,
-              "coverage_at_1pct_error": 0.0625,
-              "aurc": 0.040943653344357525,
-              "error_rate_at_0_9": 0.03076923076923077,
-              "confidence_bias": 0.022751799613769608,
+              "coverage_at_0_9": 0.8375,
+              "accuracy_at_0_9": 0.9701492537313433,
+              "coverage_at_5pct_error": 0.925,
+              "coverage_at_1pct_error": 0.075,
+              "aurc": 0.039172223012800614,
+              "error_rate_at_0_9": 0.029850746268656716,
+              "confidence_bias": 0.02518664783091651,
               "top_bins": {
                 "0.9": {
-                  "n": 65,
+                  "n": 67,
                   "errors": 2,
-                  "error_rate": 0.03076923076923077
+                  "error_rate": 0.029850746268656716
                 },
                 "0.95": {
-                  "n": 63,
+                  "n": 62,
                   "errors": 2,
-                  "error_rate": 0.031746031746031744
+                  "error_rate": 0.03225806451612903
                 },
                 "0.99": {
-                  "n": 28,
+                  "n": 26,
                   "errors": 1,
-                  "error_rate": 0.03571428571428571
+                  "error_rate": 0.038461538461538464
                 }
               },
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.975,
-                  "confidence_cutoff": 0.9841503702581936
+                  "confidence_cutoff": 0.9851611733370504
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.96875,
-                  "confidence_cutoff": 0.9355465034314021
+                  "confidence_cutoff": 0.9458952863255798
                 }
               }
             },
             "sciq": {
               "n": 80,
-              "nll": 0.004514713334100866,
+              "nll": 0.006236348098711358,
               "acc": 1.0,
-              "ece": 0.004458343565561407,
-              "brier": 0.00020475202026520086,
-              "mean_conf": 0.9955416564344386,
+              "ece": 0.006149372309216639,
+              "brier": 0.00030747770792780715,
+              "mean_conf": 0.9938506276907834,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -9089,7 +9089,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.004458343565561407,
+              "confidence_bias": -0.006149372309216639,
               "top_bins": {
                 "0.9": {
                   "n": 80,
@@ -9097,12 +9097,12 @@
                   "error_rate": 0.0
                 },
                 "0.95": {
-                  "n": 78,
+                  "n": 79,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 70,
+                  "n": 69,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -9111,38 +9111,38 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9987595494699697
+                  "confidence_cutoff": 0.9978451209130044
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9963549793899044
+                  "confidence_cutoff": 0.9938674921230647
                 }
               }
             },
             "tweet_offensive": {
               "n": 80,
-              "nll": 0.5266044697812845,
-              "acc": 0.7375,
-              "ece": 0.10983391602437403,
-              "brier": 0.3587274567128259,
-              "mean_conf": 0.8256957228540391,
+              "nll": 0.47556818540480367,
+              "acc": 0.75,
+              "ece": 0.07643617030229378,
+              "brier": 0.3199264336587134,
+              "mean_conf": 0.8258139805835978,
               "confident_error_rate": 0.0375,
-              "coverage_at_0_9": 0.425,
-              "accuracy_at_0_9": 0.9117647058823529,
-              "coverage_at_5pct_error": 0.35,
-              "coverage_at_1pct_error": 0.3,
-              "aurc": 0.11516294025516034,
-              "error_rate_at_0_9": 0.08823529411764706,
-              "confidence_bias": 0.0881957228540391,
+              "coverage_at_0_9": 0.4125,
+              "accuracy_at_0_9": 0.9090909090909091,
+              "coverage_at_5pct_error": 0.3625,
+              "coverage_at_1pct_error": 0.35,
+              "aurc": 0.09222008593150528,
+              "error_rate_at_0_9": 0.09090909090909091,
+              "confidence_bias": 0.07581398058359778,
               "top_bins": {
                 "0.9": {
-                  "n": 34,
+                  "n": 33,
                   "errors": 3,
-                  "error_rate": 0.08823529411764706
+                  "error_rate": 0.09090909090909091
                 },
                 "0.95": {
-                  "n": 23,
+                  "n": 24,
                   "errors": 0,
                   "error_rate": 0.0
                 },
@@ -9156,22 +9156,22 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.9,
-                  "confidence_cutoff": 0.8725079970057013
+                  "confidence_cutoff": 0.8609209785002616
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.765625,
-                  "confidence_cutoff": 0.6975352375964892
+                  "accuracy": 0.828125,
+                  "confidence_cutoff": 0.6948713064193726
                 }
               }
             },
             "unknowable_age_eligibility": {
               "n": 10,
-              "nll": 0.8324197880179746,
+              "nll": 0.8382027962956439,
               "acc": 0.5,
-              "ece": 0.22689295488829114,
-              "brier": 0.6148228531344591,
-              "mean_conf": 0.7268929548882912,
+              "ece": 0.23136963355477053,
+              "brier": 0.6189159675554958,
+              "mean_conf": 0.7313696335547706,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.0,
               "accuracy_at_0_9": null,
@@ -9179,7 +9179,7 @@
               "coverage_at_1pct_error": 0.0,
               "aurc": 0.5,
               "error_rate_at_0_9": null,
-              "confidence_bias": 0.2268929548882912,
+              "confidence_bias": 0.23136963355477058,
               "top_bins": {
                 "0.9": {
                   "n": 0,
@@ -9201,35 +9201,35 @@
                 "0.5": {
                   "coverage": 0.6,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.7211191224384831
+                  "confidence_cutoff": 0.7283886075019836
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.6453390525180384
+                  "confidence_cutoff": 0.6479771137237549
                 }
               }
             },
             "unknowable_authorization": {
               "n": 10,
-              "nll": 0.9725861213638396,
+              "nll": 1.1053561068085782,
               "acc": 0.5,
-              "ece": 0.3223304189350404,
-              "brier": 0.7008324242939181,
-              "mean_conf": 0.8223304189350402,
-              "confident_error_rate": 0.0,
-              "coverage_at_0_9": 0.0,
-              "accuracy_at_0_9": null,
+              "ece": 0.3656208440260864,
+              "brier": 0.7581604276839656,
+              "mean_conf": 0.8656208440260865,
+              "confident_error_rate": 0.3,
+              "coverage_at_0_9": 0.6,
+              "accuracy_at_0_9": 0.5,
               "coverage_at_5pct_error": 0.2,
               "coverage_at_1pct_error": 0.2,
               "aurc": 0.32313492063492066,
-              "error_rate_at_0_9": null,
-              "confidence_bias": 0.3223304189350402,
+              "error_rate_at_0_9": 0.5,
+              "confidence_bias": 0.3656208440260865,
               "top_bins": {
                 "0.9": {
-                  "n": 0,
-                  "errors": 0,
-                  "error_rate": null
+                  "n": 6,
+                  "errors": 3,
+                  "error_rate": 0.5
                 },
                 "0.95": {
                   "n": 0,
@@ -9246,22 +9246,22 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.6,
-                  "confidence_cutoff": 0.8713734149932861
+                  "confidence_cutoff": 0.9161812663078308
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.625,
-                  "confidence_cutoff": 0.7420693039894104
+                  "confidence_cutoff": 0.7894018645317196
                 }
               }
             },
             "unknowable_control_age_eligibility": {
               "n": 10,
-              "nll": 0.006876414004309999,
+              "nll": 0.007841155789564139,
               "acc": 1.0,
-              "ece": 0.0068486883882732386,
-              "brier": 0.00011023848598835957,
-              "mean_conf": 0.9931513116117268,
+              "ece": 0.007806247293806301,
+              "brier": 0.00013872505147745316,
+              "mean_conf": 0.9921937527061937,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -9269,7 +9269,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.0068486883882732386,
+              "confidence_bias": -0.007806247293806301,
               "top_bins": {
                 "0.9": {
                   "n": 10,
@@ -9282,7 +9282,7 @@
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 9,
+                  "n": 6,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -9291,22 +9291,22 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9946927960913962
+                  "confidence_cutoff": 0.9939010793872055
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9907611704497484
+                  "confidence_cutoff": 0.9896593136903282
                 }
               }
             },
             "unknowable_control_authorization": {
               "n": 10,
-              "nll": 0.010054244756134334,
+              "nll": 0.0107929267557592,
               "acc": 1.0,
-              "ece": 0.009977028457380022,
-              "brier": 0.0003052461514864493,
-              "mean_conf": 0.99002297154262,
+              "ece": 0.010701462639340464,
+              "brier": 0.00036120660545170313,
+              "mean_conf": 0.9892985373606595,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -9314,7 +9314,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.009977028457380022,
+              "confidence_bias": -0.010701462639340464,
               "top_bins": {
                 "0.9": {
                   "n": 10,
@@ -9336,38 +9336,38 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9966472206636919
+                  "confidence_cutoff": 0.9969034000449554
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9836865232385298
+                  "confidence_cutoff": 0.9816944573491596
                 }
               }
             },
             "unknowable_control_deadline": {
               "n": 10,
-              "nll": 0.20875349707626656,
-              "acc": 1.0,
-              "ece": 0.16950983158479752,
-              "brier": 0.10697640035089204,
-              "mean_conf": 0.8304901684152025,
+              "nll": 0.2138260506584651,
+              "acc": 0.8,
+              "ece": 0.15741611133328517,
+              "brier": 0.1276425479499806,
+              "mean_conf": 0.856580016184909,
               "confident_error_rate": 0.0,
-              "coverage_at_0_9": 0.5,
+              "coverage_at_0_9": 0.6,
               "accuracy_at_0_9": 1.0,
-              "coverage_at_5pct_error": 1.0,
-              "coverage_at_1pct_error": 1.0,
-              "aurc": 0.0,
+              "coverage_at_5pct_error": 0.8,
+              "coverage_at_1pct_error": 0.8,
+              "aurc": 0.03111111111111111,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.1695098315847975,
+              "confidence_bias": 0.05658001618490893,
               "top_bins": {
                 "0.9": {
-                  "n": 5,
+                  "n": 6,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.95": {
-                  "n": 3,
+                  "n": 4,
                   "errors": 0,
                   "error_rate": 0.0
                 },
@@ -9381,24 +9381,24 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9212612921563369
+                  "confidence_cutoff": 0.9426117959877142
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.7564743954513996
+                  "confidence_cutoff": 0.8415699130530643
                 }
               },
-              "score_mae": 0.1696031781535504,
-              "ranked_probability_score": 0.027307549723045547
+              "score_mae": 0.16071754616245987,
+              "ranked_probability_score": 0.03275971187271286
             },
             "unknowable_control_late_fee": {
               "n": 10,
-              "nll": 0.07820821810037346,
+              "nll": 0.07005694097555375,
               "acc": 1.0,
-              "ece": 0.05979035227629538,
-              "brier": 0.04652092623743477,
-              "mean_conf": 0.9402096477237045,
+              "ece": 0.0540813816797954,
+              "brier": 0.04110395968042234,
+              "mean_conf": 0.9459186183202046,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.9,
               "accuracy_at_0_9": 1.0,
@@ -9406,7 +9406,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.05979035227629548,
+              "confidence_bias": -0.05408138167979537,
               "top_bins": {
                 "0.9": {
                   "n": 9,
@@ -9419,7 +9419,7 @@
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 7,
+                  "n": 6,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -9428,24 +9428,24 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9956810330664218
+                  "confidence_cutoff": 0.993819476814992
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9593766863298722
+                  "confidence_cutoff": 0.9798565648382314
                 }
               },
-              "score_mae": 0.0541174703260209,
-              "ranked_probability_score": 0.011129613417488322
+              "score_mae": 0.0487152458265909,
+              "ranked_probability_score": 0.00975869637261514
             },
             "unknowable_control_quantity_limit": {
               "n": 10,
-              "nll": 0.10634568298556582,
+              "nll": 0.10819644087102691,
               "acc": 1.0,
-              "ece": 0.09157236085416048,
-              "brier": 0.043689973951603296,
-              "mean_conf": 0.9084276391458393,
+              "ece": 0.0954442527194344,
+              "brier": 0.039610537719924295,
+              "mean_conf": 0.9045557472805654,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.7,
               "accuracy_at_0_9": 1.0,
@@ -9453,7 +9453,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.09157236085416065,
+              "confidence_bias": -0.09544425271943457,
               "top_bins": {
                 "0.9": {
                   "n": 7,
@@ -9466,39 +9466,39 @@
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 2,
+                  "n": 0,
                   "errors": 0,
-                  "error_rate": 0.0
+                  "error_rate": null
                 }
               },
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9648931116321414
+                  "confidence_cutoff": 0.9656626177441163
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.8787499248169464
+                  "confidence_cutoff": 0.881241046212756
                 }
               }
             },
             "unknowable_control_return_window": {
               "n": 10,
-              "nll": 0.7842587765663349,
+              "nll": 0.8939128624245644,
               "acc": 0.6,
-              "ece": 0.3758565683544252,
-              "brier": 0.5730022276631428,
-              "mean_conf": 0.8042052849775017,
+              "ece": 0.39838756334240605,
+              "brier": 0.6386225586932958,
+              "mean_conf": 0.8386086717206818,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.2,
               "accuracy_at_0_9": 1.0,
-              "coverage_at_5pct_error": 0.3,
-              "coverage_at_1pct_error": 0.3,
-              "aurc": 0.25563492063492066,
+              "coverage_at_5pct_error": 0.2,
+              "coverage_at_1pct_error": 0.2,
+              "aurc": 0.288968253968254,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": 0.20420528497750168,
+              "confidence_bias": 0.2386086717206818,
               "top_bins": {
                 "0.9": {
                   "n": 2,
@@ -9520,22 +9520,22 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.8,
-                  "confidence_cutoff": 0.8447330845812883
+                  "confidence_cutoff": 0.8853410746258817
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.7434658255258695
+                  "confidence_cutoff": 0.8282266608409089
                 }
               }
             },
             "unknowable_control_shipping_delay": {
               "n": 10,
-              "nll": 0.3877484850113062,
+              "nll": 0.3878790410817737,
               "acc": 0.9,
-              "ece": 0.1957660562559332,
-              "brier": 0.23322717018981343,
-              "mean_conf": 0.8058850082032224,
+              "ece": 0.20366310890419706,
+              "brier": 0.23894788485153878,
+              "mean_conf": 0.8019263309455292,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.3,
               "accuracy_at_0_9": 1.0,
@@ -9543,7 +9543,7 @@
               "coverage_at_1pct_error": 0.5,
               "aurc": 0.06456349206349207,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.09411499179677763,
+              "confidence_bias": -0.09807366905447079,
               "top_bins": {
                 "0.9": {
                   "n": 3,
@@ -9565,42 +9565,42 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.8255768464444774
+                  "confidence_cutoff": 0.8513163369222139
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.875,
-                  "confidence_cutoff": 0.734157871218888
+                  "confidence_cutoff": 0.7305872644461858
                 }
               },
-              "score_mae": 0.2481150404394402,
-              "ranked_probability_score": 0.05778942095101965
+              "score_mae": 0.2577528085044144,
+              "ranked_probability_score": 0.05968224629533503
             },
             "unknowable_control_sla_response": {
               "n": 10,
-              "nll": 0.753852089972006,
+              "nll": 0.7470384623310196,
               "acc": 0.7,
-              "ece": 0.21322026889904205,
-              "brier": 0.44008690936983347,
-              "mean_conf": 0.9132202688990422,
+              "ece": 0.22029693644423792,
+              "brier": 0.4309736445646065,
+              "mean_conf": 0.9202969364442379,
               "confident_error_rate": 0.1,
-              "coverage_at_0_9": 0.6,
-              "accuracy_at_0_9": 0.8333333333333334,
+              "coverage_at_0_9": 0.7,
+              "accuracy_at_0_9": 0.8571428571428571,
               "coverage_at_5pct_error": 0.4,
               "coverage_at_1pct_error": 0.4,
               "aurc": 0.12817460317460316,
-              "error_rate_at_0_9": 0.16666666666666666,
-              "confidence_bias": 0.21322026889904222,
+              "error_rate_at_0_9": 0.14285714285714285,
+              "confidence_bias": 0.22029693644423798,
               "top_bins": {
                 "0.9": {
-                  "n": 6,
+                  "n": 7,
                   "errors": 1,
-                  "error_rate": 0.16666666666666666
+                  "error_rate": 0.14285714285714285
                 },
                 "0.95": {
-                  "n": 5,
+                  "n": 6,
                   "errors": 1,
-                  "error_rate": 0.2
+                  "error_rate": 0.16666666666666666
                 },
                 "0.99": {
                   "n": 3,
@@ -9612,24 +9612,24 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.8,
-                  "confidence_cutoff": 0.9756218860055605
+                  "confidence_cutoff": 0.9773512719149
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.75,
-                  "confidence_cutoff": 0.8950916416154385
+                  "confidence_cutoff": 0.8938853735717587
                 }
               },
-              "score_mae": 0.28450960978161083,
-              "ranked_probability_score": 0.10937006220858798
+              "score_mae": 0.2732574139836782,
+              "ranked_probability_score": 0.10706854764238136
             },
             "unknowable_control_spend_threshold": {
               "n": 10,
-              "nll": 0.0029451718587351072,
+              "nll": 0.004226216609183238,
               "acc": 1.0,
-              "ece": 0.0029393785852619425,
-              "brier": 2.0483928291246792e-05,
-              "mean_conf": 0.9970606214147381,
+              "ece": 0.004215976897349094,
+              "brier": 3.4649667131696104e-05,
+              "mean_conf": 0.9957840231026509,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -9637,7 +9637,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.0029393785852619425,
+              "confidence_bias": -0.004215976897349094,
               "top_bins": {
                 "0.9": {
                   "n": 10,
@@ -9659,22 +9659,22 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9985005648070538
+                  "confidence_cutoff": 0.9969052065210677
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9953866143850587
+                  "confidence_cutoff": 0.9942030038674581
                 }
               }
             },
             "unknowable_control_volume_discount": {
               "n": 10,
-              "nll": 0.00544297332787101,
+              "nll": 0.007053870282788843,
               "acc": 1.0,
-              "ece": 0.005423762486261863,
-              "brier": 6.639927679627542e-05,
-              "mean_conf": 0.9945762375137381,
+              "ece": 0.007024441570579243,
+              "brier": 9.776258511132903e-05,
+              "mean_conf": 0.9929755584294208,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 1.0,
@@ -9682,7 +9682,7 @@
               "coverage_at_1pct_error": 1.0,
               "aurc": 0.0,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": -0.005423762486261863,
+              "confidence_bias": -0.007024441570579243,
               "top_bins": {
                 "0.9": {
                   "n": 10,
@@ -9695,7 +9695,7 @@
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 10,
+                  "n": 7,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -9704,32 +9704,32 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9954970691885048
+                  "confidence_cutoff": 0.9932447768419814
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9907873254440961
+                  "confidence_cutoff": 0.9898739137651561
                 }
               },
-              "score_mae": 0.005819174097637415,
-              "ranked_probability_score": 1.8048948557090246e-05
+              "score_mae": 0.007660911551517352,
+              "ranked_probability_score": 2.757223086398964e-05
             },
             "unknowable_control_warranty_claim": {
               "n": 10,
-              "nll": 1.0149049311837293,
+              "nll": 0.9724116266401464,
               "acc": 0.5,
-              "ece": 0.42166657051283946,
-              "brier": 0.6278537954984514,
-              "mean_conf": 0.711034663533795,
+              "ece": 0.37317330379992186,
+              "brier": 0.592356663184495,
+              "mean_conf": 0.7285382769948201,
               "confident_error_rate": 0.1,
               "coverage_at_0_9": 0.2,
               "accuracy_at_0_9": 0.5,
               "coverage_at_5pct_error": 0.1,
               "coverage_at_1pct_error": 0.1,
-              "aurc": 0.3800793650793651,
+              "aurc": 0.356468253968254,
               "error_rate_at_0_9": 0.5,
-              "confidence_bias": 0.21103466353379496,
+              "confidence_bias": 0.2285382769948201,
               "top_bins": {
                 "0.9": {
                   "n": 2,
@@ -9751,32 +9751,32 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.6,
-                  "confidence_cutoff": 0.7399423277412458
+                  "confidence_cutoff": 0.7507937755021719
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.5,
-                  "confidence_cutoff": 0.5655117799494201
+                  "accuracy": 0.625,
+                  "confidence_cutoff": 0.597999206215155
                 }
               },
-              "score_mae": 0.46951970562367135,
-              "ranked_probability_score": 0.1575113657152988
+              "score_mae": 0.41248856844465875,
+              "ranked_probability_score": 0.14487339386062986
             },
             "unknowable_deadline": {
               "n": 10,
-              "nll": 1.2613031418151548,
+              "nll": 1.2106218946046243,
               "acc": 0.3,
-              "ece": 0.177877089961919,
-              "brier": 0.7619243213513173,
-              "mean_conf": 0.47787708996191897,
+              "ece": 0.17861993312941266,
+              "brier": 0.7343701548653422,
+              "mean_conf": 0.47861993312941264,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.0,
               "accuracy_at_0_9": null,
               "coverage_at_5pct_error": 0.0,
               "coverage_at_1pct_error": 0.0,
-              "aurc": 0.8029761904761905,
+              "aurc": 0.7779761904761905,
               "error_rate_at_0_9": null,
-              "confidence_bias": 0.17787708996191898,
+              "confidence_bias": 0.17861993312941266,
               "top_bins": {
                 "0.9": {
                   "n": 0,
@@ -9798,24 +9798,24 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.2,
-                  "confidence_cutoff": 0.48771480379363924
+                  "confidence_cutoff": 0.49840795241629865
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.375,
-                  "confidence_cutoff": 0.4396536350250244
+                  "confidence_cutoff": 0.4385092854499817
                 }
               },
-              "score_mae": 0.7850845186781722,
-              "ranked_probability_score": 0.25791158776503814
+              "score_mae": 0.7449380524877681,
+              "ranked_probability_score": 0.24553973235698762
             },
             "unknowable_late_fee": {
               "n": 10,
-              "nll": 1.3357243996610713,
+              "nll": 1.3276710177178344,
               "acc": 0.3,
-              "ece": 0.29319184809035176,
-              "brier": 0.8256903429309619,
-              "mean_conf": 0.5931918480903517,
+              "ece": 0.27106129862072126,
+              "brier": 0.8180838268143074,
+              "mean_conf": 0.5710612986207213,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.0,
               "accuracy_at_0_9": null,
@@ -9823,7 +9823,7 @@
               "coverage_at_1pct_error": 0.0,
               "aurc": 0.7483333333333333,
               "error_rate_at_0_9": null,
-              "confidence_bias": 0.29319184809035176,
+              "confidence_bias": 0.2710612986207213,
               "top_bins": {
                 "0.9": {
                   "n": 0,
@@ -9845,24 +9845,24 @@
                 "0.5": {
                   "coverage": 0.6,
                   "accuracy": 0.3333333333333333,
-                  "confidence_cutoff": 0.5555960536003113
+                  "confidence_cutoff": 0.550855944974189
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.375,
-                  "confidence_cutoff": 0.5130325938040461
+                  "confidence_cutoff": 0.4580442284371989
                 }
               },
-              "score_mae": 0.7734692048269523,
-              "ranked_probability_score": 0.26795404663537026
+              "score_mae": 0.7681052591130838,
+              "ranked_probability_score": 0.2644686148936013
             },
             "unknowable_quantity_limit": {
               "n": 10,
-              "nll": 2.5834206860160718,
+              "nll": 2.4681683362881595,
               "acc": 0.5,
-              "ece": 0.48893830922980164,
-              "brier": 0.9830983666698885,
-              "mean_conf": 0.9889383092298016,
+              "ece": 0.48573829809507685,
+              "brier": 0.9783875577934342,
+              "mean_conf": 0.9857382980950768,
               "confident_error_rate": 0.5,
               "coverage_at_0_9": 1.0,
               "accuracy_at_0_9": 0.5,
@@ -9870,7 +9870,7 @@
               "coverage_at_1pct_error": 0.0,
               "aurc": 0.5,
               "error_rate_at_0_9": 0.5,
-              "confidence_bias": 0.48893830922980164,
+              "confidence_bias": 0.48573829809507685,
               "top_bins": {
                 "0.9": {
                   "n": 10,
@@ -9883,39 +9883,39 @@
                   "error_rate": 0.5
                 },
                 "0.99": {
-                  "n": 4,
-                  "errors": 2,
-                  "error_rate": 0.5
+                  "n": 0,
+                  "errors": 0,
+                  "error_rate": null
                 }
               },
               "selective": {
                 "0.5": {
                   "coverage": 0.6,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.9883533902636489
+                  "confidence_cutoff": 0.9864682188832252
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.9880483656587639
+                  "confidence_cutoff": 0.9834588178510609
                 }
               }
             },
             "unknowable_return_window": {
               "n": 10,
-              "nll": 0.7914498549035975,
-              "acc": 0.4,
-              "ece": 0.2586710117345311,
-              "brier": 0.564890331573083,
-              "mean_conf": 0.5866955698025846,
+              "nll": 0.7972145494828772,
+              "acc": 0.6,
+              "ece": 0.18620925667194407,
+              "brier": 0.5676903556290133,
+              "mean_conf": 0.5894333514724946,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.0,
               "accuracy_at_0_9": null,
               "coverage_at_5pct_error": 0.0,
               "coverage_at_1pct_error": 0.0,
-              "aurc": 0.470079365079365,
+              "aurc": 0.48813492063492064,
               "error_rate_at_0_9": null,
-              "confidence_bias": 0.18669556980258462,
+              "confidence_bias": -0.010566648527505329,
               "top_bins": {
                 "0.9": {
                   "n": 0,
@@ -9936,31 +9936,31 @@
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
-                  "accuracy": 0.8,
-                  "confidence_cutoff": 0.5635976195335388
+                  "accuracy": 0.6,
+                  "confidence_cutoff": 0.5594538450241089
                 },
                 "0.8": {
                   "coverage": 0.8,
-                  "accuracy": 0.5,
-                  "confidence_cutoff": 0.5168146049027027
+                  "accuracy": 0.625,
+                  "confidence_cutoff": 0.5314476331637015
                 }
               }
             },
             "unknowable_shipping_delay": {
               "n": 10,
-              "nll": 1.220605702584383,
-              "acc": 0.1,
-              "ece": 0.3079504190526222,
-              "brier": 0.7506021687077564,
-              "mean_conf": 0.40795041905262225,
+              "nll": 1.3036386445451955,
+              "acc": 0.2,
+              "ece": 0.30095968800211725,
+              "brier": 0.8031605478655157,
+              "mean_conf": 0.4567572303779661,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.0,
               "accuracy_at_0_9": null,
               "coverage_at_5pct_error": 0.0,
               "coverage_at_1pct_error": 0.0,
-              "aurc": 0.9354365079365079,
+              "aurc": 0.9309920634920636,
               "error_rate_at_0_9": null,
-              "confidence_bias": 0.3079504190526222,
+              "confidence_bias": 0.2567572303779661,
               "top_bins": {
                 "0.9": {
                   "n": 0,
@@ -9982,32 +9982,32 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 0.0,
-                  "confidence_cutoff": 0.4146924738494299
+                  "confidence_cutoff": 0.4595779615514863
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.125,
-                  "confidence_cutoff": 0.36222461945487916
+                  "confidence_cutoff": 0.41920896115276335
                 }
               },
-              "score_mae": 0.8003548520533641,
-              "ranked_probability_score": 0.26340505165600425
+              "score_mae": 0.8655138350919522,
+              "ranked_probability_score": 0.28876623510920263
             },
             "unknowable_sla_response": {
               "n": 10,
-              "nll": 1.1849194907444107,
-              "acc": 0.3,
-              "ece": 0.23103561236057332,
-              "brier": 0.7234547014672483,
-              "mean_conf": 0.42642234811077023,
+              "nll": 1.2124338307190397,
+              "acc": 0.2,
+              "ece": 0.2761916374516639,
+              "brier": 0.746030154235981,
+              "mean_conf": 0.47619163745166393,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.0,
               "accuracy_at_0_9": null,
               "coverage_at_5pct_error": 0.0,
               "coverage_at_1pct_error": 0.0,
-              "aurc": 0.7066666666666667,
+              "aurc": 0.7266666666666668,
               "error_rate_at_0_9": null,
-              "confidence_bias": 0.12642234811077024,
+              "confidence_bias": 0.2761916374516639,
               "top_bins": {
                 "0.9": {
                   "n": 0,
@@ -10029,24 +10029,24 @@
                 "0.5": {
                   "coverage": 0.6,
                   "accuracy": 0.16666666666666666,
-                  "confidence_cutoff": 0.4009427965249298
+                  "confidence_cutoff": 0.4481450452847792
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.25,
-                  "confidence_cutoff": 0.3769512660219888
+                  "confidence_cutoff": 0.4357074330957959
                 }
               },
-              "score_mae": 0.5576178724416314,
-              "ranked_probability_score": 0.2080023406010124
+              "score_mae": 0.6120337496191068,
+              "ranked_probability_score": 0.21905875267992797
             },
             "unknowable_spend_threshold": {
               "n": 10,
-              "nll": 0.7170838524635513,
+              "nll": 0.736875114520118,
               "acc": 0.5,
-              "ece": 0.04376563079830731,
-              "brier": 0.5072300370099999,
-              "mean_conf": 0.5437656307983073,
+              "ece": 0.07921755937455173,
+              "brier": 0.5233519615225514,
+              "mean_conf": 0.5792175593745517,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.0,
               "accuracy_at_0_9": null,
@@ -10054,7 +10054,7 @@
               "coverage_at_1pct_error": 0.0,
               "aurc": 0.5,
               "error_rate_at_0_9": null,
-              "confidence_bias": 0.04376563079830731,
+              "confidence_bias": 0.07921755937455166,
               "top_bins": {
                 "0.9": {
                   "n": 0,
@@ -10076,22 +10076,22 @@
                 "0.5": {
                   "coverage": 0.6,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.5350574633706056
+                  "confidence_cutoff": 0.5943543182085235
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.5,
-                  "confidence_cutoff": 0.524120391391442
+                  "confidence_cutoff": 0.5139663834645092
                 }
               }
             },
             "unknowable_volume_discount": {
               "n": 10,
-              "nll": 2.5951350581644688,
+              "nll": 2.5691700417991905,
               "acc": 0.4,
-              "ece": 0.5367841896964539,
-              "brier": 1.1191814760526815,
-              "mean_conf": 0.9367841896964538,
+              "ece": 0.5382559019399908,
+              "brier": 1.121603982108248,
+              "mean_conf": 0.9382559019399908,
               "confident_error_rate": 0.5,
               "coverage_at_0_9": 0.8,
               "accuracy_at_0_9": 0.375,
@@ -10099,7 +10099,7 @@
               "coverage_at_1pct_error": 0.0,
               "aurc": 0.5783333333333334,
               "error_rate_at_0_9": 0.625,
-              "confidence_bias": 0.5367841896964538,
+              "confidence_bias": 0.5382559019399907,
               "top_bins": {
                 "0.9": {
                   "n": 8,
@@ -10121,32 +10121,32 @@
                 "0.5": {
                   "coverage": 0.6,
                   "accuracy": 0.3333333333333333,
-                  "confidence_cutoff": 0.9647726422546125
+                  "confidence_cutoff": 0.9634562251390246
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.375,
-                  "confidence_cutoff": 0.9562801656369482
+                  "confidence_cutoff": 0.9614389381469302
                 }
               },
-              "score_mae": 0.9910966577040741,
-              "ranked_probability_score": 0.46291982849495855
+              "score_mae": 0.9905057090116424,
+              "ranked_probability_score": 0.4635245683115593
             },
             "unknowable_warranty_claim": {
               "n": 10,
-              "nll": 1.0334042098876344,
-              "acc": 0.3,
-              "ece": 0.25287419134987343,
-              "brier": 0.6389076662084439,
-              "mean_conf": 0.49347092014402827,
+              "nll": 0.9921422537254998,
+              "acc": 0.5,
+              "ece": 0.2020352656125068,
+              "brier": 0.6100410016954086,
+              "mean_conf": 0.46137759870501566,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.0,
               "accuracy_at_0_9": null,
               "coverage_at_5pct_error": 0.1,
               "coverage_at_1pct_error": 0.1,
-              "aurc": 0.5163095238095238,
+              "aurc": 0.4818650793650794,
               "error_rate_at_0_9": null,
-              "confidence_bias": 0.19347092014402828,
+              "confidence_bias": -0.03862240129498434,
               "top_bins": {
                 "0.9": {
                   "n": 0,
@@ -10167,91 +10167,91 @@
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
-                  "accuracy": 0.4,
-                  "confidence_cutoff": 0.456228187213608
+                  "accuracy": 0.6,
+                  "confidence_cutoff": 0.42317827905448857
                 },
                 "0.8": {
                   "coverage": 0.8,
                   "accuracy": 0.375,
-                  "confidence_cutoff": 0.4479904637339573
+                  "confidence_cutoff": 0.40588767546080573
                 }
               },
-              "score_mae": 0.5853934211989685,
-              "ranked_probability_score": 0.18462148323551122
+              "score_mae": 0.5401415858390292,
+              "ranked_probability_score": 0.17205966045392337
             }
           },
           "variants": {
             "clean": {
               "n": 1156,
-              "nll": 0.6763826525078761,
-              "acc": 0.7352941176470589,
-              "ece": 0.07841307190750933,
-              "brier": 0.3427021673311552,
-              "mean_conf": 0.8137071895545681,
-              "confident_error_rate": 0.03027681660899654,
-              "coverage_at_0_9": 0.5380622837370242,
-              "accuracy_at_0_9": 0.9437299035369775,
-              "coverage_at_5pct_error": 0.49480968858131485,
-              "coverage_at_1pct_error": 0.08996539792387544,
-              "aurc": 0.08414303631087763,
-              "error_rate_at_0_9": 0.05627009646302251,
-              "confidence_bias": 0.07841307190750924,
+              "nll": 0.6634965588347153,
+              "acc": 0.7413494809688581,
+              "ece": 0.07307599837609173,
+              "brier": 0.3373915801751656,
+              "mean_conf": 0.8144254793449499,
+              "confident_error_rate": 0.0328719723183391,
+              "coverage_at_0_9": 0.551038062283737,
+              "accuracy_at_0_9": 0.9403453689167975,
+              "coverage_at_5pct_error": 0.5077854671280276,
+              "coverage_at_1pct_error": 0.1185121107266436,
+              "aurc": 0.08049307232208963,
+              "error_rate_at_0_9": 0.059654631083202514,
+              "confidence_bias": 0.0730759983760918,
               "top_bins": {
                 "0.9": {
-                  "n": 622,
-                  "errors": 35,
-                  "error_rate": 0.05627009646302251
+                  "n": 637,
+                  "errors": 38,
+                  "error_rate": 0.059654631083202514
                 },
                 "0.95": {
-                  "n": 543,
-                  "errors": 25,
-                  "error_rate": 0.04604051565377532
+                  "n": 532,
+                  "errors": 22,
+                  "error_rate": 0.041353383458646614
                 },
                 "0.99": {
-                  "n": 230,
-                  "errors": 5,
-                  "error_rate": 0.021739130434782608
+                  "n": 207,
+                  "errors": 3,
+                  "error_rate": 0.014492753623188406
                 }
               },
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
-                  "accuracy": 0.9480968858131488,
-                  "confidence_cutoff": 0.9341998365643388
+                  "accuracy": 0.9515570934256056,
+                  "confidence_cutoff": 0.9346480717751411
                 },
                 "0.8": {
-                  "coverage": 0.8001730103806228,
-                  "accuracy": 0.8367567567567568,
-                  "confidence_cutoff": 0.603660772359405
+                  "coverage": 0.801038062283737,
+                  "accuracy": 0.8412526997840173,
+                  "confidence_cutoff": 0.5943543182085235
                 }
               },
-              "score_mae": 0.4007031098123003,
-              "ranked_probability_score": 0.13423917717987888
+              "score_mae": 0.3960124111370266,
+              "ranked_probability_score": 0.13525323778002232
             },
             "none_absent": {
               "n": 36,
-              "nll": 1.348957280222087,
+              "nll": 1.2780587311745955,
               "acc": 0.5,
-              "ece": 0.27815703262096336,
-              "brier": 0.7471379703062733,
-              "mean_conf": 0.7039376629223564,
+              "ece": 0.28787280132015003,
+              "brier": 0.7144050226937334,
+              "mean_conf": 0.6869218961640411,
               "confident_error_rate": 0.08333333333333333,
-              "coverage_at_0_9": 0.19444444444444445,
-              "accuracy_at_0_9": 0.5714285714285714,
+              "coverage_at_0_9": 0.1111111111111111,
+              "accuracy_at_0_9": 0.25,
               "coverage_at_5pct_error": 0.0,
               "coverage_at_1pct_error": 0.0,
-              "aurc": 0.5426191429149073,
-              "error_rate_at_0_9": 0.42857142857142855,
-              "confidence_bias": 0.20393766292235638,
+              "aurc": 0.5376174183815399,
+              "error_rate_at_0_9": 0.75,
+              "confidence_bias": 0.1869218961640411,
               "top_bins": {
                 "0.9": {
-                  "n": 7,
+                  "n": 4,
                   "errors": 3,
-                  "error_rate": 0.42857142857142855
+                  "error_rate": 0.75
                 },
                 "0.95": {
-                  "n": 3,
-                  "errors": 3,
+                  "n": 2,
+                  "errors": 2,
                   "error_rate": 1.0
                 },
                 "0.99": {
@@ -10263,44 +10263,44 @@
               "selective": {
                 "0.5": {
                   "coverage": 0.5,
-                  "accuracy": 0.5555555555555556,
-                  "confidence_cutoff": 0.7471419202300652
+                  "accuracy": 0.6111111111111112,
+                  "confidence_cutoff": 0.696060664667048
                 },
                 "0.8": {
                   "coverage": 0.8055555555555556,
-                  "accuracy": 0.4827586206896552,
-                  "confidence_cutoff": 0.4767019156513086
+                  "accuracy": 0.5172413793103449,
+                  "confidence_cutoff": 0.4846763048647645
                 }
               }
             },
             "none_present": {
               "n": 36,
-              "nll": 0.772544457721986,
+              "nll": 0.7460177433221598,
               "acc": 0.7777777777777778,
-              "ece": 0.12491830434756149,
-              "brier": 0.2839371787047634,
-              "mean_conf": 0.8153893515459814,
+              "ece": 0.14627996115067443,
+              "brier": 0.28709997650834473,
+              "mean_conf": 0.8017359444765791,
               "confident_error_rate": 0.0,
-              "coverage_at_0_9": 0.5833333333333334,
+              "coverage_at_0_9": 0.5555555555555556,
               "accuracy_at_0_9": 1.0,
-              "coverage_at_5pct_error": 0.75,
-              "coverage_at_1pct_error": 0.7222222222222222,
-              "aurc": 0.03655983623235726,
+              "coverage_at_5pct_error": 0.7222222222222222,
+              "coverage_at_1pct_error": 0.6666666666666666,
+              "aurc": 0.04264397382032627,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": 0.037611573768203566,
+              "confidence_bias": 0.023958166698801353,
               "top_bins": {
                 "0.9": {
-                  "n": 21,
+                  "n": 20,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.95": {
-                  "n": 19,
+                  "n": 17,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 7,
+                  "n": 6,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -10309,30 +10309,30 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9658129517830465
+                  "confidence_cutoff": 0.9382298345728617
                 },
                 "0.8": {
                   "coverage": 0.8055555555555556,
-                  "accuracy": 0.896551724137931,
-                  "confidence_cutoff": 0.6244481662411653
+                  "accuracy": 0.8620689655172413,
+                  "confidence_cutoff": 0.5789058197682615
                 }
               }
             },
             "permuted": {
               "n": 36,
-              "nll": 0.747860200951205,
+              "nll": 0.7203281270997002,
               "acc": 0.7777777777777778,
-              "ece": 0.09222810570822655,
-              "brier": 0.2793216011169781,
-              "mean_conf": 0.8462041294668327,
+              "ece": 0.09527377914540269,
+              "brier": 0.27961577244776725,
+              "mean_conf": 0.836032418535974,
               "confident_error_rate": 0.0,
               "coverage_at_0_9": 0.6388888888888888,
               "accuracy_at_0_9": 1.0,
-              "coverage_at_5pct_error": 0.7222222222222222,
+              "coverage_at_5pct_error": 0.7777777777777778,
               "coverage_at_1pct_error": 0.6666666666666666,
-              "aurc": 0.03881027559007631,
+              "aurc": 0.03678940551365068,
               "error_rate_at_0_9": 0.0,
-              "confidence_bias": 0.06842635168905487,
+              "confidence_bias": 0.0582546407581962,
               "top_bins": {
                 "0.9": {
                   "n": 23,
@@ -10340,12 +10340,12 @@
                   "error_rate": 0.0
                 },
                 "0.95": {
-                  "n": 20,
+                  "n": 18,
                   "errors": 0,
                   "error_rate": 0.0
                 },
                 "0.99": {
-                  "n": 10,
+                  "n": 9,
                   "errors": 0,
                   "error_rate": 0.0
                 }
@@ -10354,12 +10354,12 @@
                 "0.5": {
                   "coverage": 0.5,
                   "accuracy": 1.0,
-                  "confidence_cutoff": 0.9705536273462746
+                  "confidence_cutoff": 0.9544422568229554
                 },
                 "0.8": {
                   "coverage": 0.8055555555555556,
                   "accuracy": 0.9310344827586207,
-                  "confidence_cutoff": 0.6554697446772016
+                  "confidence_cutoff": 0.6258627504206388
                 }
               }
             }
@@ -10367,56 +10367,56 @@
           "heldout_tasks": {},
           "permutation": {
             "n": 36,
-            "mean_max_delta": 0.08050750512860985,
+            "mean_max_delta": 0.08279943035271198,
             "flip_rate": 0.08333333333333333
           },
           "temperature": 1.0,
           "calibrated_clean": {
             "n": 1046,
-            "nll": 0.6086212459300988,
-            "acc": 0.7734225621414914,
-            "ece": 0.05889322575332617,
-            "brier": 0.3004372452589081,
-            "mean_conf": 0.8323157878948174,
+            "nll": 0.5940603022446113,
+            "acc": 0.7762906309751434,
+            "ece": 0.05558262070294892,
+            "brier": 0.2937157813621403,
+            "mean_conf": 0.8318732516780923,
             "confident_error_rate": 0.02390057361376673,
-            "coverage_at_0_9": 0.5774378585086042,
-            "accuracy_at_0_9": 0.9586092715231788,
-            "coverage_at_5pct_error": 0.6338432122370937,
-            "coverage_at_1pct_error": 0.38336520076481834,
-            "aurc": 0.061361186701696054,
-            "error_rate_at_0_9": 0.041390728476821195,
-            "confidence_bias": 0.058893225753326006,
+            "coverage_at_0_9": 0.5860420650095602,
+            "accuracy_at_0_9": 0.9592169657422512,
+            "coverage_at_5pct_error": 0.6347992351816444,
+            "coverage_at_1pct_error": 0.3030592734225621,
+            "aurc": 0.05816033320114623,
+            "error_rate_at_0_9": 0.040783034257748776,
+            "confidence_bias": 0.05558262070294895,
             "top_bins": {
               "0.9": {
-                "n": 604,
+                "n": 613,
                 "errors": 25,
-                "error_rate": 0.041390728476821195
+                "error_rate": 0.040783034257748776
               },
               "0.95": {
-                "n": 525,
-                "errors": 15,
-                "error_rate": 0.02857142857142857
+                "n": 514,
+                "errors": 12,
+                "error_rate": 0.023346303501945526
               },
               "0.99": {
-                "n": 226,
+                "n": 207,
                 "errors": 3,
-                "error_rate": 0.01327433628318584
+                "error_rate": 0.014492753623188406
               }
             },
             "selective": {
               "0.5": {
-                "coverage": 0.5,
-                "accuracy": 0.97131931166348,
-                "confidence_cutoff": 0.9503957614513975
+                "coverage": 0.5009560229445507,
+                "accuracy": 0.9770992366412213,
+                "confidence_cutoff": 0.947606086730957
               },
               "0.8": {
                 "coverage": 0.8001912045889101,
-                "accuracy": 0.8745519713261649,
-                "confidence_cutoff": 0.6527601881992761
+                "accuracy": 0.8781362007168458,
+                "confidence_cutoff": 0.647434909565297
               }
             },
-            "score_mae": 0.19182332300936422,
-            "ranked_probability_score": 0.05030124964901672
+            "score_mae": 0.18149603870298436,
+            "ranked_probability_score": 0.051063424067515496
           },
           "metric_policy": {
             "version": 2,