diff --git "a/metrics/benchmark.json" "b/metrics/benchmark.json" --- "a/metrics/benchmark.json" +++ "b/metrics/benchmark.json" @@ -1,5 +1,5 @@ { - "scope": "Complete14model native-SystemOne comparison; accuracy%, no universal architecture superiority claim.", + "scope": "Complete13model public native-SystemOne comparison; accuracy%, no universal architecture superiority claim.", "weights": { "old_core": "3/10", "v3_core": "1/4", @@ -66856,5148 +66856,6 @@ }, "metric_scope": "Refused/invalid predictions count incorrect in requested categorical denominator. Missing probabilities stay missing; covered-only metrics never presented as full metrics. Risk-coverage thresholds are retrospective diagnostics, not a deployed error guarantee." } - }, - "Kai": { - "panels": { - "old_core": { - "point": 0.42239583333333336, - "exact_fraction": "811/1920", - "ci95": [ - 0.38872767857142854, - 0.45625 - ] - }, - "v3_core": { - "point": 0.39875, - "exact_fraction": "319/800", - "ci95": [ - 0.3779166666666666, - 0.42000000000000004 - ] - }, - "v4": { - "point": 0.559375, - "exact_fraction": "179/320", - "ci95": [ - 0.5114902041237518, - 0.6073724287974684 - ] - }, - "v5": { - "point": 0.5458333333333333, - "exact_fraction": "131/240", - "ci95": [ - 0.5110750893608074, - 0.581176234826796 - ] - }, - "transfer_v9_test": { - "point": 0.4847036328871893, - "exact_fraction": "507/1046", - "ci95": [ - 0.45394096111466553, - 0.5144787644787645 - ] - }, - "original_four_panel_mean": { - "point": 0.48158854166666665, - "exact_fraction": "18493/38400", - "ci95": [ - 0.4643877712089225, - 0.4996431190752701 - ] - }, - "weighted_mean": { - "point": 0.46489304493307837, - "exact_fraction": "155609/334720", - "ci95": [ - 0.4501699641125866, - 0.47985361481544764 - ] - } - }, - "original_details": { - "old_core": { - "requested": 880, - "valid_predictions": 880, - "valid_probability_rows": 880, - "families": { - "ag_news": { - "correct": 100, - "requested": 128 - }, - "boolean_constraints": { - "correct": 34, - "requested": 64 - }, - "dbpedia_14": { - "correct": 56, - "requested": 112 - }, - "natural_intents": { - "correct": 52, - "requested": 64 - }, - "option_carrier": { - "correct": 1, - "requested": 96 - }, - "ordinal_rubric": { - "correct": 4, - "requested": 64 - }, - "relational_composition": { - "correct": 49, - "requested": 96 - }, - "scoped_evidence": { - "correct": 28, - "requested": 96 - }, - "state_tracking": { - "correct": 29, - "requested": 96 - }, - "unknown_rejection": { - "correct": 27, - "requested": 64 - } - } - }, - "v3_core": { - "requested": 880, - "valid_predictions": 880, - "valid_probability_rows": 880, - "families": { - "canonical_record_identity": { - "correct": 41, - "requested": 80 - }, - "capacitated_assignment": { - "correct": 40, - "requested": 80 - }, - "constraint_assignment": { - "correct": 25, - "requested": 80 - }, - "massive_en": { - "correct": 87, - "requested": 120 - }, - "massive_zh": { - "correct": 81, - "requested": 120 - }, - "multiset_reconciliation": { - "correct": 26, - "requested": 80 - }, - "ordinal_service_loss": { - "correct": 17, - "requested": 80 - }, - "paraconsistent_rule_closure": { - "correct": 19, - "requested": 80 - }, - "temporal_exclusion": { - "correct": 18, - "requested": 80 - }, - "transaction_recovery": { - "correct": 21, - "requested": 80 - } - } - }, - "v4": { - "requested": 480, - "valid_predictions": 480, - "valid_probability_rows": 480, - "families": { - "boolq": { - "correct": 119, - "requested": 160 - }, - "belebele_en": { - "correct": 63, - "requested": 160 - }, - "belebele_zh": { - "correct": 57, - "requested": 160 - } - } - }, - "v5": { - "requested": 480, - "valid_predictions": 480, - "valid_probability_rows": 480, - "families": { - "cosmos_qa": { - "correct": 39, - "requested": 120 - }, - "squad2_answerability": { - "correct": 67, - "requested": 120 - }, - "snli": { - "correct": 41, - "requested": 120 - }, - "qasc": { - "correct": 115, - "requested": 120 - } - } - } - }, - "transfer_diagnostics": { - "clean": { - "requested": 1046, - "valid_probability_rows": 1046, - "correct": 507, - "accuracy": 0.4847036328871893, - "full_probability_metrics": { - "n": 1046, - "nll": 1.155212302090266, - "acc": 0.4847036328871893, - "ece": 0.07953583853933748, - "brier": 0.6066466271423657, - "mean_conf": 0.5582630746492016, - "confident_error_rate": 0.03919694072657744, - "coverage_at_0_9": 0.16061185468451242, - "accuracy_at_0_9": 0.7559523809523809, - "coverage_at_5pct_error": 0.0124282982791587, - "coverage_at_1pct_error": 0.0124282982791587, - "aurc": 0.33433775549030087, - "error_rate_at_0_9": 0.24404761904761904, - "confidence_bias": 0.07355944176201223, - "top_bins": { - "0.9": { - "n": 168, - "errors": 41, - "error_rate": 0.24404761904761904 - }, - "0.95": { - "n": 81, - "errors": 10, - "error_rate": 0.12345679012345678 - }, - "0.99": { - "n": 2, - "errors": 0, - "error_rate": 0.0 - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.6424474187380497, - "confidence_cutoff": 0.5322548449039459 - }, - "0.8": { - "coverage": 0.8001912045889101, - "accuracy": 0.5495818399044206, - "confidence_cutoff": 0.3397521002646066 - } - }, - "score_mae": 0.7276275937425761, - "ranked_probability_score": 0.23169048615956875 - }, - "covered_only_probability_metrics": { - "n": 1046, - "nll": 1.155212302090266, - "acc": 0.4847036328871893, - "ece": 0.07953583853933748, - "brier": 0.6066466271423657, - "mean_conf": 0.5582630746492016, - "confident_error_rate": 0.03919694072657744, - "coverage_at_0_9": 0.16061185468451242, - "accuracy_at_0_9": 0.7559523809523809, - "coverage_at_5pct_error": 0.0124282982791587, - "coverage_at_1pct_error": 0.0124282982791587, - "aurc": 0.33433775549030087, - "error_rate_at_0_9": 0.24404761904761904, - "confidence_bias": 0.07355944176201223, - "top_bins": { - "0.9": { - "n": 168, - "errors": 41, - "error_rate": 0.24404761904761904 - }, - "0.95": { - "n": 81, - "errors": 10, - "error_rate": 0.12345679012345678 - }, - "0.99": { - "n": 2, - "errors": 0, - "error_rate": 0.0 - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.6424474187380497, - "confidence_cutoff": 0.5322548449039459 - }, - "0.8": { - "coverage": 0.8001912045889101, - "accuracy": 0.5495818399044206, - "confidence_cutoff": 0.3397521002646066 - } - }, - "score_mae": 0.7276275937425761, - "ranked_probability_score": 0.23169048615956875 - }, - "coverage": 1.0 - }, - "tasks": { - "buried_emotion": { - "requested": 20, - "valid_probability_rows": 20, - "correct": 11, - "accuracy": 0.55, - "full_probability_metrics": { - "n": 20, - "nll": 1.6434339926856585, - "acc": 0.55, - "ece": 0.20963160665409572, - "brier": 0.7352122696796819, - "mean_conf": 0.4118672036980742, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.5926301063917163, - "error_rate_at_0_9": null, - "confidence_bias": -0.13813279630192582, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.4, - "confidence_cutoff": 0.45318632980297263 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5, - "confidence_cutoff": 0.32968246936798096 - } - } - }, - "covered_only_probability_metrics": { - "n": 20, - "nll": 1.6434339926856585, - "acc": 0.55, - "ece": 0.20963160665409572, - "brier": 0.7352122696796819, - "mean_conf": 0.4118672036980742, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.5926301063917163, - "error_rate_at_0_9": null, - "confidence_bias": -0.13813279630192582, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.4, - "confidence_cutoff": 0.45318632980297263 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5, - "confidence_cutoff": 0.32968246936798096 - } - } - }, - "coverage": 1.0 - }, - "buried_paws": { - "requested": 20, - "valid_probability_rows": 20, - "correct": 7, - "accuracy": 0.35, - "full_probability_metrics": { - "n": 20, - "nll": 0.7827316896839417, - "acc": 0.35, - "ece": 0.24277215003967284, - "brier": 0.5858853695879119, - "mean_conf": 0.5927721500396729, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.6953620351956266, - "error_rate_at_0_9": null, - "confidence_bias": 0.2427721500396729, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.4, - "confidence_cutoff": 0.5672407746315002 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.375, - "confidence_cutoff": 0.5477952659130096 - } - } - }, - "covered_only_probability_metrics": { - "n": 20, - "nll": 0.7827316896839417, - "acc": 0.35, - "ece": 0.24277215003967284, - "brier": 0.5858853695879119, - "mean_conf": 0.5927721500396729, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.6953620351956266, - "error_rate_at_0_9": null, - "confidence_bias": 0.2427721500396729, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.4, - "confidence_cutoff": 0.5672407746315002 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.375, - "confidence_cutoff": 0.5477952659130096 - } - } - }, - "coverage": 1.0 - }, - "buried_qnli": { - "requested": 20, - "valid_probability_rows": 20, - "correct": 11, - "accuracy": 0.55, - "full_probability_metrics": { - "n": 20, - "nll": 0.8555022618540915, - "acc": 0.55, - "ece": 0.2654646812006831, - "brier": 0.6094519995364962, - "mean_conf": 0.7867073891684413, - "confident_error_rate": 0.05, - "coverage_at_0_9": 0.25, - "accuracy_at_0_9": 0.8, - "coverage_at_5pct_error": 0.1, - "coverage_at_1pct_error": 0.1, - "aurc": 0.34032932603172544, - "error_rate_at_0_9": 0.2, - "confidence_bias": 0.23670738916844125, - "top_bins": { - "0.9": { - "n": 5, - "errors": 1, - "error_rate": 0.2 - }, - "0.95": { - "n": 2, - "errors": 0, - "error_rate": 0.0 - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.6, - "confidence_cutoff": 0.8149255514144897 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5625, - "confidence_cutoff": 0.6623975336551666 - } - } - }, - "covered_only_probability_metrics": { - "n": 20, - "nll": 0.8555022618540915, - "acc": 0.55, - "ece": 0.2654646812006831, - "brier": 0.6094519995364962, - "mean_conf": 0.7867073891684413, - "confident_error_rate": 0.05, - "coverage_at_0_9": 0.25, - "accuracy_at_0_9": 0.8, - "coverage_at_5pct_error": 0.1, - "coverage_at_1pct_error": 0.1, - "aurc": 0.34032932603172544, - "error_rate_at_0_9": 0.2, - "confidence_bias": 0.23670738916844125, - "top_bins": { - "0.9": { - "n": 5, - "errors": 1, - "error_rate": 0.2 - }, - "0.95": { - "n": 2, - "errors": 0, - "error_rate": 0.0 - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.6, - "confidence_cutoff": 0.8149255514144897 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5625, - "confidence_cutoff": 0.6623975336551666 - } - } - }, - "coverage": 1.0 - }, - "buried_tweet_offensive": { - "requested": 20, - "valid_probability_rows": 20, - "correct": 15, - "accuracy": 0.75, - "full_probability_metrics": { - "n": 20, - "nll": 0.5035467531682788, - "acc": 0.75, - "ece": 0.19359442964196205, - "brier": 0.32708305202234894, - "mean_conf": 0.6864339232444763, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.05, - "accuracy_at_0_9": 1.0, - "coverage_at_5pct_error": 0.2, - "coverage_at_1pct_error": 0.2, - "aurc": 0.10817796152935473, - "error_rate_at_0_9": 0.0, - "confidence_bias": -0.06356607675552373, - "top_bins": { - "0.9": { - "n": 1, - "errors": 0, - "error_rate": 0.0 - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.9, - "confidence_cutoff": 0.6365915834903717 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.875, - "confidence_cutoff": 0.5576783418655396 - } - } - }, - "covered_only_probability_metrics": { - "n": 20, - "nll": 0.5035467531682788, - "acc": 0.75, - "ece": 0.19359442964196205, - "brier": 0.32708305202234894, - "mean_conf": 0.6864339232444763, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.05, - "accuracy_at_0_9": 1.0, - "coverage_at_5pct_error": 0.2, - "coverage_at_1pct_error": 0.2, - "aurc": 0.10817796152935473, - "error_rate_at_0_9": 0.0, - "confidence_bias": -0.06356607675552373, - "top_bins": { - "0.9": { - "n": 1, - "errors": 0, - "error_rate": 0.0 - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.9, - "confidence_cutoff": 0.6365915834903717 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.875, - "confidence_cutoff": 0.5576783418655396 - } - } - }, - "coverage": 1.0 - }, - "composition_final_combination": { - "requested": 32, - "valid_probability_rows": 32, - "correct": 11, - "accuracy": 0.34375, - "full_probability_metrics": { - "n": 32, - "nll": 0.7235207463951194, - "acc": 0.34375, - "ece": 0.19085747487379479, - "brier": 0.5302537703699604, - "mean_conf": 0.5346074748737948, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.03125, - "coverage_at_1pct_error": 0.03125, - "aurc": 0.6815349342961152, - "error_rate_at_0_9": null, - "confidence_bias": 0.19085747487379479, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.25, - "confidence_cutoff": 0.5424632430076599 - }, - "0.8": { - "coverage": 0.8125, - "accuracy": 0.34615384615384615, - "confidence_cutoff": 0.50784832239151 - } - } - }, - "covered_only_probability_metrics": { - "n": 32, - "nll": 0.7235207463951194, - "acc": 0.34375, - "ece": 0.19085747487379479, - "brier": 0.5302537703699604, - "mean_conf": 0.5346074748737948, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.03125, - "coverage_at_1pct_error": 0.03125, - "aurc": 0.6815349342961152, - "error_rate_at_0_9": null, - "confidence_bias": 0.19085747487379479, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.25, - "confidence_cutoff": 0.5424632430076599 - }, - "0.8": { - "coverage": 0.8125, - "accuracy": 0.34615384615384615, - "confidence_cutoff": 0.50784832239151 - } - } - }, - "coverage": 1.0 - }, - "composition_final_exception": { - "requested": 32, - "valid_probability_rows": 32, - "correct": 20, - "accuracy": 0.625, - "full_probability_metrics": { - "n": 32, - "nll": 0.6887212067364372, - "acc": 0.625, - "ece": 0.10813984954395073, - "brier": 0.4955799333552681, - "mean_conf": 0.5168601504560493, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.03125, - "coverage_at_1pct_error": 0.03125, - "aurc": 0.3965946156627872, - "error_rate_at_0_9": null, - "confidence_bias": -0.10813984954395073, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.5, - "confidence_cutoff": 0.5141306672731241 - }, - "0.8": { - "coverage": 0.8125, - "accuracy": 0.5769230769230769, - "confidence_cutoff": 0.502220630645752 - } - } - }, - "covered_only_probability_metrics": { - "n": 32, - "nll": 0.6887212067364372, - "acc": 0.625, - "ece": 0.10813984954395073, - "brier": 0.4955799333552681, - "mean_conf": 0.5168601504560493, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.03125, - "coverage_at_1pct_error": 0.03125, - "aurc": 0.3965946156627872, - "error_rate_at_0_9": null, - "confidence_bias": -0.10813984954395073, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.5, - "confidence_cutoff": 0.5141306672731241 - }, - "0.8": { - "coverage": 0.8125, - "accuracy": 0.5769230769230769, - "confidence_cutoff": 0.502220630645752 - } - } - }, - "coverage": 1.0 - }, - "composition_final_negation": { - "requested": 32, - "valid_probability_rows": 32, - "correct": 18, - "accuracy": 0.5625, - "full_probability_metrics": { - "n": 32, - "nll": 0.6923211972690404, - "acc": 0.5625, - "ece": 0.07358545691148824, - "brier": 0.4989534073073557, - "mean_conf": 0.5570303887138711, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.03125, - "coverage_at_1pct_error": 0.03125, - "aurc": 0.45051881344545247, - "error_rate_at_0_9": null, - "confidence_bias": -0.005469611286128884, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.5625, - "confidence_cutoff": 0.5467859776743663 - }, - "0.8": { - "coverage": 0.8125, - "accuracy": 0.5384615384615384, - "confidence_cutoff": 0.5280904455131027 - } - } - }, - "covered_only_probability_metrics": { - "n": 32, - "nll": 0.6923211972690404, - "acc": 0.5625, - "ece": 0.07358545691148824, - "brier": 0.4989534073073557, - "mean_conf": 0.5570303887138711, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.03125, - "coverage_at_1pct_error": 0.03125, - "aurc": 0.45051881344545247, - "error_rate_at_0_9": null, - "confidence_bias": -0.005469611286128884, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.5625, - "confidence_cutoff": 0.5467859776743663 - }, - "0.8": { - "coverage": 0.8125, - "accuracy": 0.5384615384615384, - "confidence_cutoff": 0.5280904455131027 - } - } - }, - "coverage": 1.0 - }, - "contrastive_authorization": { - "requested": 40, - "valid_probability_rows": 40, - "correct": 20, - "accuracy": 0.5, - "full_probability_metrics": { - "n": 40, - "nll": 1.4121071983456814, - "acc": 0.5, - "ece": 0.4394341051578522, - "brier": 0.879494291145727, - "mean_conf": 0.9394341051578522, - "confident_error_rate": 0.5, - "coverage_at_0_9": 1.0, - "accuracy_at_0_9": 0.5, - "coverage_at_5pct_error": 0.125, - "coverage_at_1pct_error": 0.125, - "aurc": 0.3325839665024844, - "error_rate_at_0_9": 0.5, - "confidence_bias": 0.4394341051578522, - "top_bins": { - "0.9": { - "n": 40, - "errors": 20, - "error_rate": 0.5 - }, - "0.95": { - "n": 7, - "errors": 1, - "error_rate": 0.14285714285714285 - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.55, - "confidence_cutoff": 0.9448155164718628 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.53125, - "confidence_cutoff": 0.930177628993988 - } - } - }, - "covered_only_probability_metrics": { - "n": 40, - "nll": 1.4121071983456814, - "acc": 0.5, - "ece": 0.4394341051578522, - "brier": 0.879494291145727, - "mean_conf": 0.9394341051578522, - "confident_error_rate": 0.5, - "coverage_at_0_9": 1.0, - "accuracy_at_0_9": 0.5, - "coverage_at_5pct_error": 0.125, - "coverage_at_1pct_error": 0.125, - "aurc": 0.3325839665024844, - "error_rate_at_0_9": 0.5, - "confidence_bias": 0.4394341051578522, - "top_bins": { - "0.9": { - "n": 40, - "errors": 20, - "error_rate": 0.5 - }, - "0.95": { - "n": 7, - "errors": 1, - "error_rate": 0.14285714285714285 - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.55, - "confidence_cutoff": 0.9448155164718628 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.53125, - "confidence_cutoff": 0.930177628993988 - } - } - }, - "coverage": 1.0 - }, - "contrastive_deadline": { - "requested": 40, - "valid_probability_rows": 40, - "correct": 19, - "accuracy": 0.475, - "full_probability_metrics": { - "n": 40, - "nll": 1.0561322346421498, - "acc": 0.475, - "ece": 0.14689049363580786, - "brier": 0.6378429454852624, - "mean_conf": 0.4090969440305655, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.5036059354521869, - "error_rate_at_0_9": null, - "confidence_bias": -0.06590305596943447, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.6, - "confidence_cutoff": 0.4019584655761719 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5, - "confidence_cutoff": 0.37009772658348083 - } - }, - "score_mae": 0.7496014343166787, - "ranked_probability_score": 0.2241171523926872 - }, - "covered_only_probability_metrics": { - "n": 40, - "nll": 1.0561322346421498, - "acc": 0.475, - "ece": 0.14689049363580786, - "brier": 0.6378429454852624, - "mean_conf": 0.4090969440305655, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.5036059354521869, - "error_rate_at_0_9": null, - "confidence_bias": -0.06590305596943447, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.6, - "confidence_cutoff": 0.4019584655761719 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5, - "confidence_cutoff": 0.37009772658348083 - } - }, - "score_mae": 0.7496014343166787, - "ranked_probability_score": 0.2241171523926872 - }, - "coverage": 1.0 - }, - "emotion": { - "requested": 80, - "valid_probability_rows": 80, - "correct": 42, - "accuracy": 0.525, - "full_probability_metrics": { - "n": 80, - "nll": 1.2463184110640215, - "acc": 0.525, - "ece": 0.06492680838097321, - "brier": 0.6039373195122416, - "mean_conf": 0.4982290427346596, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0125, - "accuracy_at_0_9": 1.0, - "coverage_at_5pct_error": 0.0125, - "coverage_at_1pct_error": 0.0125, - "aurc": 0.32277654532687594, - "error_rate_at_0_9": 0.0, - "confidence_bias": -0.0267709572653404, - "top_bins": { - "0.9": { - "n": 1, - "errors": 0, - "error_rate": 0.0 - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.7, - "confidence_cutoff": 0.45944378192850677 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5625, - "confidence_cutoff": 0.357612487989571 - } - } - }, - "covered_only_probability_metrics": { - "n": 80, - "nll": 1.2463184110640215, - "acc": 0.525, - "ece": 0.06492680838097321, - "brier": 0.6039373195122416, - "mean_conf": 0.4982290427346596, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0125, - "accuracy_at_0_9": 1.0, - "coverage_at_5pct_error": 0.0125, - "coverage_at_1pct_error": 0.0125, - "aurc": 0.32277654532687594, - "error_rate_at_0_9": 0.0, - "confidence_bias": -0.0267709572653404, - "top_bins": { - "0.9": { - "n": 1, - "errors": 0, - "error_rate": 0.0 - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.7, - "confidence_cutoff": 0.45944378192850677 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5625, - "confidence_cutoff": 0.357612487989571 - } - } - }, - "coverage": 1.0 - }, - "mmlu": { - "requested": 80, - "valid_probability_rows": 80, - "correct": 25, - "accuracy": 0.3125, - "full_probability_metrics": { - "n": 80, - "nll": 1.4095109007441815, - "acc": 0.3125, - "ece": 0.12076733954335728, - "brier": 0.7561817190247881, - "mean_conf": 0.4170573963746701, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.6634931331685283, - "error_rate_at_0_9": null, - "confidence_bias": 0.1045573963746701, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.375, - "confidence_cutoff": 0.38152980235919653 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.3125, - "confidence_cutoff": 0.3282719703791155 - } - } - }, - "covered_only_probability_metrics": { - "n": 80, - "nll": 1.4095109007441815, - "acc": 0.3125, - "ece": 0.12076733954335728, - "brier": 0.7561817190247881, - "mean_conf": 0.4170573963746701, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.6634931331685283, - "error_rate_at_0_9": null, - "confidence_bias": 0.1045573963746701, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.375, - "confidence_cutoff": 0.38152980235919653 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.3125, - "confidence_cutoff": 0.3282719703791155 - } - } - }, - "coverage": 1.0 - }, - "mmlu_pro": { - "requested": 200, - "valid_probability_rows": 200, - "correct": 35, - "accuracy": 0.175, - "full_probability_metrics": { - "n": 200, - "nll": 2.2624765159160414, - "acc": 0.175, - "ece": 0.08681084104213858, - "brier": 0.9071117655594688, - "mean_conf": 0.23055428589672733, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.863247364071338, - "error_rate_at_0_9": null, - "confidence_bias": 0.05555428589672734, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.17, - "confidence_cutoff": 0.19904440930882775 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.175, - "confidence_cutoff": 0.15484475801056702 - } - } - }, - "covered_only_probability_metrics": { - "n": 200, - "nll": 2.2624765159160414, - "acc": 0.175, - "ece": 0.08681084104213858, - "brier": 0.9071117655594688, - "mean_conf": 0.23055428589672733, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.863247364071338, - "error_rate_at_0_9": null, - "confidence_bias": 0.05555428589672734, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.17, - "confidence_cutoff": 0.19904440930882775 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.175, - "confidence_cutoff": 0.15484475801056702 - } - } - }, - "coverage": 1.0 - }, - "paws": { - "requested": 80, - "valid_probability_rows": 80, - "correct": 38, - "accuracy": 0.475, - "full_probability_metrics": { - "n": 80, - "nll": 0.7090254071044348, - "acc": 0.475, - "ece": 0.1666798379272222, - "brier": 0.5166084866955849, - "mean_conf": 0.6398283705115319, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0125, - "accuracy_at_0_9": 1.0, - "coverage_at_5pct_error": 0.05, - "coverage_at_1pct_error": 0.05, - "aurc": 0.3926720126883591, - "error_rate_at_0_9": 0.0, - "confidence_bias": 0.16482837051153187, - "top_bins": { - "0.9": { - "n": 1, - "errors": 0, - "error_rate": 0.0 - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.625, - "confidence_cutoff": 0.6297575533390045 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.515625, - "confidence_cutoff": 0.5585934221744537 - } - } - }, - "covered_only_probability_metrics": { - "n": 80, - "nll": 0.7090254071044348, - "acc": 0.475, - "ece": 0.1666798379272222, - "brier": 0.5166084866955849, - "mean_conf": 0.6398283705115319, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0125, - "accuracy_at_0_9": 1.0, - "coverage_at_5pct_error": 0.05, - "coverage_at_1pct_error": 0.05, - "aurc": 0.3926720126883591, - "error_rate_at_0_9": 0.0, - "confidence_bias": 0.16482837051153187, - "top_bins": { - "0.9": { - "n": 1, - "errors": 0, - "error_rate": 0.0 - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.625, - "confidence_cutoff": 0.6297575533390045 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.515625, - "confidence_cutoff": 0.5585934221744537 - } - } - }, - "coverage": 1.0 - }, - "qnli": { - "requested": 80, - "valid_probability_rows": 80, - "correct": 57, - "accuracy": 0.7125, - "full_probability_metrics": { - "n": 80, - "nll": 0.7029168919881199, - "acc": 0.7125, - "ece": 0.1819537915289402, - "brier": 0.44881665172376445, - "mean_conf": 0.8604280255734921, - "confident_error_rate": 0.1125, - "coverage_at_0_9": 0.45, - "accuracy_at_0_9": 0.75, - "coverage_at_5pct_error": 0.05, - "coverage_at_1pct_error": 0.05, - "aurc": 0.22011525782827887, - "error_rate_at_0_9": 0.25, - "confidence_bias": 0.14792802557349205, - "top_bins": { - "0.9": { - "n": 36, - "errors": 9, - "error_rate": 0.25 - }, - "0.95": { - "n": 22, - "errors": 3, - "error_rate": 0.13636363636363635 - }, - "0.99": { - "n": 1, - "errors": 0, - "error_rate": 0.0 - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.75, - "confidence_cutoff": 0.8962836265563965 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.734375, - "confidence_cutoff": 0.7553151249885559 - } - } - }, - "covered_only_probability_metrics": { - "n": 80, - "nll": 0.7029168919881199, - "acc": 0.7125, - "ece": 0.1819537915289402, - "brier": 0.44881665172376445, - "mean_conf": 0.8604280255734921, - "confident_error_rate": 0.1125, - "coverage_at_0_9": 0.45, - "accuracy_at_0_9": 0.75, - "coverage_at_5pct_error": 0.05, - "coverage_at_1pct_error": 0.05, - "aurc": 0.22011525782827887, - "error_rate_at_0_9": 0.25, - "confidence_bias": 0.14792802557349205, - "top_bins": { - "0.9": { - "n": 36, - "errors": 9, - "error_rate": 0.25 - }, - "0.95": { - "n": 22, - "errors": 3, - "error_rate": 0.13636363636363635 - }, - "0.99": { - "n": 1, - "errors": 0, - "error_rate": 0.0 - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.75, - "confidence_cutoff": 0.8962836265563965 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.734375, - "confidence_cutoff": 0.7553151249885559 - } - } - }, - "coverage": 1.0 - }, - "sciq": { - "requested": 80, - "valid_probability_rows": 80, - "correct": 74, - "accuracy": 0.925, - "full_probability_metrics": { - "n": 80, - "nll": 0.3412942378197378, - "acc": 0.925, - "ece": 0.17180305325181675, - "brier": 0.14992498517720326, - "mean_conf": 0.7651692762303128, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.2875, - "accuracy_at_0_9": 1.0, - "coverage_at_5pct_error": 0.95, - "coverage_at_1pct_error": 0.8, - "aurc": 0.006306048389760769, - "error_rate_at_0_9": 0.0, - "confidence_bias": -0.1598307237696872, - "top_bins": { - "0.9": { - "n": 23, - "errors": 0, - "error_rate": 0.0 - }, - "0.95": { - "n": 8, - "errors": 0, - "error_rate": 0.0 - }, - "0.99": { - "n": 1, - "errors": 0, - "error_rate": 0.0 - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 1.0, - "confidence_cutoff": 0.8314547840754842 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 1.0, - "confidence_cutoff": 0.599574194858055 - } - } - }, - "covered_only_probability_metrics": { - "n": 80, - "nll": 0.3412942378197378, - "acc": 0.925, - "ece": 0.17180305325181675, - "brier": 0.14992498517720326, - "mean_conf": 0.7651692762303128, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.2875, - "accuracy_at_0_9": 1.0, - "coverage_at_5pct_error": 0.95, - "coverage_at_1pct_error": 0.8, - "aurc": 0.006306048389760769, - "error_rate_at_0_9": 0.0, - "confidence_bias": -0.1598307237696872, - "top_bins": { - "0.9": { - "n": 23, - "errors": 0, - "error_rate": 0.0 - }, - "0.95": { - "n": 8, - "errors": 0, - "error_rate": 0.0 - }, - "0.99": { - "n": 1, - "errors": 0, - "error_rate": 0.0 - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 1.0, - "confidence_cutoff": 0.8314547840754842 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 1.0, - "confidence_cutoff": 0.599574194858055 - } - } - }, - "coverage": 1.0 - }, - "tweet_offensive": { - "requested": 80, - "valid_probability_rows": 80, - "correct": 63, - "accuracy": 0.7875, - "full_probability_metrics": { - "n": 80, - "nll": 0.38772103274242486, - "acc": 0.7875, - "ece": 0.07751428142655643, - "brier": 0.25593492533075707, - "mean_conf": 0.8444197329459712, - "confident_error_rate": 0.0125, - "coverage_at_0_9": 0.5125, - "accuracy_at_0_9": 0.975609756097561, - "coverage_at_5pct_error": 0.525, - "coverage_at_1pct_error": 0.4875, - "aurc": 0.05995296937845444, - "error_rate_at_0_9": 0.024390243902439025, - "confidence_bias": 0.05691973294597119, - "top_bins": { - "0.9": { - "n": 41, - "errors": 1, - "error_rate": 0.024390243902439025 - }, - "0.95": { - "n": 29, - "errors": 0, - "error_rate": 0.0 - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.975, - "confidence_cutoff": 0.9004859104752541 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.875, - "confidence_cutoff": 0.7327178418636322 - } - } - }, - "covered_only_probability_metrics": { - "n": 80, - "nll": 0.38772103274242486, - "acc": 0.7875, - "ece": 0.07751428142655643, - "brier": 0.25593492533075707, - "mean_conf": 0.8444197329459712, - "confident_error_rate": 0.0125, - "coverage_at_0_9": 0.5125, - "accuracy_at_0_9": 0.975609756097561, - "coverage_at_5pct_error": 0.525, - "coverage_at_1pct_error": 0.4875, - "aurc": 0.05995296937845444, - "error_rate_at_0_9": 0.024390243902439025, - "confidence_bias": 0.05691973294597119, - "top_bins": { - "0.9": { - "n": 41, - "errors": 1, - "error_rate": 0.024390243902439025 - }, - "0.95": { - "n": 29, - "errors": 0, - "error_rate": 0.0 - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.975, - "confidence_cutoff": 0.9004859104752541 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.875, - "confidence_cutoff": 0.7327178418636322 - } - } - }, - "coverage": 1.0 - }, - "unknowable_control_age_eligibility": { - "requested": 10, - "valid_probability_rows": 10, - "correct": 5, - "accuracy": 0.5, - "full_probability_metrics": { - "n": 10, - "nll": 1.6191405658106865, - "acc": 0.5, - "ece": 0.45958920121192937, - "brier": 0.9210477728721707, - "mean_conf": 0.9595892012119294, - "confident_error_rate": 0.5, - "coverage_at_0_9": 1.0, - "accuracy_at_0_9": 0.5, - "coverage_at_5pct_error": 0.1, - "coverage_at_1pct_error": 0.1, - "aurc": 0.4106349206349207, - "error_rate_at_0_9": 0.5, - "confidence_bias": 0.45958920121192937, - "top_bins": { - "0.9": { - "n": 10, - "errors": 5, - "error_rate": 0.5 - }, - "0.95": { - "n": 10, - "errors": 5, - "error_rate": 0.5 - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.6, - "confidence_cutoff": 0.9605617523193359 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5, - "confidence_cutoff": 0.9569776654243469 - } - } - }, - "covered_only_probability_metrics": { - "n": 10, - "nll": 1.6191405658106865, - "acc": 0.5, - "ece": 0.45958920121192937, - "brier": 0.9210477728721707, - "mean_conf": 0.9595892012119294, - "confident_error_rate": 0.5, - "coverage_at_0_9": 1.0, - "accuracy_at_0_9": 0.5, - "coverage_at_5pct_error": 0.1, - "coverage_at_1pct_error": 0.1, - "aurc": 0.4106349206349207, - "error_rate_at_0_9": 0.5, - "confidence_bias": 0.45958920121192937, - "top_bins": { - "0.9": { - "n": 10, - "errors": 5, - "error_rate": 0.5 - }, - "0.95": { - "n": 10, - "errors": 5, - "error_rate": 0.5 - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.6, - "confidence_cutoff": 0.9605617523193359 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5, - "confidence_cutoff": 0.9569776654243469 - } - } - }, - "coverage": 1.0 - }, - "unknowable_control_authorization": { - "requested": 10, - "valid_probability_rows": 10, - "correct": 5, - "accuracy": 0.5, - "full_probability_metrics": { - "n": 10, - "nll": 1.4433423409086612, - "acc": 0.5, - "ece": 0.4431973397731781, - "brier": 0.8877473385116084, - "mean_conf": 0.9431973397731781, - "confident_error_rate": 0.5, - "coverage_at_0_9": 1.0, - "accuracy_at_0_9": 0.5, - "coverage_at_5pct_error": 0.2, - "coverage_at_1pct_error": 0.2, - "aurc": 0.31007936507936507, - "error_rate_at_0_9": 0.5, - "confidence_bias": 0.4431973397731781, - "top_bins": { - "0.9": { - "n": 10, - "errors": 5, - "error_rate": 0.5 - }, - "0.95": { - "n": 3, - "errors": 1, - "error_rate": 0.3333333333333333 - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.8, - "confidence_cutoff": 0.9448668360710144 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5, - "confidence_cutoff": 0.9334832429885864 - } - } - }, - "covered_only_probability_metrics": { - "n": 10, - "nll": 1.4433423409086612, - "acc": 0.5, - "ece": 0.4431973397731781, - "brier": 0.8877473385116084, - "mean_conf": 0.9431973397731781, - "confident_error_rate": 0.5, - "coverage_at_0_9": 1.0, - "accuracy_at_0_9": 0.5, - "coverage_at_5pct_error": 0.2, - "coverage_at_1pct_error": 0.2, - "aurc": 0.31007936507936507, - "error_rate_at_0_9": 0.5, - "confidence_bias": 0.4431973397731781, - "top_bins": { - "0.9": { - "n": 10, - "errors": 5, - "error_rate": 0.5 - }, - "0.95": { - "n": 3, - "errors": 1, - "error_rate": 0.3333333333333333 - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.8, - "confidence_cutoff": 0.9448668360710144 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5, - "confidence_cutoff": 0.9334832429885864 - } - } - }, - "coverage": 1.0 - }, - "unknowable_control_deadline": { - "requested": 10, - "valid_probability_rows": 10, - "correct": 5, - "accuracy": 0.5, - "full_probability_metrics": { - "n": 10, - "nll": 1.0534611380665015, - "acc": 0.5, - "ece": 0.10049157245677873, - "brier": 0.6380457356055647, - "mean_conf": 0.39950842754322125, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.49992063492063493, - "error_rate_at_0_9": null, - "confidence_bias": -0.10049157245677875, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.6, - "confidence_cutoff": 0.40330082178115845 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5, - "confidence_cutoff": 0.3643658350597073 - } - }, - "score_mae": 0.6959988628582565, - "ranked_probability_score": 0.2123149634956698 - }, - "covered_only_probability_metrics": { - "n": 10, - "nll": 1.0534611380665015, - "acc": 0.5, - "ece": 0.10049157245677873, - "brier": 0.6380457356055647, - "mean_conf": 0.39950842754322125, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.49992063492063493, - "error_rate_at_0_9": null, - "confidence_bias": -0.10049157245677875, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.6, - "confidence_cutoff": 0.40330082178115845 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5, - "confidence_cutoff": 0.3643658350597073 - } - }, - "score_mae": 0.6959988628582565, - "ranked_probability_score": 0.2123149634956698 - }, - "coverage": 1.0 - }, - "unknowable_control_late_fee": { - "requested": 10, - "valid_probability_rows": 10, - "correct": 3, - "accuracy": 0.3, - "full_probability_metrics": { - "n": 10, - "nll": 1.2971820726980325, - "acc": 0.3, - "ece": 0.317867469435447, - "brier": 0.8158706675338253, - "mean_conf": 0.6178674694354471, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.7722619047619048, - "error_rate_at_0_9": null, - "confidence_bias": 0.31786746943544714, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.4, - "confidence_cutoff": 0.6193228548563378 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.375, - "confidence_cutoff": 0.6018166721408568 - } - }, - "score_mae": 0.7810463516446486, - "ranked_probability_score": 0.2694903445971064 - }, - "covered_only_probability_metrics": { - "n": 10, - "nll": 1.2971820726980325, - "acc": 0.3, - "ece": 0.317867469435447, - "brier": 0.8158706675338253, - "mean_conf": 0.6178674694354471, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.7722619047619048, - "error_rate_at_0_9": null, - "confidence_bias": 0.31786746943544714, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.4, - "confidence_cutoff": 0.6193228548563378 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.375, - "confidence_cutoff": 0.6018166721408568 - } - }, - "score_mae": 0.7810463516446486, - "ranked_probability_score": 0.2694903445971064 - }, - "coverage": 1.0 - }, - "unknowable_control_quantity_limit": { - "requested": 10, - "valid_probability_rows": 10, - "correct": 5, - "accuracy": 0.5, - "full_probability_metrics": { - "n": 10, - "nll": 1.0767010069332539, - "acc": 0.5, - "ece": 0.12858793102984267, - "brier": 0.6522324327497165, - "mean_conf": 0.3714120689701573, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.2, - "coverage_at_1pct_error": 0.2, - "aurc": 0.37492063492063493, - "error_rate_at_0_9": null, - "confidence_bias": -0.12858793102984267, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.6, - "confidence_cutoff": 0.3743864484314189 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5, - "confidence_cutoff": 0.37018098323997906 - } - } - }, - "covered_only_probability_metrics": { - "n": 10, - "nll": 1.0767010069332539, - "acc": 0.5, - "ece": 0.12858793102984267, - "brier": 0.6522324327497165, - "mean_conf": 0.3714120689701573, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.2, - "coverage_at_1pct_error": 0.2, - "aurc": 0.37492063492063493, - "error_rate_at_0_9": null, - "confidence_bias": -0.12858793102984267, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.6, - "confidence_cutoff": 0.3743864484314189 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5, - "confidence_cutoff": 0.37018098323997906 - } - } - }, - "coverage": 1.0 - }, - "unknowable_control_return_window": { - "requested": 10, - "valid_probability_rows": 10, - "correct": 5, - "accuracy": 0.5, - "full_probability_metrics": { - "n": 10, - "nll": 0.8455590159265375, - "acc": 0.5, - "ece": 0.25936471819877627, - "brier": 0.6308352614839883, - "mean_conf": 0.7593647181987763, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.5249206349206349, - "error_rate_at_0_9": null, - "confidence_bias": 0.25936471819877627, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.6, - "confidence_cutoff": 0.7657169103622437 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5, - "confidence_cutoff": 0.7524310350418091 - } - } - }, - "covered_only_probability_metrics": { - "n": 10, - "nll": 0.8455590159265375, - "acc": 0.5, - "ece": 0.25936471819877627, - "brier": 0.6308352614839883, - "mean_conf": 0.7593647181987763, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.5249206349206349, - "error_rate_at_0_9": null, - "confidence_bias": 0.25936471819877627, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.6, - "confidence_cutoff": 0.7657169103622437 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5, - "confidence_cutoff": 0.7524310350418091 - } - } - }, - "coverage": 1.0 - }, - "unknowable_control_shipping_delay": { - "requested": 10, - "valid_probability_rows": 10, - "correct": 2, - "accuracy": 0.2, - "full_probability_metrics": { - "n": 10, - "nll": 1.3095807829358257, - "acc": 0.2, - "ece": 0.22123073572301039, - "brier": 0.7834211221927093, - "mean_conf": 0.4212307357230104, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.7475396825396825, - "error_rate_at_0_9": null, - "confidence_bias": 0.22123073572301039, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.4, - "confidence_cutoff": 0.42378672325928923 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.25, - "confidence_cutoff": 0.41109954698235474 - } - }, - "score_mae": 0.8366539786236817, - "ranked_probability_score": 0.27876383225163887 - }, - "covered_only_probability_metrics": { - "n": 10, - "nll": 1.3095807829358257, - "acc": 0.2, - "ece": 0.22123073572301039, - "brier": 0.7834211221927093, - "mean_conf": 0.4212307357230104, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.7475396825396825, - "error_rate_at_0_9": null, - "confidence_bias": 0.22123073572301039, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.4, - "confidence_cutoff": 0.42378672325928923 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.25, - "confidence_cutoff": 0.41109954698235474 - } - }, - "score_mae": 0.8366539786236817, - "ranked_probability_score": 0.27876383225163887 - }, - "coverage": 1.0 - }, - "unknowable_control_sla_response": { - "requested": 10, - "valid_probability_rows": 10, - "correct": 2, - "accuracy": 0.2, - "full_probability_metrics": { - "n": 10, - "nll": 1.129793073433971, - "acc": 0.2, - "ece": 0.3474792748375072, - "brier": 0.6929538515677903, - "mean_conf": 0.44954587363931975, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.1, - "coverage_at_1pct_error": 0.1, - "aurc": 0.6225396825396825, - "error_rate_at_0_9": null, - "confidence_bias": 0.24954587363931974, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.4, - "confidence_cutoff": 0.4481564857889793 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.25, - "confidence_cutoff": 0.42062819630843695 - } - }, - "score_mae": 0.6323024165832265, - "ranked_probability_score": 0.2112726311516188 - }, - "covered_only_probability_metrics": { - "n": 10, - "nll": 1.129793073433971, - "acc": 0.2, - "ece": 0.3474792748375072, - "brier": 0.6929538515677903, - "mean_conf": 0.44954587363931975, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.1, - "coverage_at_1pct_error": 0.1, - "aurc": 0.6225396825396825, - "error_rate_at_0_9": null, - "confidence_bias": 0.24954587363931974, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.4, - "confidence_cutoff": 0.4481564857889793 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.25, - "confidence_cutoff": 0.42062819630843695 - } - }, - "score_mae": 0.6323024165832265, - "ranked_probability_score": 0.2112726311516188 - }, - "coverage": 1.0 - }, - "unknowable_control_spend_threshold": { - "requested": 10, - "valid_probability_rows": 10, - "correct": 5, - "accuracy": 0.5, - "full_probability_metrics": { - "n": 10, - "nll": 1.0100486225373588, - "acc": 0.5, - "ece": 0.12467851115452944, - "brier": 0.6105381721612377, - "mean_conf": 0.37532148884547056, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.1, - "coverage_at_1pct_error": 0.1, - "aurc": 0.41103174603174597, - "error_rate_at_0_9": null, - "confidence_bias": -0.12467851115452944, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.6, - "confidence_cutoff": 0.3749748282127031 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5, - "confidence_cutoff": 0.37206384539604187 - } - } - }, - "covered_only_probability_metrics": { - "n": 10, - "nll": 1.0100486225373588, - "acc": 0.5, - "ece": 0.12467851115452944, - "brier": 0.6105381721612377, - "mean_conf": 0.37532148884547056, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.1, - "coverage_at_1pct_error": 0.1, - "aurc": 0.41103174603174597, - "error_rate_at_0_9": null, - "confidence_bias": -0.12467851115452944, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.6, - "confidence_cutoff": 0.3749748282127031 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5, - "confidence_cutoff": 0.37206384539604187 - } - } - }, - "coverage": 1.0 - }, - "unknowable_control_volume_discount": { - "requested": 10, - "valid_probability_rows": 10, - "correct": 3, - "accuracy": 0.3, - "full_probability_metrics": { - "n": 10, - "nll": 1.076472913470353, - "acc": 0.3, - "ece": 0.14305523858076016, - "brier": 0.6513871176823558, - "mean_conf": 0.3939050057518472, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.1, - "coverage_at_1pct_error": 0.1, - "aurc": 0.5163095238095238, - "error_rate_at_0_9": null, - "confidence_bias": 0.09390500575184724, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.4, - "confidence_cutoff": 0.3807198518729166 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.375, - "confidence_cutoff": 0.35275470159188804 - } - }, - "score_mae": 0.8019682511451943, - "ranked_probability_score": 0.2424029763114047 - }, - "covered_only_probability_metrics": { - "n": 10, - "nll": 1.076472913470353, - "acc": 0.3, - "ece": 0.14305523858076016, - "brier": 0.6513871176823558, - "mean_conf": 0.3939050057518472, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.1, - "coverage_at_1pct_error": 0.1, - "aurc": 0.5163095238095238, - "error_rate_at_0_9": null, - "confidence_bias": 0.09390500575184724, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.4, - "confidence_cutoff": 0.3807198518729166 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.375, - "confidence_cutoff": 0.35275470159188804 - } - }, - "score_mae": 0.8019682511451943, - "ranked_probability_score": 0.2424029763114047 - }, - "coverage": 1.0 - }, - "unknowable_control_warranty_claim": { - "requested": 10, - "valid_probability_rows": 10, - "correct": 1, - "accuracy": 0.1, - "full_probability_metrics": { - "n": 10, - "nll": 1.1721106878792473, - "acc": 0.1, - "ece": 0.2943556323195833, - "brier": 0.713756632345296, - "mean_conf": 0.39435563231958326, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.1, - "coverage_at_1pct_error": 0.1, - "aurc": 0.7071031746031746, - "error_rate_at_0_9": null, - "confidence_bias": 0.2943556323195833, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.2, - "confidence_cutoff": 0.40616671461363774 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.125, - "confidence_cutoff": 0.3655887842178345 - } - }, - "score_mae": 0.5299003393040385, - "ranked_probability_score": 0.20619150421749982 - }, - "covered_only_probability_metrics": { - "n": 10, - "nll": 1.1721106878792473, - "acc": 0.1, - "ece": 0.2943556323195833, - "brier": 0.713756632345296, - "mean_conf": 0.39435563231958326, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.1, - "coverage_at_1pct_error": 0.1, - "aurc": 0.7071031746031746, - "error_rate_at_0_9": null, - "confidence_bias": 0.2943556323195833, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.2, - "confidence_cutoff": 0.40616671461363774 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.125, - "confidence_cutoff": 0.3655887842178345 - } - }, - "score_mae": 0.5299003393040385, - "ranked_probability_score": 0.20619150421749982 - }, - "coverage": 1.0 - } - }, - "clean_task_macro_accuracy": 0.4710648148148149, - "permutation": { - "requested_pairs": 36, - "valid_pairs": 36, - "both_correct_requested": 0.4166666666666667, - "flip_rate": 0.2777777777777778, - "covered_only_flip_rate": 0.2777777777777778, - "half_l1": 0.11699831081224105 - }, - "unknowable": { - "requested": 110, - "valid": 110, - "accuracy": null, - "full_confidence_metrics": { - "mean_max_p": 0.5539235213303535, - "share_at_0_9": 0.18181818181818182, - "normalized_entropy": 0.8261541457439553 - }, - "covered_only_confidence_metrics": { - "mean_max_p": 0.5539235213303535, - "share_at_0_9": 0.18181818181818182, - "normalized_entropy": 0.8261541457439553 - }, - "controls": { - "requested": 110, - "valid_probability_rows": 110, - "correct": 41, - "accuracy": 0.37272727272727274, - "full_probability_metrics": { - "n": 110, - "nll": 1.1848538382364027, - "acc": 0.37272727272727274, - "ece": 0.18048163285563096, - "brier": 0.7270760095187511, - "mean_conf": 0.5532089055829036, - "confident_error_rate": 0.09090909090909091, - "coverage_at_0_9": 0.18181818181818182, - "accuracy_at_0_9": 0.5, - "coverage_at_5pct_error": 0.00909090909090909, - "coverage_at_1pct_error": 0.00909090909090909, - "aurc": 0.5521530805542693, - "error_rate_at_0_9": 0.5, - "confidence_bias": 0.18048163285563085, - "top_bins": { - "0.9": { - "n": 20, - "errors": 10, - "error_rate": 0.5 - }, - "0.95": { - "n": 13, - "errors": 6, - "error_rate": 0.46153846153846156 - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.43636363636363634, - "confidence_cutoff": 0.428019808887845 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.375, - "confidence_cutoff": 0.37472726031325043 - } - }, - "score_mae": 0.7129783666931744, - "ranked_probability_score": 0.23673937533748973 - }, - "covered_only_probability_metrics": { - "n": 110, - "nll": 1.1848538382364027, - "acc": 0.37272727272727274, - "ece": 0.18048163285563096, - "brier": 0.7270760095187511, - "mean_conf": 0.5532089055829036, - "confident_error_rate": 0.09090909090909091, - "coverage_at_0_9": 0.18181818181818182, - "accuracy_at_0_9": 0.5, - "coverage_at_5pct_error": 0.00909090909090909, - "coverage_at_1pct_error": 0.00909090909090909, - "aurc": 0.5521530805542693, - "error_rate_at_0_9": 0.5, - "confidence_bias": 0.18048163285563085, - "top_bins": { - "0.9": { - "n": 20, - "errors": 10, - "error_rate": 0.5 - }, - "0.95": { - "n": 13, - "errors": 6, - "error_rate": 0.46153846153846156 - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.43636363636363634, - "confidence_cutoff": 0.428019808887845 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.375, - "confidence_cutoff": 0.37472726031325043 - } - }, - "score_mae": 0.7129783666931744, - "ranked_probability_score": 0.23673937533748973 - }, - "coverage": 1.0 - }, - "requested_pairs": 110, - "valid_pairs": 110, - "paired_confidence_drop": -0.0007146157474498129 - }, - "coverage": { - "questions": 1264, - "valid": 1264, - "failures_by_status": {}, - "truncated_questions": 0 - }, - "complete_unchanged_upstream_report": { - "objective": -1.053728625879992, - "paired_flip": { - "pairs": 64, - "flip_rate": 0.046875, - "both_correct_rate": 0.03125, - "invariant_pairs": 24, - "invariance_rate": 0.9583333333333334, - "invariant_both_correct_rate": 0.4583333333333333 - }, - "unknowable": { - "n": 110, - "mean_max_p": 0.5539235213303535, - "share_at_0_9": 0.18181818181818182, - "control_mean_max_p": 0.5532089055829036, - "control_share_at_0_9": 0.18181818181818182, - "control_acc": 0.37272727272727274, - "paired_confidence_drop": -0.0007146157474498129, - "share_less_confident_than_control": 0.32727272727272727 - }, - "clean": { - "n": 1046, - "nll": 1.155212302090266, - "acc": 0.4847036328871893, - "ece": 0.07953583853933748, - "brier": 0.6066466271423657, - "mean_conf": 0.5582630746492016, - "confident_error_rate": 0.03919694072657744, - "coverage_at_0_9": 0.16061185468451242, - "accuracy_at_0_9": 0.7559523809523809, - "coverage_at_5pct_error": 0.0124282982791587, - "coverage_at_1pct_error": 0.0124282982791587, - "aurc": 0.33433775549030087, - "error_rate_at_0_9": 0.24404761904761904, - "confidence_bias": 0.07355944176201223, - "top_bins": { - "0.9": { - "n": 168, - "errors": 41, - "error_rate": 0.24404761904761904 - }, - "0.95": { - "n": 81, - "errors": 10, - "error_rate": 0.12345679012345678 - }, - "0.99": { - "n": 2, - "errors": 0, - "error_rate": 0.0 - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.6424474187380497, - "confidence_cutoff": 0.5322548449039459 - }, - "0.8": { - "coverage": 0.8001912045889101, - "accuracy": 0.5495818399044206, - "confidence_cutoff": 0.3397521002646066 - } - }, - "score_mae": 0.7276275937425761, - "ranked_probability_score": 0.23169048615956875 - }, - "tasks": { - "buried_emotion": { - "n": 20, - "nll": 1.6434339926856585, - "acc": 0.55, - "ece": 0.20963160665409572, - "brier": 0.7352122696796819, - "mean_conf": 0.4118672036980742, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.5926301063917163, - "error_rate_at_0_9": null, - "confidence_bias": -0.13813279630192582, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.4, - "confidence_cutoff": 0.45318632980297263 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5, - "confidence_cutoff": 0.32968246936798096 - } - } - }, - "buried_paws": { - "n": 20, - "nll": 0.7827316896839417, - "acc": 0.35, - "ece": 0.24277215003967284, - "brier": 0.5858853695879119, - "mean_conf": 0.5927721500396729, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.6953620351956266, - "error_rate_at_0_9": null, - "confidence_bias": 0.2427721500396729, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.4, - "confidence_cutoff": 0.5672407746315002 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.375, - "confidence_cutoff": 0.5477952659130096 - } - } - }, - "buried_qnli": { - "n": 20, - "nll": 0.8555022618540915, - "acc": 0.55, - "ece": 0.2654646812006831, - "brier": 0.6094519995364962, - "mean_conf": 0.7867073891684413, - "confident_error_rate": 0.05, - "coverage_at_0_9": 0.25, - "accuracy_at_0_9": 0.8, - "coverage_at_5pct_error": 0.1, - "coverage_at_1pct_error": 0.1, - "aurc": 0.34032932603172544, - "error_rate_at_0_9": 0.2, - "confidence_bias": 0.23670738916844125, - "top_bins": { - "0.9": { - "n": 5, - "errors": 1, - "error_rate": 0.2 - }, - "0.95": { - "n": 2, - "errors": 0, - "error_rate": 0.0 - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.6, - "confidence_cutoff": 0.8149255514144897 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5625, - "confidence_cutoff": 0.6623975336551666 - } - } - }, - "buried_tweet_offensive": { - "n": 20, - "nll": 0.5035467531682788, - "acc": 0.75, - "ece": 0.19359442964196205, - "brier": 0.32708305202234894, - "mean_conf": 0.6864339232444763, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.05, - "accuracy_at_0_9": 1.0, - "coverage_at_5pct_error": 0.2, - "coverage_at_1pct_error": 0.2, - "aurc": 0.10817796152935473, - "error_rate_at_0_9": 0.0, - "confidence_bias": -0.06356607675552373, - "top_bins": { - "0.9": { - "n": 1, - "errors": 0, - "error_rate": 0.0 - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.9, - "confidence_cutoff": 0.6365915834903717 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.875, - "confidence_cutoff": 0.5576783418655396 - } - } - }, - "composition_final_combination": { - "n": 32, - "nll": 0.7235207463951194, - "acc": 0.34375, - "ece": 0.19085747487379479, - "brier": 0.5302537703699604, - "mean_conf": 0.5346074748737948, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.03125, - "coverage_at_1pct_error": 0.03125, - "aurc": 0.6815349342961152, - "error_rate_at_0_9": null, - "confidence_bias": 0.19085747487379479, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.25, - "confidence_cutoff": 0.5424632430076599 - }, - "0.8": { - "coverage": 0.8125, - "accuracy": 0.34615384615384615, - "confidence_cutoff": 0.50784832239151 - } - } - }, - "composition_final_exception": { - "n": 32, - "nll": 0.6887212067364372, - "acc": 0.625, - "ece": 0.10813984954395073, - "brier": 0.4955799333552681, - "mean_conf": 0.5168601504560493, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.03125, - "coverage_at_1pct_error": 0.03125, - "aurc": 0.3965946156627872, - "error_rate_at_0_9": null, - "confidence_bias": -0.10813984954395073, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.5, - "confidence_cutoff": 0.5141306672731241 - }, - "0.8": { - "coverage": 0.8125, - "accuracy": 0.5769230769230769, - "confidence_cutoff": 0.502220630645752 - } - } - }, - "composition_final_negation": { - "n": 32, - "nll": 0.6923211972690404, - "acc": 0.5625, - "ece": 0.07358545691148824, - "brier": 0.4989534073073557, - "mean_conf": 0.5570303887138711, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.03125, - "coverage_at_1pct_error": 0.03125, - "aurc": 0.45051881344545247, - "error_rate_at_0_9": null, - "confidence_bias": -0.005469611286128884, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.5625, - "confidence_cutoff": 0.5467859776743663 - }, - "0.8": { - "coverage": 0.8125, - "accuracy": 0.5384615384615384, - "confidence_cutoff": 0.5280904455131027 - } - } - }, - "contrastive_authorization": { - "n": 40, - "nll": 1.4121071983456814, - "acc": 0.5, - "ece": 0.4394341051578522, - "brier": 0.879494291145727, - "mean_conf": 0.9394341051578522, - "confident_error_rate": 0.5, - "coverage_at_0_9": 1.0, - "accuracy_at_0_9": 0.5, - "coverage_at_5pct_error": 0.125, - "coverage_at_1pct_error": 0.125, - "aurc": 0.3325839665024844, - "error_rate_at_0_9": 0.5, - "confidence_bias": 0.4394341051578522, - "top_bins": { - "0.9": { - "n": 40, - "errors": 20, - "error_rate": 0.5 - }, - "0.95": { - "n": 7, - "errors": 1, - "error_rate": 0.14285714285714285 - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.55, - "confidence_cutoff": 0.9448155164718628 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.53125, - "confidence_cutoff": 0.930177628993988 - } - } - }, - "contrastive_deadline": { - "n": 40, - "nll": 1.0561322346421498, - "acc": 0.475, - "ece": 0.14689049363580786, - "brier": 0.6378429454852624, - "mean_conf": 0.4090969440305655, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.5036059354521869, - "error_rate_at_0_9": null, - "confidence_bias": -0.06590305596943447, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.6, - "confidence_cutoff": 0.4019584655761719 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5, - "confidence_cutoff": 0.37009772658348083 - } - }, - "score_mae": 0.7496014343166787, - "ranked_probability_score": 0.2241171523926872 - }, - "emotion": { - "n": 80, - "nll": 1.2463184110640215, - "acc": 0.525, - "ece": 0.06492680838097321, - "brier": 0.6039373195122416, - "mean_conf": 0.4982290427346596, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0125, - "accuracy_at_0_9": 1.0, - "coverage_at_5pct_error": 0.0125, - "coverage_at_1pct_error": 0.0125, - "aurc": 0.32277654532687594, - "error_rate_at_0_9": 0.0, - "confidence_bias": -0.0267709572653404, - "top_bins": { - "0.9": { - "n": 1, - "errors": 0, - "error_rate": 0.0 - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.7, - "confidence_cutoff": 0.45944378192850677 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5625, - "confidence_cutoff": 0.357612487989571 - } - } - }, - "mmlu": { - "n": 80, - "nll": 1.4095109007441815, - "acc": 0.3125, - "ece": 0.12076733954335728, - "brier": 0.7561817190247881, - "mean_conf": 0.4170573963746701, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.6634931331685283, - "error_rate_at_0_9": null, - "confidence_bias": 0.1045573963746701, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.375, - "confidence_cutoff": 0.38152980235919653 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.3125, - "confidence_cutoff": 0.3282719703791155 - } - } - }, - "mmlu_pro": { - "n": 200, - "nll": 2.2624765159160414, - "acc": 0.175, - "ece": 0.08681084104213858, - "brier": 0.9071117655594688, - "mean_conf": 0.23055428589672733, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.863247364071338, - "error_rate_at_0_9": null, - "confidence_bias": 0.05555428589672734, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.17, - "confidence_cutoff": 0.19904440930882775 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.175, - "confidence_cutoff": 0.15484475801056702 - } - } - }, - "paws": { - "n": 80, - "nll": 0.7090254071044348, - "acc": 0.475, - "ece": 0.1666798379272222, - "brier": 0.5166084866955849, - "mean_conf": 0.6398283705115319, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0125, - "accuracy_at_0_9": 1.0, - "coverage_at_5pct_error": 0.05, - "coverage_at_1pct_error": 0.05, - "aurc": 0.3926720126883591, - "error_rate_at_0_9": 0.0, - "confidence_bias": 0.16482837051153187, - "top_bins": { - "0.9": { - "n": 1, - "errors": 0, - "error_rate": 0.0 - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.625, - "confidence_cutoff": 0.6297575533390045 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.515625, - "confidence_cutoff": 0.5585934221744537 - } - } - }, - "qnli": { - "n": 80, - "nll": 0.7029168919881199, - "acc": 0.7125, - "ece": 0.1819537915289402, - "brier": 0.44881665172376445, - "mean_conf": 0.8604280255734921, - "confident_error_rate": 0.1125, - "coverage_at_0_9": 0.45, - "accuracy_at_0_9": 0.75, - "coverage_at_5pct_error": 0.05, - "coverage_at_1pct_error": 0.05, - "aurc": 0.22011525782827887, - "error_rate_at_0_9": 0.25, - "confidence_bias": 0.14792802557349205, - "top_bins": { - "0.9": { - "n": 36, - "errors": 9, - "error_rate": 0.25 - }, - "0.95": { - "n": 22, - "errors": 3, - "error_rate": 0.13636363636363635 - }, - "0.99": { - "n": 1, - "errors": 0, - "error_rate": 0.0 - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.75, - "confidence_cutoff": 0.8962836265563965 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.734375, - "confidence_cutoff": 0.7553151249885559 - } - } - }, - "sciq": { - "n": 80, - "nll": 0.3412942378197378, - "acc": 0.925, - "ece": 0.17180305325181675, - "brier": 0.14992498517720326, - "mean_conf": 0.7651692762303128, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.2875, - "accuracy_at_0_9": 1.0, - "coverage_at_5pct_error": 0.95, - "coverage_at_1pct_error": 0.8, - "aurc": 0.006306048389760769, - "error_rate_at_0_9": 0.0, - "confidence_bias": -0.1598307237696872, - "top_bins": { - "0.9": { - "n": 23, - "errors": 0, - "error_rate": 0.0 - }, - "0.95": { - "n": 8, - "errors": 0, - "error_rate": 0.0 - }, - "0.99": { - "n": 1, - "errors": 0, - "error_rate": 0.0 - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 1.0, - "confidence_cutoff": 0.8314547840754842 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 1.0, - "confidence_cutoff": 0.599574194858055 - } - } - }, - "tweet_offensive": { - "n": 80, - "nll": 0.38772103274242486, - "acc": 0.7875, - "ece": 0.07751428142655643, - "brier": 0.25593492533075707, - "mean_conf": 0.8444197329459712, - "confident_error_rate": 0.0125, - "coverage_at_0_9": 0.5125, - "accuracy_at_0_9": 0.975609756097561, - "coverage_at_5pct_error": 0.525, - "coverage_at_1pct_error": 0.4875, - "aurc": 0.05995296937845444, - "error_rate_at_0_9": 0.024390243902439025, - "confidence_bias": 0.05691973294597119, - "top_bins": { - "0.9": { - "n": 41, - "errors": 1, - "error_rate": 0.024390243902439025 - }, - "0.95": { - "n": 29, - "errors": 0, - "error_rate": 0.0 - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.975, - "confidence_cutoff": 0.9004859104752541 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.875, - "confidence_cutoff": 0.7327178418636322 - } - } - }, - "unknowable_age_eligibility": { - "n": 10, - "nll": 1.642865228921649, - "acc": 0.5, - "ece": 0.46098687648773196, - "brier": 0.9250307537209096, - "mean_conf": 0.960986876487732, - "confident_error_rate": 0.5, - "coverage_at_0_9": 1.0, - "accuracy_at_0_9": 0.5, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.5, - "error_rate_at_0_9": 0.5, - "confidence_bias": 0.46098687648773196, - "top_bins": { - "0.9": { - "n": 10, - "errors": 5, - "error_rate": 0.5 - }, - "0.95": { - "n": 10, - "errors": 5, - "error_rate": 0.5 - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.6, - "accuracy": 0.5, - "confidence_cutoff": 0.9612478613853455 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5, - "confidence_cutoff": 0.9606207013130188 - } - } - }, - "unknowable_authorization": { - "n": 10, - "nll": 1.5255140954789224, - "acc": 0.5, - "ece": 0.4480596125125885, - "brier": 0.9047932053537331, - "mean_conf": 0.9480596125125885, - "confident_error_rate": 0.5, - "coverage_at_0_9": 1.0, - "accuracy_at_0_9": 0.5, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.5818650793650793, - "error_rate_at_0_9": 0.5, - "confidence_bias": 0.4480596125125885, - "top_bins": { - "0.9": { - "n": 10, - "errors": 5, - "error_rate": 0.5 - }, - "0.95": { - "n": 4, - "errors": 2, - "error_rate": 0.5 - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.6, - "confidence_cutoff": 0.9487297534942627 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.375, - "confidence_cutoff": 0.9429174065589905 - } - } - }, - "unknowable_control_age_eligibility": { - "n": 10, - "nll": 1.6191405658106865, - "acc": 0.5, - "ece": 0.45958920121192937, - "brier": 0.9210477728721707, - "mean_conf": 0.9595892012119294, - "confident_error_rate": 0.5, - "coverage_at_0_9": 1.0, - "accuracy_at_0_9": 0.5, - "coverage_at_5pct_error": 0.1, - "coverage_at_1pct_error": 0.1, - "aurc": 0.4106349206349207, - "error_rate_at_0_9": 0.5, - "confidence_bias": 0.45958920121192937, - "top_bins": { - "0.9": { - "n": 10, - "errors": 5, - "error_rate": 0.5 - }, - "0.95": { - "n": 10, - "errors": 5, - "error_rate": 0.5 - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.6, - "confidence_cutoff": 0.9605617523193359 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5, - "confidence_cutoff": 0.9569776654243469 - } - } - }, - "unknowable_control_authorization": { - "n": 10, - "nll": 1.4433423409086612, - "acc": 0.5, - "ece": 0.4431973397731781, - "brier": 0.8877473385116084, - "mean_conf": 0.9431973397731781, - "confident_error_rate": 0.5, - "coverage_at_0_9": 1.0, - "accuracy_at_0_9": 0.5, - "coverage_at_5pct_error": 0.2, - "coverage_at_1pct_error": 0.2, - "aurc": 0.31007936507936507, - "error_rate_at_0_9": 0.5, - "confidence_bias": 0.4431973397731781, - "top_bins": { - "0.9": { - "n": 10, - "errors": 5, - "error_rate": 0.5 - }, - "0.95": { - "n": 3, - "errors": 1, - "error_rate": 0.3333333333333333 - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.8, - "confidence_cutoff": 0.9448668360710144 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5, - "confidence_cutoff": 0.9334832429885864 - } - } - }, - "unknowable_control_deadline": { - "n": 10, - "nll": 1.0534611380665015, - "acc": 0.5, - "ece": 0.10049157245677873, - "brier": 0.6380457356055647, - "mean_conf": 0.39950842754322125, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.49992063492063493, - "error_rate_at_0_9": null, - "confidence_bias": -0.10049157245677875, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.6, - "confidence_cutoff": 0.40330082178115845 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5, - "confidence_cutoff": 0.3643658350597073 - } - }, - "score_mae": 0.6959988628582565, - "ranked_probability_score": 0.2123149634956698 - }, - "unknowable_control_late_fee": { - "n": 10, - "nll": 1.2971820726980325, - "acc": 0.3, - "ece": 0.317867469435447, - "brier": 0.8158706675338253, - "mean_conf": 0.6178674694354471, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.7722619047619048, - "error_rate_at_0_9": null, - "confidence_bias": 0.31786746943544714, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.4, - "confidence_cutoff": 0.6193228548563378 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.375, - "confidence_cutoff": 0.6018166721408568 - } - }, - "score_mae": 0.7810463516446486, - "ranked_probability_score": 0.2694903445971064 - }, - "unknowable_control_quantity_limit": { - "n": 10, - "nll": 1.0767010069332539, - "acc": 0.5, - "ece": 0.12858793102984267, - "brier": 0.6522324327497165, - "mean_conf": 0.3714120689701573, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.2, - "coverage_at_1pct_error": 0.2, - "aurc": 0.37492063492063493, - "error_rate_at_0_9": null, - "confidence_bias": -0.12858793102984267, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.6, - "confidence_cutoff": 0.3743864484314189 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5, - "confidence_cutoff": 0.37018098323997906 - } - } - }, - "unknowable_control_return_window": { - "n": 10, - "nll": 0.8455590159265375, - "acc": 0.5, - "ece": 0.25936471819877627, - "brier": 0.6308352614839883, - "mean_conf": 0.7593647181987763, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.5249206349206349, - "error_rate_at_0_9": null, - "confidence_bias": 0.25936471819877627, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.6, - "confidence_cutoff": 0.7657169103622437 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5, - "confidence_cutoff": 0.7524310350418091 - } - } - }, - "unknowable_control_shipping_delay": { - "n": 10, - "nll": 1.3095807829358257, - "acc": 0.2, - "ece": 0.22123073572301039, - "brier": 0.7834211221927093, - "mean_conf": 0.4212307357230104, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.7475396825396825, - "error_rate_at_0_9": null, - "confidence_bias": 0.22123073572301039, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.4, - "confidence_cutoff": 0.42378672325928923 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.25, - "confidence_cutoff": 0.41109954698235474 - } - }, - "score_mae": 0.8366539786236817, - "ranked_probability_score": 0.27876383225163887 - }, - "unknowable_control_sla_response": { - "n": 10, - "nll": 1.129793073433971, - "acc": 0.2, - "ece": 0.3474792748375072, - "brier": 0.6929538515677903, - "mean_conf": 0.44954587363931975, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.1, - "coverage_at_1pct_error": 0.1, - "aurc": 0.6225396825396825, - "error_rate_at_0_9": null, - "confidence_bias": 0.24954587363931974, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.4, - "confidence_cutoff": 0.4481564857889793 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.25, - "confidence_cutoff": 0.42062819630843695 - } - }, - "score_mae": 0.6323024165832265, - "ranked_probability_score": 0.2112726311516188 - }, - "unknowable_control_spend_threshold": { - "n": 10, - "nll": 1.0100486225373588, - "acc": 0.5, - "ece": 0.12467851115452944, - "brier": 0.6105381721612377, - "mean_conf": 0.37532148884547056, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.1, - "coverage_at_1pct_error": 0.1, - "aurc": 0.41103174603174597, - "error_rate_at_0_9": null, - "confidence_bias": -0.12467851115452944, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.6, - "confidence_cutoff": 0.3749748282127031 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5, - "confidence_cutoff": 0.37206384539604187 - } - } - }, - "unknowable_control_volume_discount": { - "n": 10, - "nll": 1.076472913470353, - "acc": 0.3, - "ece": 0.14305523858076016, - "brier": 0.6513871176823558, - "mean_conf": 0.3939050057518472, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.1, - "coverage_at_1pct_error": 0.1, - "aurc": 0.5163095238095238, - "error_rate_at_0_9": null, - "confidence_bias": 0.09390500575184724, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.4, - "confidence_cutoff": 0.3807198518729166 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.375, - "confidence_cutoff": 0.35275470159188804 - } - }, - "score_mae": 0.8019682511451943, - "ranked_probability_score": 0.2424029763114047 - }, - "unknowable_control_warranty_claim": { - "n": 10, - "nll": 1.1721106878792473, - "acc": 0.1, - "ece": 0.2943556323195833, - "brier": 0.713756632345296, - "mean_conf": 0.39435563231958326, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.1, - "coverage_at_1pct_error": 0.1, - "aurc": 0.7071031746031746, - "error_rate_at_0_9": null, - "confidence_bias": 0.2943556323195833, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.2, - "confidence_cutoff": 0.40616671461363774 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.125, - "confidence_cutoff": 0.3655887842178345 - } - }, - "score_mae": 0.5299003393040385, - "ranked_probability_score": 0.20619150421749982 - }, - "unknowable_deadline": { - "n": 10, - "nll": 1.0325794464267106, - "acc": 0.5, - "ece": 0.07300670152576706, - "brier": 0.622048611190791, - "mean_conf": 0.42699329847423295, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.1, - "coverage_at_1pct_error": 0.1, - "aurc": 0.39674603174603174, - "error_rate_at_0_9": null, - "confidence_bias": -0.07300670152576705, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.6, - "confidence_cutoff": 0.43429750595703687 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5, - "confidence_cutoff": 0.40986567735671997 - } - }, - "score_mae": 0.692927005134265, - "ranked_probability_score": 0.20633211944843896 - }, - "unknowable_late_fee": { - "n": 10, - "nll": 1.3449419049439826, - "acc": 0.3, - "ece": 0.3365630169287717, - "brier": 0.8398352499672098, - "mean_conf": 0.6365630169287717, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.8566666666666667, - "error_rate_at_0_9": null, - "confidence_bias": 0.3365630169287717, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.6, - "accuracy": 0.16666666666666666, - "confidence_cutoff": 0.6396053528726898 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.25, - "confidence_cutoff": 0.629168409978402 - } - }, - "score_mae": 0.7993052691271727, - "ranked_probability_score": 0.28159488856612563 - }, - "unknowable_quantity_limit": { - "n": 10, - "nll": 1.1049937312161007, - "acc": 0.2, - "ece": 0.17462785826885252, - "brier": 0.671576058181679, - "mean_conf": 0.37462785826885253, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.6933333333333334, - "error_rate_at_0_9": null, - "confidence_bias": 0.17462785826885252, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.6, - "accuracy": 0.3333333333333333, - "confidence_cutoff": 0.3779213315867981 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.25, - "confidence_cutoff": 0.37251858793143666 - } - } - }, - "unknowable_return_window": { - "n": 10, - "nll": 0.712707396335963, - "acc": 0.5, - "ece": 0.10319202542304992, - "brier": 0.5191555614314189, - "mean_conf": 0.60319202542305, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.1, - "coverage_at_1pct_error": 0.1, - "aurc": 0.451468253968254, - "error_rate_at_0_9": null, - "confidence_bias": 0.10319202542304995, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.4, - "confidence_cutoff": 0.5948264002799988 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.625, - "confidence_cutoff": 0.5866559147834778 - } - } - }, - "unknowable_shipping_delay": { - "n": 10, - "nll": 1.2837946482197433, - "acc": 0.2, - "ece": 0.23561829186722166, - "brier": 0.7779788096765419, - "mean_conf": 0.4356182918672217, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.9184920634920637, - "error_rate_at_0_9": null, - "confidence_bias": 0.23561829186722166, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.0, - "confidence_cutoff": 0.4452194207128969 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.25, - "confidence_cutoff": 0.41273740548994253 - } - }, - "score_mae": 0.8503499623614804, - "ranked_probability_score": 0.28238443704437793 - }, - "unknowable_sla_response": { - "n": 10, - "nll": 1.241322797122184, - "acc": 0.2, - "ece": 0.2985316400098889, - "brier": 0.7601905769326773, - "mean_conf": 0.49853164000988903, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.8516666666666668, - "error_rate_at_0_9": null, - "confidence_bias": 0.298531640009889, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.6, - "accuracy": 0.16666666666666666, - "confidence_cutoff": 0.505884222598609 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.125, - "confidence_cutoff": 0.49391087889671326 - } - }, - "score_mae": 0.717774508587917, - "ranked_probability_score": 0.24466640720619778 - }, - "unknowable_spend_threshold": { - "n": 10, - "nll": 0.9640492848702706, - "acc": 0.5, - "ece": 0.09429512128526485, - "brier": 0.5846642683056338, - "mean_conf": 0.40570487871473515, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.5, - "error_rate_at_0_9": null, - "confidence_bias": -0.09429512128526485, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.6, - "accuracy": 0.5, - "confidence_cutoff": 0.4120677052130147 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.5, - "confidence_cutoff": 0.40694461485578465 - } - } - }, - "unknowable_volume_discount": { - "n": 10, - "nll": 1.104836790437616, - "acc": 0.3, - "ece": 0.14561180484401773, - "brier": 0.6697748792572266, - "mean_conf": 0.4129858960689459, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.5650000000000001, - "error_rate_at_0_9": null, - "confidence_bias": 0.11298589606894593, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.6, - "accuracy": 0.5, - "confidence_cutoff": 0.3991083621713031 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.375, - "confidence_cutoff": 0.38743923269031144 - } - }, - "score_mae": 0.8158904616800088, - "ranked_probability_score": 0.24781331072505153 - }, - "unknowable_warranty_claim": { - "n": 10, - "nll": 1.2161216725090753, - "acc": 0.1, - "ece": 0.2898953398778692, - "brier": 0.7405138438432338, - "mean_conf": 0.3898953398778692, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.0, - "coverage_at_1pct_error": 0.0, - "aurc": 0.966388888888889, - "error_rate_at_0_9": null, - "confidence_bias": 0.2898953398778692, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.0, - "confidence_cutoff": 0.380980390877116 - }, - "0.8": { - "coverage": 0.8, - "accuracy": 0.125, - "confidence_cutoff": 0.37668469548225403 - } - }, - "score_mae": 0.527737131741856, - "ranked_probability_score": 0.21150302511451152 - } - }, - "variants": { - "clean": { - "n": 1156, - "nll": 1.1592468321377507, - "acc": 0.47145328719723184, - "ece": 0.09180456268035113, - "brier": 0.6182595070670632, - "mean_conf": 0.5578501413749166, - "confident_error_rate": 0.04411764705882353, - "coverage_at_0_9": 0.16262975778546712, - "accuracy_at_0_9": 0.7287234042553191, - "coverage_at_5pct_error": 0.01124567474048443, - "coverage_at_1pct_error": 0.01124567474048443, - "aurc": 0.35554461074697097, - "error_rate_at_0_9": 0.2712765957446808, - "confidence_bias": 0.08639685417768478, - "top_bins": { - "0.9": { - "n": 188, - "errors": 51, - "error_rate": 0.2712765957446808 - }, - "0.95": { - "n": 95, - "errors": 17, - "error_rate": 0.17894736842105263 - }, - "0.99": { - "n": 2, - "errors": 0, - "error_rate": 0.0 - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.6245674740484429, - "confidence_cutoff": 0.5256392516386286 - }, - "0.8": { - "coverage": 0.8001730103806228, - "accuracy": 0.532972972972973, - "confidence_cutoff": 0.35297433593792726 - } - }, - "score_mae": 0.7300162672536539, - "ranked_probability_score": 0.23694994060627442 - }, - "none_absent": { - "n": 36, - "nll": 1.0379818411185247, - "acc": 0.5833333333333334, - "ece": 0.17411443589453532, - "brier": 0.5552370302734961, - "mean_conf": 0.4561156804965632, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.1111111111111111, - "coverage_at_1pct_error": 0.1111111111111111, - "aurc": 0.2983798768194237, - "error_rate_at_0_9": null, - "confidence_bias": -0.12721765283677017, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.6666666666666666, - "confidence_cutoff": 0.4208044111728668 - }, - "0.8": { - "coverage": 0.8055555555555556, - "accuracy": 0.6551724137931034, - "confidence_cutoff": 0.3314213119660916 - } - } - }, - "none_present": { - "n": 36, - "nll": 1.4843819230502198, - "acc": 0.4722222222222222, - "ece": 0.15080881481784217, - "brier": 0.7114589140504244, - "mean_conf": 0.46934038448752713, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.0, - "accuracy_at_0_9": null, - "coverage_at_5pct_error": 0.08333333333333333, - "coverage_at_1pct_error": 0.08333333333333333, - "aurc": 0.3908382820878864, - "error_rate_at_0_9": null, - "confidence_bias": -0.0028818377346950808, - "top_bins": { - "0.9": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.95": { - "n": 0, - "errors": 0, - "error_rate": null - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.5, - "confidence_cutoff": 0.4568726156418965 - }, - "0.8": { - "coverage": 0.8055555555555556, - "accuracy": 0.4827586206896552, - "confidence_cutoff": 0.31162673467119584 - } - } - }, - "permuted": { - "n": 36, - "nll": 1.185435349866166, - "acc": 0.4722222222222222, - "ece": 0.1863391325262919, - "brier": 0.6137128264232319, - "mean_conf": 0.5286949008634976, - "confident_error_rate": 0.0, - "coverage_at_0_9": 0.05555555555555555, - "accuracy_at_0_9": 1.0, - "coverage_at_5pct_error": 0.19444444444444445, - "coverage_at_1pct_error": 0.19444444444444445, - "aurc": 0.3368570334095183, - "error_rate_at_0_9": 0.0, - "confidence_bias": 0.056472678641275365, - "top_bins": { - "0.9": { - "n": 2, - "errors": 0, - "error_rate": 0.0 - }, - "0.95": { - "n": 1, - "errors": 0, - "error_rate": 0.0 - }, - "0.99": { - "n": 0, - "errors": 0, - "error_rate": null - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.5555555555555556, - "confidence_cutoff": 0.42931715613567434 - }, - "0.8": { - "coverage": 0.8055555555555556, - "accuracy": 0.4827586206896552, - "confidence_cutoff": 0.3680201796294762 - } - } - } - }, - "heldout_tasks": {}, - "permutation": { - "n": 36, - "mean_max_delta": 0.10131048171841536, - "flip_rate": 0.2777777777777778 - }, - "temperature": 1.0, - "calibrated_clean": { - "n": 1046, - "nll": 1.155212302090266, - "acc": 0.4847036328871893, - "ece": 0.07953583853933748, - "brier": 0.6066466271423657, - "mean_conf": 0.5582630746492016, - "confident_error_rate": 0.03919694072657744, - "coverage_at_0_9": 0.16061185468451242, - "accuracy_at_0_9": 0.7559523809523809, - "coverage_at_5pct_error": 0.0124282982791587, - "coverage_at_1pct_error": 0.0124282982791587, - "aurc": 0.33433775549030087, - "error_rate_at_0_9": 0.24404761904761904, - "confidence_bias": 0.07355944176201223, - "top_bins": { - "0.9": { - "n": 168, - "errors": 41, - "error_rate": 0.24404761904761904 - }, - "0.95": { - "n": 81, - "errors": 10, - "error_rate": 0.12345679012345678 - }, - "0.99": { - "n": 2, - "errors": 0, - "error_rate": 0.0 - } - }, - "selective": { - "0.5": { - "coverage": 0.5, - "accuracy": 0.6424474187380497, - "confidence_cutoff": 0.5322548449039459 - }, - "0.8": { - "coverage": 0.8001912045889101, - "accuracy": 0.5495818399044206, - "confidence_cutoff": 0.3397521002646066 - } - }, - "score_mae": 0.7276275937425761, - "ranked_probability_score": 0.23169048615956875 - }, - "metric_policy": { - "version": 2, - "selective_ties": "whole_confidence_groups", - "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", - "aurc": "right-step integral over whole confidence groups", - "confident_error_rate": "high-confidence errors divided by all questions", - "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", - "nll": "exact from logits when recorded; otherwise from floored probabilities", - "nll_floor": 1e-09, - "renormalize_returned_probabilities": true, - "raw_sums_outside_1e_5": 0, - "returned_zeros": 0 - } - }, - "metric_scope": "Refused/invalid predictions count incorrect in requested categorical denominator. Missing probabilities stay missing; covered-only metrics never presented as full metrics. Risk-coverage thresholds are retrospective diagnostics, not a deployed error guarantee." - } } }, "paired_comparisons": { @@ -72291,64 +67149,6 @@ ] } }, - "Nox minus Kai": { - "old_core": { - "delta": 0.4076264880952381, - "exact_fraction": "10957/26880", - "paired_ci95": [ - 0.36502883184523816, - 0.4506696428571429 - ] - }, - "v3_core": { - "delta": 0.11916666666666667, - "exact_fraction": "143/1200", - "paired_ci95": [ - 0.08541666666666659, - 0.1533333333333332 - ] - }, - "v4": { - "delta": 0.23125, - "exact_fraction": "37/160", - "paired_ci95": [ - 0.17582368129908946, - 0.28593749999999996 - ] - }, - "v5": { - "delta": 0.31666666666666665, - "exact_fraction": "19/60", - "paired_ci95": [ - 0.27375714718155764, - 0.3585540213739065 - ] - }, - "transfer_v9_test": { - "delta": 0.1950286806883365, - "exact_fraction": "102/523", - "paired_ci95": [ - 0.15697674418604657, - 0.2330899600169745 - ] - }, - "original_four_panel_mean": { - "delta": 0.2686774553571429, - "exact_fraction": "48147/179200", - "paired_ci95": [ - 0.2461794707501723, - 0.29045725483541707 - ] - }, - "weighted_mean": { - "delta": 0.2635214151984886, - "exact_fraction": "37046473/140582400", - "paired_ci95": [ - 0.24360157352685957, - 0.2826532318578912 - ] - } - }, "Nox minus Laya-base": { "old_core": { "delta": 0.26458333333333334, @@ -73045,64 +67845,6 @@ ] } }, - "Sol minus Kai": { - "old_core": { - "delta": 0.3150669642857143, - "exact_fraction": "2823/8960", - "paired_ci95": [ - 0.26815476190476184, - 0.36183035714285705 - ] - }, - "v3_core": { - "delta": 0.06208333333333333, - "exact_fraction": "149/2400", - "paired_ci95": [ - 0.03583333333333333, - 0.08916666666666662 - ] - }, - "v4": { - "delta": 0.20625, - "exact_fraction": "33/160", - "paired_ci95": [ - 0.1511514729299363, - 0.26026290729020163 - ] - }, - "v5": { - "delta": 0.29583333333333334, - "exact_fraction": "71/240", - "paired_ci95": [ - 0.25295853774664184, - 0.3384472789185137 - ] - }, - "transfer_v9_test": { - "delta": 0.08604206500956023, - "exact_fraction": "45/523", - "paired_ci95": [ - 0.04694465743842222, - 0.1264500411986247 - ] - }, - "original_four_panel_mean": { - "delta": 0.21980840773809524, - "exact_fraction": "118169/537600", - "paired_ci95": [ - 0.19777820434709284, - 0.2412620356797066 - ] - }, - "weighted_mean": { - "delta": 0.19825973237048167, - "exact_fraction": "27871829/140582400", - "paired_ci95": [ - 0.17842727319237753, - 0.2177328205637521 - ] - } - }, "Sol minus Laya-base": { "old_core": { "delta": 0.17202380952380952, @@ -73857,64 +68599,6 @@ ] } }, - "Lux minus Kai": { - "old_core": { - "delta": 0.40970982142857143, - "exact_fraction": "3671/8960", - "paired_ci95": [ - 0.36941778273809517, - 0.45044642857142847 - ] - }, - "v3_core": { - "delta": 0.12, - "exact_fraction": "3/25", - "paired_ci95": [ - 0.08249999999999996, - 0.15875 - ] - }, - "v4": { - "delta": 0.34375, - "exact_fraction": "11/32", - "paired_ci95": [ - 0.2890625000000001, - 0.39619820628668523 - ] - }, - "v5": { - "delta": 0.3625, - "exact_fraction": "29/80", - "paired_ci95": [ - 0.3223385649885553, - 0.40208333333333324 - ] - }, - "transfer_v9_test": { - "delta": 0.28967495219885275, - "exact_fraction": "303/1046", - "paired_ci95": [ - 0.2528620532616239, - 0.32664756446991405 - ] - }, - "original_four_panel_mean": { - "delta": 0.30898995535714285, - "exact_fraction": "55371/179200", - "paired_ci95": [ - 0.28719901968981626, - 0.3298872396154221 - ] - }, - "weighted_mean": { - "delta": 0.30230168925839934, - "exact_fraction": "14166099/46860800", - "paired_ci95": [ - 0.28317297093556093, - 0.32131241076703493 - ] - } - }, "Lux minus Laya-base": { "old_core": { "delta": 0.26666666666666666, @@ -74398,16 +69082,10 @@ 0.4566341812351861, 0.48745648347218246 ] - }, - "Kai": { - "point": 0.4680084349107712, - "exact_fraction": "9399107/20083200", - "ci95": [ - 0.45368578168634693, - 0.48256479238590355 - ] } }, "hosted_frontier": "All official calls returned jev-1.13.0; closed weight identity/input retention cannot be independently inspected.", - "size_note": "Qwen3.5 family size labels name their official parent checkpoint. Lux deployed text+decision weights total7,940,895,744 parameters; no vision or generativeLMhead in its serving bundle." + "size_note": "Qwen3.5 family size labels name their official parent checkpoint. Lux deployed text+decision weights total7,940,895,744 parameters; no vision or generativeLMhead in its serving bundle.", + "presentation_amendment_sha256": "f078ecb12e7f4f2baaab522d07af6695c18d9c4669a5f61623e85c93b910fd1f", + "presentation_note": "Public display roster changed only; frozen protocol, complete internal evidence and every retained numeric result are unchanged." }