diff --git "a/evaluation/full-data.json" "b/evaluation/full-data.json" new file mode 100644--- /dev/null +++ "b/evaluation/full-data.json" @@ -0,0 +1,5340 @@ +{ + "model": "Qwen/Qwen3.5-2B", + "revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", + "checkpoint": { + "model": "Qwen/Qwen3.5-2B", + "revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", + "checkpoint_sha256": "8c37b393c27d2b58009463a89dd1a873b7c010ecd1b8c184c7ada5efe13f8bfb", + "temperature": 1.518796342858676, + "calibration": { + "temperature": 1.518796342858676, + "split": "calibration", + "n": 512, + "ids_sha256": "6ff7db59c1ae2485179b80316b3cb4f0161df77704692dcab48871a295ab94f1" + }, + "max_length": 4096, + "training_completion": { + "steps": 20204, + "trained_rows_consumed": 80816, + "checkpoint_reload_max_error": 0.0, + "run_sha256": "8ee935b52c0ee1796a25d8d8b0fcfc8aa41eddee815c454016c956f204b1416a", + "summary_sha256": "ea61d4b1cc28c57a826c57986323abe4bbf715ff28985d7a6fa89fec2119c8bf" + } + }, + "package_bound_inference_files": { + "checkpoint/adapter/adapter_config.json": { + "sha256": "d27624bdbbe984cf175e10c7fa3d9beaea9876231ce943a248ad40e12742fe74", + "bytes": 1170 + }, + "checkpoint/adapter/adapter_model.safetensors": { + "sha256": "2d23935b1a7380db444abac572c04646918ba794e59002d1588236182a3ca18f", + "bytes": 10042480 + }, + "checkpoint/head.pt": { + "sha256": "3532cd576c58d5ad5bf17c3e9f2df4be8c70e08c07fa6bb7fa673dcd7b401f2a", + "bytes": 9997 + }, + "checkpoint/model.json": { + "sha256": "d985eafa4d635113ee8cc912e0512a95aac0bed3426e9256931235b75cb3d6c5", + "bytes": 186 + }, + "checkpoint/temperature.json": { + "sha256": "090fb330a616338210a924d1c370c934e934f30f53f92c8a52a5cc7852cadd99", + "bytes": 161 + } + }, + "data_identity": { + "manifest_sha256": "56105dc9fc89ef74919f5beb60bb6ae8c6e17bb95699dab59205f67d8b338d97", + "split_sha256": { + "train": "80a9ec19da9c4065b72f7a9e5ff8d67e62c1d1dd2572dfd296e46bfe2ea5a182", + "calibration": "dd21b26912a6b26a09c3b00ae66a7985ee760e737b94ada4fcc6320406d4db63", + "validation": "1dc97079d4fc034a1ba16315f801b0a4715b30c2a3874ae05aeb60dd1e40e8bb", + "test": "fa8d62775b96c1514f238c2d8a8f8b66515dada67d490978376545296593e29b", + "ood": "9152bff7f3d9c83cce3f1423cc6e0e6b33286b3092b3f544cb18dcc37b35a7ee" + }, + "counts": { + "test": 10532, + "ood": 15920 + }, + "ordered_ids_sha256": { + "test": "98e1af223471c08d84e44c8864e51cefa1bc671687c00bfbf64b7e5a22e8bf52", + "ood": "52fa6cc83c0a172516609257b6048aaece9a3fef75f3f5f694bc6213e5c30a42" + } + }, + "coverage": { + "expected_rows": 26452, + "unique_ids": 26452, + "shard_rows": [ + 6613, + 6613, + 6613, + 6613 + ], + "missing": 0, + "duplicate": 0, + "failed": 0, + "merged_original_order_verified": true + }, + "metrics": { + "test": { + "count": 10532, + "successful": 10532, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 10046, + "accuracy_all_rows_failures_incorrect": 0.9471431415488751, + "expected_accuracy_all_rows_failures_zero": 0.9195253563797051, + "coverage": [ + { + "threshold": 0.5, + "selected": 9830, + "total": 10532, + "coverage": 0.9333459931636916 + }, + { + "threshold": 0.7, + "selected": 9500, + "total": 10532, + "coverage": 0.9020129130269654 + }, + { + "threshold": 0.8, + "selected": 9388, + "total": 10532, + "coverage": 0.8913786555260159 + }, + { + "threshold": 0.9, + "selected": 9116, + "total": 10532, + "coverage": 0.8655526015951386 + }, + { + "threshold": 0.95, + "selected": 8634, + "total": 10532, + "coverage": 0.819787314849981 + }, + { + "threshold": 0.99, + "selected": 6742, + "total": 10532, + "coverage": 0.6401443220660843 + } + ], + "probability_metrics_all_rows": { + "count": 10532, + "hard_count": 10046, + "accuracy": 0.9471431415488751, + "expected_accuracy": 0.9195253563797051, + "nll": 0.19538003409263777, + "brier": 0.07611697556114595, + "expected_brier": 0.10416793987302937, + "ece_bins": 15, + "multiclass_ece": 0.011605590146725192, + "coverage": [ + { + "threshold": 0.5, + "selected": 9830, + "coverage": 0.9333459931636916, + "accuracy": 0.9686659772492244, + "expected_accuracy": 0.9607494065785012, + "expected_risk": 0.039250593421498836 + }, + { + "threshold": 0.7, + "selected": 9500, + "coverage": 0.9020129130269654, + "accuracy": 0.976356343677433, + "expected_accuracy": 0.9749649122807017, + "expected_risk": 0.0250350877192983 + }, + { + "threshold": 0.8, + "selected": 9388, + "coverage": 0.8913786555260159, + "accuracy": 0.9796136193830718, + "expected_accuracy": 0.9785719358045732, + "expected_risk": 0.021428064195426755 + }, + { + "threshold": 0.9, + "selected": 9116, + "coverage": 0.8655526015951386, + "accuracy": 0.9880377524143986, + "expected_accuracy": 0.9878236068451075, + "expected_risk": 0.012176393154892495 + }, + { + "threshold": 0.95, + "selected": 8634, + "coverage": 0.819787314849981, + "accuracy": 0.9950196895992588, + "expected_accuracy": 0.9950196895992588, + "expected_risk": 0.004980310400741206 + }, + { + "threshold": 0.99, + "selected": 6742, + "coverage": 0.6401443220660843, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 10532, + "hard_count": 10046, + "accuracy": 0.9471431415488751, + "expected_accuracy": 0.9195253563797051, + "nll": 0.19538003409263777, + "brier": 0.07611697556114595, + "expected_brier": 0.10416793987302937, + "ece_bins": 15, + "multiclass_ece": 0.011605590146725192, + "coverage": [ + { + "threshold": 0.5, + "selected": 9830, + "coverage": 0.9333459931636916, + "accuracy": 0.9686659772492244, + "expected_accuracy": 0.9607494065785012, + "expected_risk": 0.039250593421498836 + }, + { + "threshold": 0.7, + "selected": 9500, + "coverage": 0.9020129130269654, + "accuracy": 0.976356343677433, + "expected_accuracy": 0.9749649122807017, + "expected_risk": 0.0250350877192983 + }, + { + "threshold": 0.8, + "selected": 9388, + "coverage": 0.8913786555260159, + "accuracy": 0.9796136193830718, + "expected_accuracy": 0.9785719358045732, + "expected_risk": 0.021428064195426755 + }, + { + "threshold": 0.9, + "selected": 9116, + "coverage": 0.8655526015951386, + "accuracy": 0.9880377524143986, + "expected_accuracy": 0.9878236068451075, + "expected_risk": 0.012176393154892495 + }, + { + "threshold": 0.95, + "selected": 8634, + "coverage": 0.819787314849981, + "accuracy": 0.9950196895992588, + "expected_accuracy": 0.9950196895992588, + "expected_risk": 0.004980310400741206 + }, + { + "threshold": 0.99, + "selected": 6742, + "coverage": 0.6401443220660843, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "by_source": { + "customer-control-v1": { + "count": 606, + "successful": 606, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 574, + "accuracy_all_rows_failures_incorrect": 1.0, + "expected_accuracy_all_rows_failures_zero": 0.9722222222222222, + "coverage": [ + { + "threshold": 0.5, + "selected": 598, + "total": 606, + "coverage": 0.9867986798679867 + }, + { + "threshold": 0.7, + "selected": 576, + "total": 606, + "coverage": 0.9504950495049505 + }, + { + "threshold": 0.8, + "selected": 576, + "total": 606, + "coverage": 0.9504950495049505 + }, + { + "threshold": 0.9, + "selected": 573, + "total": 606, + "coverage": 0.9455445544554455 + }, + { + "threshold": 0.95, + "selected": 565, + "total": 606, + "coverage": 0.9323432343234324 + }, + { + "threshold": 0.99, + "selected": 504, + "total": 606, + "coverage": 0.8316831683168316 + } + ], + "probability_metrics_all_rows": { + "count": 606, + "hard_count": 574, + "accuracy": 1.0, + "expected_accuracy": 0.9722222222222222, + "nll": 0.04670156852740247, + "brier": 0.0013588176050983829, + "expected_brier": 0.029136595382876163, + "ece_bins": 15, + "multiclass_ece": 0.005803372421616342, + "coverage": [ + { + "threshold": 0.5, + "selected": 598, + "coverage": 0.9867986798679867, + "accuracy": 1.0, + "expected_accuracy": 0.979933110367893, + "expected_risk": 0.02006688963210701 + }, + { + "threshold": 0.7, + "selected": 576, + "coverage": 0.9504950495049505, + "accuracy": 1.0, + "expected_accuracy": 0.9982638888888888, + "expected_risk": 0.0017361111111111605 + }, + { + "threshold": 0.8, + "selected": 576, + "coverage": 0.9504950495049505, + "accuracy": 1.0, + "expected_accuracy": 0.9982638888888888, + "expected_risk": 0.0017361111111111605 + }, + { + "threshold": 0.9, + "selected": 573, + "coverage": 0.9455445544554455, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 565, + "coverage": 0.9323432343234324, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 504, + "coverage": 0.8316831683168316, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 606, + "hard_count": 574, + "accuracy": 1.0, + "expected_accuracy": 0.9722222222222222, + "nll": 0.04670156852740247, + "brier": 0.0013588176050983829, + "expected_brier": 0.029136595382876163, + "ece_bins": 15, + "multiclass_ece": 0.005803372421616342, + "coverage": [ + { + "threshold": 0.5, + "selected": 598, + "coverage": 0.9867986798679867, + "accuracy": 1.0, + "expected_accuracy": 0.979933110367893, + "expected_risk": 0.02006688963210701 + }, + { + "threshold": 0.7, + "selected": 576, + "coverage": 0.9504950495049505, + "accuracy": 1.0, + "expected_accuracy": 0.9982638888888888, + "expected_risk": 0.0017361111111111605 + }, + { + "threshold": 0.8, + "selected": 576, + "coverage": 0.9504950495049505, + "accuracy": 1.0, + "expected_accuracy": 0.9982638888888888, + "expected_risk": 0.0017361111111111605 + }, + { + "threshold": 0.9, + "selected": 573, + "coverage": 0.9455445544554455, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 565, + "coverage": 0.9323432343234324, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 504, + "coverage": 0.8316831683168316, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "painting-geometry-v1": { + "count": 3072, + "successful": 3072, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 3072, + "accuracy_all_rows_failures_incorrect": 0.9964192708333334, + "expected_accuracy_all_rows_failures_zero": 0.9964192708333334, + "coverage": [ + { + "threshold": 0.5, + "selected": 3072, + "total": 3072, + "coverage": 1.0 + }, + { + "threshold": 0.7, + "selected": 3064, + "total": 3072, + "coverage": 0.9973958333333334 + }, + { + "threshold": 0.8, + "selected": 3060, + "total": 3072, + "coverage": 0.99609375 + }, + { + "threshold": 0.9, + "selected": 3058, + "total": 3072, + "coverage": 0.9954427083333334 + }, + { + "threshold": 0.95, + "selected": 3039, + "total": 3072, + "coverage": 0.9892578125 + }, + { + "threshold": 0.99, + "selected": 2768, + "total": 3072, + "coverage": 0.9010416666666666 + } + ], + "probability_metrics_all_rows": { + "count": 3072, + "hard_count": 3072, + "accuracy": 0.9964192708333334, + "expected_accuracy": 0.9964192708333334, + "nll": 0.012696533729082774, + "brier": 0.005503376394520538, + "expected_brier": 0.0055033763945205386, + "ece_bins": 15, + "multiclass_ece": 0.0037695271810740086, + "coverage": [ + { + "threshold": 0.5, + "selected": 3072, + "coverage": 1.0, + "accuracy": 0.9964192708333334, + "expected_accuracy": 0.9964192708333334, + "expected_risk": 0.0035807291666666297 + }, + { + "threshold": 0.7, + "selected": 3064, + "coverage": 0.9973958333333334, + "accuracy": 0.9977154046997389, + "expected_accuracy": 0.9977154046997389, + "expected_risk": 0.0022845953002611497 + }, + { + "threshold": 0.8, + "selected": 3060, + "coverage": 0.99609375, + "accuracy": 0.9980392156862745, + "expected_accuracy": 0.9980392156862745, + "expected_risk": 0.0019607843137254832 + }, + { + "threshold": 0.9, + "selected": 3058, + "coverage": 0.9954427083333334, + "accuracy": 0.9983649444081099, + "expected_accuracy": 0.9983649444081099, + "expected_risk": 0.0016350555918901444 + }, + { + "threshold": 0.95, + "selected": 3039, + "coverage": 0.9892578125, + "accuracy": 0.9983547219480092, + "expected_accuracy": 0.9983547219480092, + "expected_risk": 0.0016452780519907728 + }, + { + "threshold": 0.99, + "selected": 2768, + "coverage": 0.9010416666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 3072, + "hard_count": 3072, + "accuracy": 0.9964192708333334, + "expected_accuracy": 0.9964192708333334, + "nll": 0.012696533729082774, + "brier": 0.005503376394520538, + "expected_brier": 0.0055033763945205386, + "ece_bins": 15, + "multiclass_ece": 0.0037695271810740086, + "coverage": [ + { + "threshold": 0.5, + "selected": 3072, + "coverage": 1.0, + "accuracy": 0.9964192708333334, + "expected_accuracy": 0.9964192708333334, + "expected_risk": 0.0035807291666666297 + }, + { + "threshold": 0.7, + "selected": 3064, + "coverage": 0.9973958333333334, + "accuracy": 0.9977154046997389, + "expected_accuracy": 0.9977154046997389, + "expected_risk": 0.0022845953002611497 + }, + { + "threshold": 0.8, + "selected": 3060, + "coverage": 0.99609375, + "accuracy": 0.9980392156862745, + "expected_accuracy": 0.9980392156862745, + "expected_risk": 0.0019607843137254832 + }, + { + "threshold": 0.9, + "selected": 3058, + "coverage": 0.9954427083333334, + "accuracy": 0.9983649444081099, + "expected_accuracy": 0.9983649444081099, + "expected_risk": 0.0016350555918901444 + }, + { + "threshold": 0.95, + "selected": 3039, + "coverage": 0.9892578125, + "accuracy": 0.9983547219480092, + "expected_accuracy": 0.9983547219480092, + "expected_risk": 0.0016452780519907728 + }, + { + "threshold": 0.99, + "selected": 2768, + "coverage": 0.9010416666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "reasoning-control-v1": { + "count": 681, + "successful": 681, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 681, + "accuracy_all_rows_failures_incorrect": 0.7797356828193832, + "expected_accuracy_all_rows_failures_zero": 0.7797356828193832, + "coverage": [ + { + "threshold": 0.5, + "selected": 572, + "total": 681, + "coverage": 0.8399412628487518 + }, + { + "threshold": 0.7, + "selected": 515, + "total": 681, + "coverage": 0.7562408223201175 + }, + { + "threshold": 0.8, + "selected": 487, + "total": 681, + "coverage": 0.7151248164464024 + }, + { + "threshold": 0.9, + "selected": 424, + "total": 681, + "coverage": 0.6226138032305433 + }, + { + "threshold": 0.95, + "selected": 351, + "total": 681, + "coverage": 0.5154185022026432 + }, + { + "threshold": 0.99, + "selected": 180, + "total": 681, + "coverage": 0.2643171806167401 + } + ], + "probability_metrics_all_rows": { + "count": 681, + "hard_count": 681, + "accuracy": 0.7797356828193832, + "expected_accuracy": 0.7797356828193832, + "nll": 0.5276862396629047, + "brier": 0.30043397912430997, + "expected_brier": 0.30043397912430997, + "ece_bins": 15, + "multiclass_ece": 0.06116577188210845, + "coverage": [ + { + "threshold": 0.5, + "selected": 572, + "coverage": 0.8399412628487518, + "accuracy": 0.8513986013986014, + "expected_accuracy": 0.8513986013986014, + "expected_risk": 0.14860139860139865 + }, + { + "threshold": 0.7, + "selected": 515, + "coverage": 0.7562408223201175, + "accuracy": 0.883495145631068, + "expected_accuracy": 0.883495145631068, + "expected_risk": 0.11650485436893199 + }, + { + "threshold": 0.8, + "selected": 487, + "coverage": 0.7151248164464024, + "accuracy": 0.9055441478439425, + "expected_accuracy": 0.9055441478439425, + "expected_risk": 0.09445585215605745 + }, + { + "threshold": 0.9, + "selected": 424, + "coverage": 0.6226138032305433, + "accuracy": 0.9410377358490566, + "expected_accuracy": 0.9410377358490566, + "expected_risk": 0.05896226415094341 + }, + { + "threshold": 0.95, + "selected": 351, + "coverage": 0.5154185022026432, + "accuracy": 0.9886039886039886, + "expected_accuracy": 0.9886039886039886, + "expected_risk": 0.01139601139601143 + }, + { + "threshold": 0.99, + "selected": 180, + "coverage": 0.2643171806167401, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 681, + "hard_count": 681, + "accuracy": 0.7797356828193832, + "expected_accuracy": 0.7797356828193832, + "nll": 0.5276862396629047, + "brier": 0.30043397912430997, + "expected_brier": 0.30043397912430997, + "ece_bins": 15, + "multiclass_ece": 0.06116577188210845, + "coverage": [ + { + "threshold": 0.5, + "selected": 572, + "coverage": 0.8399412628487518, + "accuracy": 0.8513986013986014, + "expected_accuracy": 0.8513986013986014, + "expected_risk": 0.14860139860139865 + }, + { + "threshold": 0.7, + "selected": 515, + "coverage": 0.7562408223201175, + "accuracy": 0.883495145631068, + "expected_accuracy": 0.883495145631068, + "expected_risk": 0.11650485436893199 + }, + { + "threshold": 0.8, + "selected": 487, + "coverage": 0.7151248164464024, + "accuracy": 0.9055441478439425, + "expected_accuracy": 0.9055441478439425, + "expected_risk": 0.09445585215605745 + }, + { + "threshold": 0.9, + "selected": 424, + "coverage": 0.6226138032305433, + "accuracy": 0.9410377358490566, + "expected_accuracy": 0.9410377358490566, + "expected_risk": 0.05896226415094341 + }, + { + "threshold": 0.95, + "selected": 351, + "coverage": 0.5154185022026432, + "accuracy": 0.9886039886039886, + "expected_accuracy": 0.9886039886039886, + "expected_risk": 0.01139601139601143 + }, + { + "threshold": 0.99, + "selected": 180, + "coverage": 0.2643171806167401, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "snake-v1": { + "count": 1392, + "successful": 1392, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 1279, + "accuracy_all_rows_failures_incorrect": 0.8780297107114934, + "expected_accuracy_all_rows_failures_zero": 0.8461446360153256, + "coverage": [ + { + "threshold": 0.5, + "selected": 1352, + "total": 1392, + "coverage": 0.9712643678160919 + }, + { + "threshold": 0.7, + "selected": 1237, + "total": 1392, + "coverage": 0.8886494252873564 + }, + { + "threshold": 0.8, + "selected": 1186, + "total": 1392, + "coverage": 0.8520114942528736 + }, + { + "threshold": 0.9, + "selected": 1014, + "total": 1392, + "coverage": 0.728448275862069 + }, + { + "threshold": 0.95, + "selected": 743, + "total": 1392, + "coverage": 0.5337643678160919 + }, + { + "threshold": 0.99, + "selected": 159, + "total": 1392, + "coverage": 0.11422413793103449 + } + ], + "probability_metrics_all_rows": { + "count": 1392, + "hard_count": 1279, + "accuracy": 0.8780297107114934, + "expected_accuracy": 0.8461446360153256, + "nll": 0.3775572072892607, + "brier": 0.18597837736131362, + "expected_brier": 0.22704638502414884, + "ece_bins": 15, + "multiclass_ece": 0.0526435483183463, + "coverage": [ + { + "threshold": 0.5, + "selected": 1352, + "coverage": 0.9712643678160919, + "accuracy": 0.8844022169437846, + "expected_accuracy": 0.8583579881656804, + "expected_risk": 0.14164201183431957 + }, + { + "threshold": 0.7, + "selected": 1237, + "coverage": 0.8886494252873564, + "accuracy": 0.8962108731466227, + "expected_accuracy": 0.8881703045001348, + "expected_risk": 0.11182969549986521 + }, + { + "threshold": 0.8, + "selected": 1186, + "coverage": 0.8520114942528736, + "accuracy": 0.9024807527801539, + "expected_accuracy": 0.8961495222034851, + "expected_risk": 0.1038504777965149 + }, + { + "threshold": 0.9, + "selected": 1014, + "coverage": 0.728448275862069, + "accuracy": 0.9376237623762376, + "expected_accuracy": 0.9358974358974359, + "expected_risk": 0.0641025641025641 + }, + { + "threshold": 0.95, + "selected": 743, + "coverage": 0.5337643678160919, + "accuracy": 0.9663526244952894, + "expected_accuracy": 0.9663526244952894, + "expected_risk": 0.03364737550471064 + }, + { + "threshold": 0.99, + "selected": 159, + "coverage": 0.11422413793103449, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 1392, + "hard_count": 1279, + "accuracy": 0.8780297107114934, + "expected_accuracy": 0.8461446360153256, + "nll": 0.3775572072892607, + "brier": 0.18597837736131362, + "expected_brier": 0.22704638502414884, + "ece_bins": 15, + "multiclass_ece": 0.0526435483183463, + "coverage": [ + { + "threshold": 0.5, + "selected": 1352, + "coverage": 0.9712643678160919, + "accuracy": 0.8844022169437846, + "expected_accuracy": 0.8583579881656804, + "expected_risk": 0.14164201183431957 + }, + { + "threshold": 0.7, + "selected": 1237, + "coverage": 0.8886494252873564, + "accuracy": 0.8962108731466227, + "expected_accuracy": 0.8881703045001348, + "expected_risk": 0.11182969549986521 + }, + { + "threshold": 0.8, + "selected": 1186, + "coverage": 0.8520114942528736, + "accuracy": 0.9024807527801539, + "expected_accuracy": 0.8961495222034851, + "expected_risk": 0.1038504777965149 + }, + { + "threshold": 0.9, + "selected": 1014, + "coverage": 0.728448275862069, + "accuracy": 0.9376237623762376, + "expected_accuracy": 0.9358974358974359, + "expected_risk": 0.0641025641025641 + }, + { + "threshold": 0.95, + "selected": 743, + "coverage": 0.5337643678160919, + "accuracy": 0.9663526244952894, + "expected_accuracy": 0.9663526244952894, + "expected_risk": 0.03364737550471064 + }, + { + "threshold": 0.99, + "selected": 159, + "coverage": 0.11422413793103449, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "tic_tac_toe-v1": { + "count": 472, + "successful": 472, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 256, + "accuracy_all_rows_failures_incorrect": 0.4765625, + "expected_accuracy_all_rows_failures_zero": 0.41646489104116224, + "coverage": [ + { + "threshold": 0.5, + "selected": 107, + "total": 472, + "coverage": 0.2266949152542373 + }, + { + "threshold": 0.7, + "selected": 10, + "total": 472, + "coverage": 0.0211864406779661 + }, + { + "threshold": 0.8, + "selected": 1, + "total": 472, + "coverage": 0.00211864406779661 + }, + { + "threshold": 0.9, + "selected": 0, + "total": 472, + "coverage": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "total": 472, + "coverage": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "total": 472, + "coverage": 0.0 + } + ], + "probability_metrics_all_rows": { + "count": 472, + "hard_count": 256, + "accuracy": 0.4765625, + "expected_accuracy": 0.41646489104116224, + "nll": 1.1856874570601634, + "brier": 0.3986960943836407, + "expected_brier": 0.6686718813085801, + "ece_bins": 15, + "multiclass_ece": 0.05099574700428666, + "coverage": [ + { + "threshold": 0.5, + "selected": 107, + "coverage": 0.2266949152542373, + "accuracy": 0.75, + "expected_accuracy": 0.6230529595015577, + "expected_risk": 0.3769470404984423 + }, + { + "threshold": 0.7, + "selected": 10, + "coverage": 0.0211864406779661, + "accuracy": 0.8888888888888888, + "expected_accuracy": 0.85, + "expected_risk": 0.15000000000000002 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 0.00211864406779661, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 472, + "hard_count": 256, + "accuracy": 0.4765625, + "expected_accuracy": 0.41646489104116224, + "nll": 1.1856874570601634, + "brier": 0.3986960943836407, + "expected_brier": 0.6686718813085801, + "ece_bins": 15, + "multiclass_ece": 0.05099574700428666, + "coverage": [ + { + "threshold": 0.5, + "selected": 107, + "coverage": 0.2266949152542373, + "accuracy": 0.75, + "expected_accuracy": 0.6230529595015577, + "expected_risk": 0.3769470404984423 + }, + { + "threshold": 0.7, + "selected": 10, + "coverage": 0.0211864406779661, + "accuracy": 0.8888888888888888, + "expected_accuracy": 0.85, + "expected_risk": 0.15000000000000002 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 0.00211864406779661, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + } + }, + "tile_platformer-v1": { + "count": 202, + "successful": 202, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 202, + "accuracy_all_rows_failures_incorrect": 0.9752475247524752, + "expected_accuracy_all_rows_failures_zero": 0.9752475247524752, + "coverage": [ + { + "threshold": 0.5, + "selected": 202, + "total": 202, + "coverage": 1.0 + }, + { + "threshold": 0.7, + "selected": 202, + "total": 202, + "coverage": 1.0 + }, + { + "threshold": 0.8, + "selected": 200, + "total": 202, + "coverage": 0.9900990099009901 + }, + { + "threshold": 0.9, + "selected": 198, + "total": 202, + "coverage": 0.9801980198019802 + }, + { + "threshold": 0.95, + "selected": 183, + "total": 202, + "coverage": 0.905940594059406 + }, + { + "threshold": 0.99, + "selected": 8, + "total": 202, + "coverage": 0.039603960396039604 + } + ], + "probability_metrics_all_rows": { + "count": 202, + "hard_count": 202, + "accuracy": 0.9752475247524752, + "expected_accuracy": 0.9752475247524752, + "nll": 0.13723787641738036, + "brier": 0.048090580988872436, + "expected_brier": 0.048090580988872436, + "ece_bins": 15, + "multiclass_ece": 0.01462437849252733, + "coverage": [ + { + "threshold": 0.5, + "selected": 202, + "coverage": 1.0, + "accuracy": 0.9752475247524752, + "expected_accuracy": 0.9752475247524752, + "expected_risk": 0.024752475247524774 + }, + { + "threshold": 0.7, + "selected": 202, + "coverage": 1.0, + "accuracy": 0.9752475247524752, + "expected_accuracy": 0.9752475247524752, + "expected_risk": 0.024752475247524774 + }, + { + "threshold": 0.8, + "selected": 200, + "coverage": 0.9900990099009901, + "accuracy": 0.975, + "expected_accuracy": 0.975, + "expected_risk": 0.025000000000000022 + }, + { + "threshold": 0.9, + "selected": 198, + "coverage": 0.9801980198019802, + "accuracy": 0.9797979797979798, + "expected_accuracy": 0.9797979797979798, + "expected_risk": 0.02020202020202022 + }, + { + "threshold": 0.95, + "selected": 183, + "coverage": 0.905940594059406, + "accuracy": 0.9781420765027322, + "expected_accuracy": 0.9781420765027322, + "expected_risk": 0.021857923497267784 + }, + { + "threshold": 0.99, + "selected": 8, + "coverage": 0.039603960396039604, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 202, + "hard_count": 202, + "accuracy": 0.9752475247524752, + "expected_accuracy": 0.9752475247524752, + "nll": 0.13723787641738036, + "brier": 0.048090580988872436, + "expected_brier": 0.048090580988872436, + "ece_bins": 15, + "multiclass_ece": 0.01462437849252733, + "coverage": [ + { + "threshold": 0.5, + "selected": 202, + "coverage": 1.0, + "accuracy": 0.9752475247524752, + "expected_accuracy": 0.9752475247524752, + "expected_risk": 0.024752475247524774 + }, + { + "threshold": 0.7, + "selected": 202, + "coverage": 1.0, + "accuracy": 0.9752475247524752, + "expected_accuracy": 0.9752475247524752, + "expected_risk": 0.024752475247524774 + }, + { + "threshold": 0.8, + "selected": 200, + "coverage": 0.9900990099009901, + "accuracy": 0.975, + "expected_accuracy": 0.975, + "expected_risk": 0.025000000000000022 + }, + { + "threshold": 0.9, + "selected": 198, + "coverage": 0.9801980198019802, + "accuracy": 0.9797979797979798, + "expected_accuracy": 0.9797979797979798, + "expected_risk": 0.02020202020202022 + }, + { + "threshold": 0.95, + "selected": 183, + "coverage": 0.905940594059406, + "accuracy": 0.9781420765027322, + "expected_accuracy": 0.9781420765027322, + "expected_risk": 0.021857923497267784 + }, + { + "threshold": 0.99, + "selected": 8, + "coverage": 0.039603960396039604, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "trex_runner-v1": { + "count": 4, + "successful": 4, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 4, + "accuracy_all_rows_failures_incorrect": 0.25, + "expected_accuracy_all_rows_failures_zero": 0.25, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "total": 4, + "coverage": 1.0 + }, + { + "threshold": 0.7, + "selected": 2, + "total": 4, + "coverage": 0.5 + }, + { + "threshold": 0.8, + "selected": 2, + "total": 4, + "coverage": 0.5 + }, + { + "threshold": 0.9, + "selected": 1, + "total": 4, + "coverage": 0.25 + }, + { + "threshold": 0.95, + "selected": 0, + "total": 4, + "coverage": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "total": 4, + "coverage": 0.0 + } + ], + "probability_metrics_all_rows": { + "count": 4, + "hard_count": 4, + "accuracy": 0.25, + "expected_accuracy": 0.25, + "nll": 1.380181719997761, + "brier": 0.8197811047574941, + "expected_brier": 0.8197811047574941, + "ece_bins": 15, + "multiclass_ece": 0.47413478912915286, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.25, + "expected_accuracy": 0.25, + "expected_risk": 0.75 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 0.5, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 0.5, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 0.25, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 4, + "hard_count": 4, + "accuracy": 0.25, + "expected_accuracy": 0.25, + "nll": 1.380181719997761, + "brier": 0.8197811047574941, + "expected_brier": 0.8197811047574941, + "ece_bins": 15, + "multiclass_ece": 0.47413478912915286, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.25, + "expected_accuracy": 0.25, + "expected_risk": 0.75 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 0.5, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 0.5, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 0.25, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + } + }, + "vizdoom-basic-v1": { + "count": 858, + "successful": 858, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 858, + "accuracy_all_rows_failures_incorrect": 0.9685314685314685, + "expected_accuracy_all_rows_failures_zero": 0.9685314685314685, + "coverage": [ + { + "threshold": 0.5, + "selected": 853, + "total": 858, + "coverage": 0.9941724941724942 + }, + { + "threshold": 0.7, + "selected": 834, + "total": 858, + "coverage": 0.972027972027972 + }, + { + "threshold": 0.8, + "selected": 821, + "total": 858, + "coverage": 0.9568764568764568 + }, + { + "threshold": 0.9, + "selected": 800, + "total": 858, + "coverage": 0.9324009324009324 + }, + { + "threshold": 0.95, + "selected": 752, + "total": 858, + "coverage": 0.8764568764568764 + }, + { + "threshold": 0.99, + "selected": 480, + "total": 858, + "coverage": 0.5594405594405595 + } + ], + "probability_metrics_all_rows": { + "count": 858, + "hard_count": 858, + "accuracy": 0.9685314685314685, + "expected_accuracy": 0.9685314685314685, + "nll": 0.08747268287742492, + "brier": 0.04462544178795084, + "expected_brier": 0.04462544178795084, + "ece_bins": 15, + "multiclass_ece": 0.0214576175261959, + "coverage": [ + { + "threshold": 0.5, + "selected": 853, + "coverage": 0.9941724941724942, + "accuracy": 0.9718640093786636, + "expected_accuracy": 0.9718640093786636, + "expected_risk": 0.028135990621336426 + }, + { + "threshold": 0.7, + "selected": 834, + "coverage": 0.972027972027972, + "accuracy": 0.9844124700239808, + "expected_accuracy": 0.9844124700239808, + "expected_risk": 0.015587529976019199 + }, + { + "threshold": 0.8, + "selected": 821, + "coverage": 0.9568764568764568, + "accuracy": 0.9866017052375152, + "expected_accuracy": 0.9866017052375152, + "expected_risk": 0.013398294762484775 + }, + { + "threshold": 0.9, + "selected": 800, + "coverage": 0.9324009324009324, + "accuracy": 0.99375, + "expected_accuracy": 0.99375, + "expected_risk": 0.006249999999999978 + }, + { + "threshold": 0.95, + "selected": 752, + "coverage": 0.8764568764568764, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 480, + "coverage": 0.5594405594405595, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 858, + "hard_count": 858, + "accuracy": 0.9685314685314685, + "expected_accuracy": 0.9685314685314685, + "nll": 0.08747268287742492, + "brier": 0.04462544178795084, + "expected_brier": 0.04462544178795084, + "ece_bins": 15, + "multiclass_ece": 0.0214576175261959, + "coverage": [ + { + "threshold": 0.5, + "selected": 853, + "coverage": 0.9941724941724942, + "accuracy": 0.9718640093786636, + "expected_accuracy": 0.9718640093786636, + "expected_risk": 0.028135990621336426 + }, + { + "threshold": 0.7, + "selected": 834, + "coverage": 0.972027972027972, + "accuracy": 0.9844124700239808, + "expected_accuracy": 0.9844124700239808, + "expected_risk": 0.015587529976019199 + }, + { + "threshold": 0.8, + "selected": 821, + "coverage": 0.9568764568764568, + "accuracy": 0.9866017052375152, + "expected_accuracy": 0.9866017052375152, + "expected_risk": 0.013398294762484775 + }, + { + "threshold": 0.9, + "selected": 800, + "coverage": 0.9324009324009324, + "accuracy": 0.99375, + "expected_accuracy": 0.99375, + "expected_risk": 0.006249999999999978 + }, + { + "threshold": 0.95, + "selected": 752, + "coverage": 0.8764568764568764, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 480, + "coverage": 0.5594405594405595, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "wikispeedia-v1": { + "count": 176, + "successful": 176, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 51, + "accuracy_all_rows_failures_incorrect": 0.39215686274509803, + "expected_accuracy_all_rows_failures_zero": 0.25494105011150464, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "total": 176, + "coverage": 0.005681818181818182 + }, + { + "threshold": 0.7, + "selected": 0, + "total": 176, + "coverage": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "total": 176, + "coverage": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "total": 176, + "coverage": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "total": 176, + "coverage": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "total": 176, + "coverage": 0.0 + } + ], + "probability_metrics_all_rows": { + "count": 176, + "hard_count": 51, + "accuracy": 0.39215686274509803, + "expected_accuracy": 0.25494105011150464, + "nll": 2.2057266068313712, + "brier": 0.32864256792900676, + "expected_brier": 0.862757253748238, + "ece_bins": 15, + "multiclass_ece": 0.09196286653058698, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 0.005681818181818182, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 176, + "hard_count": 51, + "accuracy": 0.39215686274509803, + "expected_accuracy": 0.25494105011150464, + "nll": 2.2057266068313712, + "brier": 0.32864256792900676, + "expected_brier": 0.862757253748238, + "ece_bins": 15, + "multiclass_ece": 0.09196286653058698, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 0.005681818181818182, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + } + }, + "workflow-controls-v1/agent_trace_observability": { + "count": 357, + "successful": 357, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 357, + "accuracy_all_rows_failures_incorrect": 0.988795518207283, + "expected_accuracy_all_rows_failures_zero": 0.988795518207283, + "coverage": [ + { + "threshold": 0.5, + "selected": 357, + "total": 357, + "coverage": 1.0 + }, + { + "threshold": 0.7, + "selected": 355, + "total": 357, + "coverage": 0.9943977591036415 + }, + { + "threshold": 0.8, + "selected": 354, + "total": 357, + "coverage": 0.9915966386554622 + }, + { + "threshold": 0.9, + "selected": 353, + "total": 357, + "coverage": 0.988795518207283 + }, + { + "threshold": 0.95, + "selected": 352, + "total": 357, + "coverage": 0.9859943977591037 + }, + { + "threshold": 0.99, + "selected": 301, + "total": 357, + "coverage": 0.8431372549019608 + } + ], + "probability_metrics_all_rows": { + "count": 357, + "hard_count": 357, + "accuracy": 0.988795518207283, + "expected_accuracy": 0.988795518207283, + "nll": 0.026771587416750012, + "brier": 0.0157428306754681, + "expected_brier": 0.0157428306754681, + "ece_bins": 15, + "multiclass_ece": 0.006613012775607079, + "coverage": [ + { + "threshold": 0.5, + "selected": 357, + "coverage": 1.0, + "accuracy": 0.988795518207283, + "expected_accuracy": 0.988795518207283, + "expected_risk": 0.011204481792717047 + }, + { + "threshold": 0.7, + "selected": 355, + "coverage": 0.9943977591036415, + "accuracy": 0.9915492957746479, + "expected_accuracy": 0.9915492957746479, + "expected_risk": 0.008450704225352101 + }, + { + "threshold": 0.8, + "selected": 354, + "coverage": 0.9915966386554622, + "accuracy": 0.9943502824858758, + "expected_accuracy": 0.9943502824858758, + "expected_risk": 0.005649717514124242 + }, + { + "threshold": 0.9, + "selected": 353, + "coverage": 0.988795518207283, + "accuracy": 0.9971671388101983, + "expected_accuracy": 0.9971671388101983, + "expected_risk": 0.0028328611898017497 + }, + { + "threshold": 0.95, + "selected": 352, + "coverage": 0.9859943977591037, + "accuracy": 0.9971590909090909, + "expected_accuracy": 0.9971590909090909, + "expected_risk": 0.0028409090909090606 + }, + { + "threshold": 0.99, + "selected": 301, + "coverage": 0.8431372549019608, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 357, + "hard_count": 357, + "accuracy": 0.988795518207283, + "expected_accuracy": 0.988795518207283, + "nll": 0.026771587416750012, + "brier": 0.0157428306754681, + "expected_brier": 0.0157428306754681, + "ece_bins": 15, + "multiclass_ece": 0.006613012775607079, + "coverage": [ + { + "threshold": 0.5, + "selected": 357, + "coverage": 1.0, + "accuracy": 0.988795518207283, + "expected_accuracy": 0.988795518207283, + "expected_risk": 0.011204481792717047 + }, + { + "threshold": 0.7, + "selected": 355, + "coverage": 0.9943977591036415, + "accuracy": 0.9915492957746479, + "expected_accuracy": 0.9915492957746479, + "expected_risk": 0.008450704225352101 + }, + { + "threshold": 0.8, + "selected": 354, + "coverage": 0.9915966386554622, + "accuracy": 0.9943502824858758, + "expected_accuracy": 0.9943502824858758, + "expected_risk": 0.005649717514124242 + }, + { + "threshold": 0.9, + "selected": 353, + "coverage": 0.988795518207283, + "accuracy": 0.9971671388101983, + "expected_accuracy": 0.9971671388101983, + "expected_risk": 0.0028328611898017497 + }, + { + "threshold": 0.95, + "selected": 352, + "coverage": 0.9859943977591037, + "accuracy": 0.9971590909090909, + "expected_accuracy": 0.9971590909090909, + "expected_risk": 0.0028409090909090606 + }, + { + "threshold": 0.99, + "selected": 301, + "coverage": 0.8431372549019608, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "workflow-controls-v1/customer_service": { + "count": 456, + "successful": 456, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 456, + "accuracy_all_rows_failures_incorrect": 0.9912280701754386, + "expected_accuracy_all_rows_failures_zero": 0.9912280701754386, + "coverage": [ + { + "threshold": 0.5, + "selected": 456, + "total": 456, + "coverage": 1.0 + }, + { + "threshold": 0.7, + "selected": 455, + "total": 456, + "coverage": 0.9978070175438597 + }, + { + "threshold": 0.8, + "selected": 454, + "total": 456, + "coverage": 0.9956140350877193 + }, + { + "threshold": 0.9, + "selected": 454, + "total": 456, + "coverage": 0.9956140350877193 + }, + { + "threshold": 0.95, + "selected": 452, + "total": 456, + "coverage": 0.9912280701754386 + }, + { + "threshold": 0.99, + "selected": 382, + "total": 456, + "coverage": 0.8377192982456141 + } + ], + "probability_metrics_all_rows": { + "count": 456, + "hard_count": 456, + "accuracy": 0.9912280701754386, + "expected_accuracy": 0.9912280701754386, + "nll": 0.024863508782455843, + "brier": 0.012699999145872375, + "expected_brier": 0.012699999145872375, + "ece_bins": 15, + "multiclass_ece": 0.004145846144954706, + "coverage": [ + { + "threshold": 0.5, + "selected": 456, + "coverage": 1.0, + "accuracy": 0.9912280701754386, + "expected_accuracy": 0.9912280701754386, + "expected_risk": 0.00877192982456143 + }, + { + "threshold": 0.7, + "selected": 455, + "coverage": 0.9978070175438597, + "accuracy": 0.9934065934065934, + "expected_accuracy": 0.9934065934065934, + "expected_risk": 0.00659340659340657 + }, + { + "threshold": 0.8, + "selected": 454, + "coverage": 0.9956140350877193, + "accuracy": 0.9955947136563876, + "expected_accuracy": 0.9955947136563876, + "expected_risk": 0.004405286343612369 + }, + { + "threshold": 0.9, + "selected": 454, + "coverage": 0.9956140350877193, + "accuracy": 0.9955947136563876, + "expected_accuracy": 0.9955947136563876, + "expected_risk": 0.004405286343612369 + }, + { + "threshold": 0.95, + "selected": 452, + "coverage": 0.9912280701754386, + "accuracy": 0.995575221238938, + "expected_accuracy": 0.995575221238938, + "expected_risk": 0.004424778761061954 + }, + { + "threshold": 0.99, + "selected": 382, + "coverage": 0.8377192982456141, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 456, + "hard_count": 456, + "accuracy": 0.9912280701754386, + "expected_accuracy": 0.9912280701754386, + "nll": 0.024863508782455843, + "brier": 0.012699999145872375, + "expected_brier": 0.012699999145872375, + "ece_bins": 15, + "multiclass_ece": 0.004145846144954706, + "coverage": [ + { + "threshold": 0.5, + "selected": 456, + "coverage": 1.0, + "accuracy": 0.9912280701754386, + "expected_accuracy": 0.9912280701754386, + "expected_risk": 0.00877192982456143 + }, + { + "threshold": 0.7, + "selected": 455, + "coverage": 0.9978070175438597, + "accuracy": 0.9934065934065934, + "expected_accuracy": 0.9934065934065934, + "expected_risk": 0.00659340659340657 + }, + { + "threshold": 0.8, + "selected": 454, + "coverage": 0.9956140350877193, + "accuracy": 0.9955947136563876, + "expected_accuracy": 0.9955947136563876, + "expected_risk": 0.004405286343612369 + }, + { + "threshold": 0.9, + "selected": 454, + "coverage": 0.9956140350877193, + "accuracy": 0.9955947136563876, + "expected_accuracy": 0.9955947136563876, + "expected_risk": 0.004405286343612369 + }, + { + "threshold": 0.95, + "selected": 452, + "coverage": 0.9912280701754386, + "accuracy": 0.995575221238938, + "expected_accuracy": 0.995575221238938, + "expected_risk": 0.004424778761061954 + }, + { + "threshold": 0.99, + "selected": 382, + "coverage": 0.8377192982456141, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "workflow-controls-v1/invoice_processing": { + "count": 930, + "successful": 930, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 930, + "accuracy_all_rows_failures_incorrect": 0.9989247311827957, + "expected_accuracy_all_rows_failures_zero": 0.9989247311827957, + "coverage": [ + { + "threshold": 0.5, + "selected": 930, + "total": 930, + "coverage": 1.0 + }, + { + "threshold": 0.7, + "selected": 926, + "total": 930, + "coverage": 0.9956989247311828 + }, + { + "threshold": 0.8, + "selected": 924, + "total": 930, + "coverage": 0.9935483870967742 + }, + { + "threshold": 0.9, + "selected": 919, + "total": 930, + "coverage": 0.9881720430107527 + }, + { + "threshold": 0.95, + "selected": 902, + "total": 930, + "coverage": 0.9698924731182795 + }, + { + "threshold": 0.99, + "selected": 732, + "total": 930, + "coverage": 0.7870967741935484 + } + ], + "probability_metrics_all_rows": { + "count": 930, + "hard_count": 930, + "accuracy": 0.9989247311827957, + "expected_accuracy": 0.9989247311827957, + "nll": 0.010428173350270523, + "brier": 0.0031761278683533505, + "expected_brier": 0.0031761278683533475, + "ece_bins": 15, + "multiclass_ece": 0.009015233488700234, + "coverage": [ + { + "threshold": 0.5, + "selected": 930, + "coverage": 1.0, + "accuracy": 0.9989247311827957, + "expected_accuracy": 0.9989247311827957, + "expected_risk": 0.0010752688172043223 + }, + { + "threshold": 0.7, + "selected": 926, + "coverage": 0.9956989247311828, + "accuracy": 0.9989200863930886, + "expected_accuracy": 0.9989200863930886, + "expected_risk": 0.001079913606911398 + }, + { + "threshold": 0.8, + "selected": 924, + "coverage": 0.9935483870967742, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 919, + "coverage": 0.9881720430107527, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 902, + "coverage": 0.9698924731182795, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 732, + "coverage": 0.7870967741935484, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 930, + "hard_count": 930, + "accuracy": 0.9989247311827957, + "expected_accuracy": 0.9989247311827957, + "nll": 0.010428173350270523, + "brier": 0.0031761278683533505, + "expected_brier": 0.0031761278683533475, + "ece_bins": 15, + "multiclass_ece": 0.009015233488700234, + "coverage": [ + { + "threshold": 0.5, + "selected": 930, + "coverage": 1.0, + "accuracy": 0.9989247311827957, + "expected_accuracy": 0.9989247311827957, + "expected_risk": 0.0010752688172043223 + }, + { + "threshold": 0.7, + "selected": 926, + "coverage": 0.9956989247311828, + "accuracy": 0.9989200863930886, + "expected_accuracy": 0.9989200863930886, + "expected_risk": 0.001079913606911398 + }, + { + "threshold": 0.8, + "selected": 924, + "coverage": 0.9935483870967742, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 919, + "coverage": 0.9881720430107527, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 902, + "coverage": 0.9698924731182795, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 732, + "coverage": 0.7870967741935484, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "workflow-controls-v1/security_incidents": { + "count": 1326, + "successful": 1326, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 1326, + "accuracy_all_rows_failures_incorrect": 0.9962292609351433, + "expected_accuracy_all_rows_failures_zero": 0.9962292609351433, + "coverage": [ + { + "threshold": 0.5, + "selected": 1326, + "total": 1326, + "coverage": 1.0 + }, + { + "threshold": 0.7, + "selected": 1324, + "total": 1326, + "coverage": 0.9984917043740573 + }, + { + "threshold": 0.8, + "selected": 1323, + "total": 1326, + "coverage": 0.997737556561086 + }, + { + "threshold": 0.9, + "selected": 1322, + "total": 1326, + "coverage": 0.9969834087481146 + }, + { + "threshold": 0.95, + "selected": 1295, + "total": 1326, + "coverage": 0.9766214177978884 + }, + { + "threshold": 0.99, + "selected": 1228, + "total": 1326, + "coverage": 0.9260935143288085 + } + ], + "probability_metrics_all_rows": { + "count": 1326, + "hard_count": 1326, + "accuracy": 0.9962292609351433, + "expected_accuracy": 0.9962292609351433, + "nll": 0.014165561745883562, + "brier": 0.0066243147587275175, + "expected_brier": 0.006624314758727516, + "ece_bins": 15, + "multiclass_ece": 0.001806122714016116, + "coverage": [ + { + "threshold": 0.5, + "selected": 1326, + "coverage": 1.0, + "accuracy": 0.9962292609351433, + "expected_accuracy": 0.9962292609351433, + "expected_risk": 0.0037707390648566985 + }, + { + "threshold": 0.7, + "selected": 1324, + "coverage": 0.9984917043740573, + "accuracy": 0.9969788519637462, + "expected_accuracy": 0.9969788519637462, + "expected_risk": 0.0030211480362537513 + }, + { + "threshold": 0.8, + "selected": 1323, + "coverage": 0.997737556561086, + "accuracy": 0.9969765684051398, + "expected_accuracy": 0.9969765684051398, + "expected_risk": 0.003023431594860182 + }, + { + "threshold": 0.9, + "selected": 1322, + "coverage": 0.9969834087481146, + "accuracy": 0.9969742813918305, + "expected_accuracy": 0.9969742813918305, + "expected_risk": 0.00302571860816947 + }, + { + "threshold": 0.95, + "selected": 1295, + "coverage": 0.9766214177978884, + "accuracy": 0.9984555984555985, + "expected_accuracy": 0.9984555984555985, + "expected_risk": 0.001544401544401497 + }, + { + "threshold": 0.99, + "selected": 1228, + "coverage": 0.9260935143288085, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 1326, + "hard_count": 1326, + "accuracy": 0.9962292609351433, + "expected_accuracy": 0.9962292609351433, + "nll": 0.014165561745883562, + "brier": 0.0066243147587275175, + "expected_brier": 0.006624314758727516, + "ece_bins": 15, + "multiclass_ece": 0.001806122714016116, + "coverage": [ + { + "threshold": 0.5, + "selected": 1326, + "coverage": 1.0, + "accuracy": 0.9962292609351433, + "expected_accuracy": 0.9962292609351433, + "expected_risk": 0.0037707390648566985 + }, + { + "threshold": 0.7, + "selected": 1324, + "coverage": 0.9984917043740573, + "accuracy": 0.9969788519637462, + "expected_accuracy": 0.9969788519637462, + "expected_risk": 0.0030211480362537513 + }, + { + "threshold": 0.8, + "selected": 1323, + "coverage": 0.997737556561086, + "accuracy": 0.9969765684051398, + "expected_accuracy": 0.9969765684051398, + "expected_risk": 0.003023431594860182 + }, + { + "threshold": 0.9, + "selected": 1322, + "coverage": 0.9969834087481146, + "accuracy": 0.9969742813918305, + "expected_accuracy": 0.9969742813918305, + "expected_risk": 0.00302571860816947 + }, + { + "threshold": 0.95, + "selected": 1295, + "coverage": 0.9766214177978884, + "accuracy": 0.9984555984555985, + "expected_accuracy": 0.9984555984555985, + "expected_risk": 0.001544401544401497 + }, + { + "threshold": 0.99, + "selected": 1228, + "coverage": 0.9260935143288085, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + } + }, + "by_kind": { + "choice": { + "count": 2584, + "successful": 2584, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 2119, + "accuracy_all_rows_failures_incorrect": 0.8744690891930156, + "expected_accuracy_all_rows_failures_zero": 0.7789374561626884, + "coverage": [ + { + "threshold": 0.5, + "selected": 1948, + "total": 2584, + "coverage": 0.7538699690402477 + }, + { + "threshold": 0.7, + "selected": 1703, + "total": 2584, + "coverage": 0.6590557275541795 + }, + { + "threshold": 0.8, + "selected": 1648, + "total": 2584, + "coverage": 0.6377708978328174 + }, + { + "threshold": 0.9, + "selected": 1550, + "total": 2584, + "coverage": 0.5998452012383901 + }, + { + "threshold": 0.95, + "selected": 1399, + "total": 2584, + "coverage": 0.5414086687306502 + }, + { + "threshold": 0.99, + "selected": 936, + "total": 2584, + "coverage": 0.3622291021671827 + } + ], + "probability_metrics_all_rows": { + "count": 2584, + "hard_count": 2119, + "accuracy": 0.8744690891930156, + "expected_accuracy": 0.7789374561626884, + "nll": 0.5182847492916147, + "brier": 0.1546043322635334, + "expected_brier": 0.26454992932213356, + "ece_bins": 15, + "multiclass_ece": 0.02196788032076633, + "coverage": [ + { + "threshold": 0.5, + "selected": 1948, + "coverage": 0.7538699690402477, + "accuracy": 0.9578480310593456, + "expected_accuracy": 0.9223134839151267, + "expected_risk": 0.07768651608487331 + }, + { + "threshold": 0.7, + "selected": 1703, + "coverage": 0.6590557275541795, + "accuracy": 0.9815146094215862, + "expected_accuracy": 0.9736739087884126, + "expected_risk": 0.026326091211587355 + }, + { + "threshold": 0.8, + "selected": 1648, + "coverage": 0.6377708978328174, + "accuracy": 0.9852670349907919, + "expected_accuracy": 0.9792677993527508, + "expected_risk": 0.020732200647249233 + }, + { + "threshold": 0.9, + "selected": 1550, + "coverage": 0.5998452012383901, + "accuracy": 0.9922380336351876, + "expected_accuracy": 0.9909677419354839, + "expected_risk": 0.00903225806451613 + }, + { + "threshold": 0.95, + "selected": 1399, + "coverage": 0.5414086687306502, + "accuracy": 0.9964260185847034, + "expected_accuracy": 0.9964260185847034, + "expected_risk": 0.0035739814152966343 + }, + { + "threshold": 0.99, + "selected": 936, + "coverage": 0.3622291021671827, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 2584, + "hard_count": 2119, + "accuracy": 0.8744690891930156, + "expected_accuracy": 0.7789374561626884, + "nll": 0.5182847492916147, + "brier": 0.1546043322635334, + "expected_brier": 0.26454992932213356, + "ece_bins": 15, + "multiclass_ece": 0.02196788032076633, + "coverage": [ + { + "threshold": 0.5, + "selected": 1948, + "coverage": 0.7538699690402477, + "accuracy": 0.9578480310593456, + "expected_accuracy": 0.9223134839151267, + "expected_risk": 0.07768651608487331 + }, + { + "threshold": 0.7, + "selected": 1703, + "coverage": 0.6590557275541795, + "accuracy": 0.9815146094215862, + "expected_accuracy": 0.9736739087884126, + "expected_risk": 0.026326091211587355 + }, + { + "threshold": 0.8, + "selected": 1648, + "coverage": 0.6377708978328174, + "accuracy": 0.9852670349907919, + "expected_accuracy": 0.9792677993527508, + "expected_risk": 0.020732200647249233 + }, + { + "threshold": 0.9, + "selected": 1550, + "coverage": 0.5998452012383901, + "accuracy": 0.9922380336351876, + "expected_accuracy": 0.9909677419354839, + "expected_risk": 0.00903225806451613 + }, + { + "threshold": 0.95, + "selected": 1399, + "coverage": 0.5414086687306502, + "accuracy": 0.9964260185847034, + "expected_accuracy": 0.9964260185847034, + "expected_risk": 0.0035739814152966343 + }, + { + "threshold": 0.99, + "selected": 936, + "coverage": 0.3622291021671827, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "noul": { + "count": 6364, + "successful": 6364, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 6364, + "accuracy_all_rows_failures_incorrect": 0.9693588937774984, + "expected_accuracy_all_rows_failures_zero": 0.9693588937774984, + "coverage": [ + { + "threshold": 0.5, + "selected": 6364, + "total": 6364, + "coverage": 1.0 + }, + { + "threshold": 0.7, + "selected": 6302, + "total": 6364, + "coverage": 0.9902576995600252 + }, + { + "threshold": 0.8, + "selected": 6255, + "total": 6364, + "coverage": 0.982872407291012 + }, + { + "threshold": 0.9, + "selected": 6091, + "total": 6364, + "coverage": 0.9571024512884978 + }, + { + "threshold": 0.95, + "selected": 5801, + "total": 6364, + "coverage": 0.9115336266499057 + }, + { + "threshold": 0.99, + "selected": 4545, + "total": 6364, + "coverage": 0.7141734758013828 + } + ], + "probability_metrics_all_rows": { + "count": 6364, + "hard_count": 6364, + "accuracy": 0.9693588937774984, + "expected_accuracy": 0.9693588937774984, + "nll": 0.08299025396923065, + "brier": 0.04881816041081464, + "expected_brier": 0.04881816041081464, + "ece_bins": 15, + "multiclass_ece": 0.01363401934594279, + "coverage": [ + { + "threshold": 0.5, + "selected": 6364, + "coverage": 1.0, + "accuracy": 0.9693588937774984, + "expected_accuracy": 0.9693588937774984, + "expected_risk": 0.030641106222501602 + }, + { + "threshold": 0.7, + "selected": 6302, + "coverage": 0.9902576995600252, + "accuracy": 0.973341796255157, + "expected_accuracy": 0.973341796255157, + "expected_risk": 0.026658203744842957 + }, + { + "threshold": 0.8, + "selected": 6255, + "coverage": 0.982872407291012, + "accuracy": 0.9761790567545964, + "expected_accuracy": 0.9761790567545964, + "expected_risk": 0.023820943245403625 + }, + { + "threshold": 0.9, + "selected": 6091, + "coverage": 0.9571024512884978, + "accuracy": 0.9860449844032179, + "expected_accuracy": 0.9860449844032179, + "expected_risk": 0.013955015596782139 + }, + { + "threshold": 0.95, + "selected": 5801, + "coverage": 0.9115336266499057, + "accuracy": 0.9937941734183762, + "expected_accuracy": 0.9937941734183762, + "expected_risk": 0.00620582658162383 + }, + { + "threshold": 0.99, + "selected": 4545, + "coverage": 0.7141734758013828, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 6364, + "hard_count": 6364, + "accuracy": 0.9693588937774984, + "expected_accuracy": 0.9693588937774984, + "nll": 0.08299025396923065, + "brier": 0.04881816041081464, + "expected_brier": 0.04881816041081464, + "ece_bins": 15, + "multiclass_ece": 0.01363401934594279, + "coverage": [ + { + "threshold": 0.5, + "selected": 6364, + "coverage": 1.0, + "accuracy": 0.9693588937774984, + "expected_accuracy": 0.9693588937774984, + "expected_risk": 0.030641106222501602 + }, + { + "threshold": 0.7, + "selected": 6302, + "coverage": 0.9902576995600252, + "accuracy": 0.973341796255157, + "expected_accuracy": 0.973341796255157, + "expected_risk": 0.026658203744842957 + }, + { + "threshold": 0.8, + "selected": 6255, + "coverage": 0.982872407291012, + "accuracy": 0.9761790567545964, + "expected_accuracy": 0.9761790567545964, + "expected_risk": 0.023820943245403625 + }, + { + "threshold": 0.9, + "selected": 6091, + "coverage": 0.9571024512884978, + "accuracy": 0.9860449844032179, + "expected_accuracy": 0.9860449844032179, + "expected_risk": 0.013955015596782139 + }, + { + "threshold": 0.95, + "selected": 5801, + "coverage": 0.9115336266499057, + "accuracy": 0.9937941734183762, + "expected_accuracy": 0.9937941734183762, + "expected_risk": 0.00620582658162383 + }, + { + "threshold": 0.99, + "selected": 4545, + "coverage": 0.7141734758013828, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "score": { + "count": 1584, + "successful": 1584, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 1563, + "accuracy_all_rows_failures_incorrect": 0.9552143314139475, + "expected_accuracy_all_rows_failures_zero": 0.9486531986531987, + "coverage": [ + { + "threshold": 0.5, + "selected": 1518, + "total": 1584, + "coverage": 0.9583333333333334 + }, + { + "threshold": 0.7, + "selected": 1495, + "total": 1584, + "coverage": 0.9438131313131313 + }, + { + "threshold": 0.8, + "selected": 1485, + "total": 1584, + "coverage": 0.9375 + }, + { + "threshold": 0.9, + "selected": 1475, + "total": 1584, + "coverage": 0.9311868686868687 + }, + { + "threshold": 0.95, + "selected": 1434, + "total": 1584, + "coverage": 0.9053030303030303 + }, + { + "threshold": 0.99, + "selected": 1261, + "total": 1584, + "coverage": 0.7960858585858586 + } + ], + "probability_metrics_all_rows": { + "count": 1584, + "hard_count": 1563, + "accuracy": 0.9552143314139475, + "expected_accuracy": 0.9486531986531987, + "nll": 0.1201671405517328, + "brier": 0.05775733534507227, + "expected_brier": 0.06491221749995442, + "ece_bins": 15, + "multiclass_ece": 0.014709357196524655, + "coverage": [ + { + "threshold": 0.5, + "selected": 1518, + "coverage": 0.9583333333333334, + "accuracy": 0.978709248170326, + "expected_accuracy": 0.9739789196310935, + "expected_risk": 0.02602108036890649 + }, + { + "threshold": 0.7, + "selected": 1495, + "coverage": 0.9438131313131313, + "accuracy": 0.9832775919732442, + "expected_accuracy": 0.9832775919732442, + "expected_risk": 0.01672240802675584 + }, + { + "threshold": 0.8, + "selected": 1485, + "coverage": 0.9375, + "accuracy": 0.9878787878787879, + "expected_accuracy": 0.9878787878787879, + "expected_risk": 0.012121212121212088 + }, + { + "threshold": 0.9, + "selected": 1475, + "coverage": 0.9311868686868687, + "accuracy": 0.991864406779661, + "expected_accuracy": 0.991864406779661, + "expected_risk": 0.008135593220338966 + }, + { + "threshold": 0.95, + "selected": 1434, + "coverage": 0.9053030303030303, + "accuracy": 0.99860529986053, + "expected_accuracy": 0.99860529986053, + "expected_risk": 0.001394700139469962 + }, + { + "threshold": 0.99, + "selected": 1261, + "coverage": 0.7960858585858586, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 1584, + "hard_count": 1563, + "accuracy": 0.9552143314139475, + "expected_accuracy": 0.9486531986531987, + "nll": 0.1201671405517328, + "brier": 0.05775733534507227, + "expected_brier": 0.06491221749995442, + "ece_bins": 15, + "multiclass_ece": 0.014709357196524655, + "coverage": [ + { + "threshold": 0.5, + "selected": 1518, + "coverage": 0.9583333333333334, + "accuracy": 0.978709248170326, + "expected_accuracy": 0.9739789196310935, + "expected_risk": 0.02602108036890649 + }, + { + "threshold": 0.7, + "selected": 1495, + "coverage": 0.9438131313131313, + "accuracy": 0.9832775919732442, + "expected_accuracy": 0.9832775919732442, + "expected_risk": 0.01672240802675584 + }, + { + "threshold": 0.8, + "selected": 1485, + "coverage": 0.9375, + "accuracy": 0.9878787878787879, + "expected_accuracy": 0.9878787878787879, + "expected_risk": 0.012121212121212088 + }, + { + "threshold": 0.9, + "selected": 1475, + "coverage": 0.9311868686868687, + "accuracy": 0.991864406779661, + "expected_accuracy": 0.991864406779661, + "expected_risk": 0.008135593220338966 + }, + { + "threshold": 0.95, + "selected": 1434, + "coverage": 0.9053030303030303, + "accuracy": 0.99860529986053, + "expected_accuracy": 0.99860529986053, + "expected_risk": 0.001394700139469962 + }, + { + "threshold": 0.99, + "selected": 1261, + "coverage": 0.7960858585858586, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + } + } + }, + "ood": { + "count": 15920, + "successful": 15920, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 15446, + "accuracy_all_rows_failures_incorrect": 0.8602227113815875, + "expected_accuracy_all_rows_failures_zero": 0.8461552917002038, + "coverage": [ + { + "threshold": 0.5, + "selected": 15467, + "total": 15920, + "coverage": 0.9715452261306533 + }, + { + "threshold": 0.7, + "selected": 14974, + "total": 15920, + "coverage": 0.9405778894472362 + }, + { + "threshold": 0.8, + "selected": 14755, + "total": 15920, + "coverage": 0.926821608040201 + }, + { + "threshold": 0.9, + "selected": 14212, + "total": 15920, + "coverage": 0.892713567839196 + }, + { + "threshold": 0.95, + "selected": 13285, + "total": 15920, + "coverage": 0.8344849246231156 + }, + { + "threshold": 0.99, + "selected": 9996, + "total": 15920, + "coverage": 0.6278894472361809 + } + ], + "probability_metrics_all_rows": { + "count": 15920, + "hard_count": 15446, + "accuracy": 0.8602227113815875, + "expected_accuracy": 0.8461552917002038, + "nll": 0.8029899145502479, + "brier": 0.24645039628025225, + "expected_brier": 0.26378113592726077, + "ece_bins": 15, + "multiclass_ece": 0.10559429485648715, + "coverage": [ + { + "threshold": 0.5, + "selected": 15467, + "coverage": 0.9715452261306533, + "accuracy": 0.8670391061452514, + "expected_accuracy": 0.8610051507510613, + "expected_risk": 0.13899484924893868 + }, + { + "threshold": 0.7, + "selected": 14974, + "coverage": 0.9405778894472362, + "accuracy": 0.8734346748811357, + "expected_accuracy": 0.8723899203063087, + "expected_risk": 0.12761007969369131 + }, + { + "threshold": 0.8, + "selected": 14755, + "coverage": 0.926821608040201, + "accuracy": 0.8785141925845443, + "expected_accuracy": 0.8777589517677624, + "expected_risk": 0.12224104823223758 + }, + { + "threshold": 0.9, + "selected": 14212, + "coverage": 0.892713567839196, + "accuracy": 0.886173135169402, + "expected_accuracy": 0.8857655502392344, + "expected_risk": 0.11423444976076558 + }, + { + "threshold": 0.95, + "selected": 13285, + "coverage": 0.8344849246231156, + "accuracy": 0.8918918918918919, + "expected_accuracy": 0.8918328942416259, + "expected_risk": 0.10816710575837407 + }, + { + "threshold": 0.99, + "selected": 9996, + "coverage": 0.6278894472361809, + "accuracy": 0.8904561824729892, + "expected_accuracy": 0.8904561824729892, + "expected_risk": 0.10954381752701081 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 15920, + "hard_count": 15446, + "accuracy": 0.8602227113815875, + "expected_accuracy": 0.8461552917002038, + "nll": 0.8029899145502479, + "brier": 0.24645039628025225, + "expected_brier": 0.26378113592726077, + "ece_bins": 15, + "multiclass_ece": 0.10559429485648715, + "coverage": [ + { + "threshold": 0.5, + "selected": 15467, + "coverage": 0.9715452261306533, + "accuracy": 0.8670391061452514, + "expected_accuracy": 0.8610051507510613, + "expected_risk": 0.13899484924893868 + }, + { + "threshold": 0.7, + "selected": 14974, + "coverage": 0.9405778894472362, + "accuracy": 0.8734346748811357, + "expected_accuracy": 0.8723899203063087, + "expected_risk": 0.12761007969369131 + }, + { + "threshold": 0.8, + "selected": 14755, + "coverage": 0.926821608040201, + "accuracy": 0.8785141925845443, + "expected_accuracy": 0.8777589517677624, + "expected_risk": 0.12224104823223758 + }, + { + "threshold": 0.9, + "selected": 14212, + "coverage": 0.892713567839196, + "accuracy": 0.886173135169402, + "expected_accuracy": 0.8857655502392344, + "expected_risk": 0.11423444976076558 + }, + { + "threshold": 0.95, + "selected": 13285, + "coverage": 0.8344849246231156, + "accuracy": 0.8918918918918919, + "expected_accuracy": 0.8918328942416259, + "expected_risk": 0.10816710575837407 + }, + { + "threshold": 0.99, + "selected": 9996, + "coverage": 0.6278894472361809, + "accuracy": 0.8904561824729892, + "expected_accuracy": 0.8904561824729892, + "expected_risk": 0.10954381752701081 + } + ] + }, + "by_source": { + "customer-control-v1": { + "count": 600, + "successful": 600, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 560, + "accuracy_all_rows_failures_incorrect": 0.9732142857142857, + "expected_accuracy_all_rows_failures_zero": 0.9383333333333334, + "coverage": [ + { + "threshold": 0.5, + "selected": 585, + "total": 600, + "coverage": 0.975 + }, + { + "threshold": 0.7, + "selected": 550, + "total": 600, + "coverage": 0.9166666666666666 + }, + { + "threshold": 0.8, + "selected": 544, + "total": 600, + "coverage": 0.9066666666666666 + }, + { + "threshold": 0.9, + "selected": 534, + "total": 600, + "coverage": 0.89 + }, + { + "threshold": 0.95, + "selected": 516, + "total": 600, + "coverage": 0.86 + }, + { + "threshold": 0.99, + "selected": 439, + "total": 600, + "coverage": 0.7316666666666667 + } + ], + "probability_metrics_all_rows": { + "count": 600, + "hard_count": 560, + "accuracy": 0.9732142857142857, + "expected_accuracy": 0.9383333333333334, + "nll": 0.18931729447777135, + "brier": 0.053790307653173525, + "expected_brier": 0.09045697431984019, + "ece_bins": 15, + "multiclass_ece": 0.02661442402817384, + "coverage": [ + { + "threshold": 0.5, + "selected": 585, + "coverage": 0.975, + "accuracy": 0.9731663685152058, + "expected_accuracy": 0.9521367521367521, + "expected_risk": 0.047863247863247915 + }, + { + "threshold": 0.7, + "selected": 550, + "coverage": 0.9166666666666666, + "accuracy": 0.9744058500914077, + "expected_accuracy": 0.9718181818181818, + "expected_risk": 0.028181818181818197 + }, + { + "threshold": 0.8, + "selected": 544, + "coverage": 0.9066666666666666, + "accuracy": 0.9760589318600368, + "expected_accuracy": 0.9751838235294118, + "expected_risk": 0.024816176470588203 + }, + { + "threshold": 0.9, + "selected": 534, + "coverage": 0.89, + "accuracy": 0.9756554307116105, + "expected_accuracy": 0.9756554307116105, + "expected_risk": 0.02434456928838946 + }, + { + "threshold": 0.95, + "selected": 516, + "coverage": 0.86, + "accuracy": 0.9786821705426356, + "expected_accuracy": 0.9786821705426356, + "expected_risk": 0.02131782945736438 + }, + { + "threshold": 0.99, + "selected": 439, + "coverage": 0.7316666666666667, + "accuracy": 0.9840546697038725, + "expected_accuracy": 0.9840546697038725, + "expected_risk": 0.015945330296127547 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 600, + "hard_count": 560, + "accuracy": 0.9732142857142857, + "expected_accuracy": 0.9383333333333334, + "nll": 0.18931729447777135, + "brier": 0.053790307653173525, + "expected_brier": 0.09045697431984019, + "ece_bins": 15, + "multiclass_ece": 0.02661442402817384, + "coverage": [ + { + "threshold": 0.5, + "selected": 585, + "coverage": 0.975, + "accuracy": 0.9731663685152058, + "expected_accuracy": 0.9521367521367521, + "expected_risk": 0.047863247863247915 + }, + { + "threshold": 0.7, + "selected": 550, + "coverage": 0.9166666666666666, + "accuracy": 0.9744058500914077, + "expected_accuracy": 0.9718181818181818, + "expected_risk": 0.028181818181818197 + }, + { + "threshold": 0.8, + "selected": 544, + "coverage": 0.9066666666666666, + "accuracy": 0.9760589318600368, + "expected_accuracy": 0.9751838235294118, + "expected_risk": 0.024816176470588203 + }, + { + "threshold": 0.9, + "selected": 534, + "coverage": 0.89, + "accuracy": 0.9756554307116105, + "expected_accuracy": 0.9756554307116105, + "expected_risk": 0.02434456928838946 + }, + { + "threshold": 0.95, + "selected": 516, + "coverage": 0.86, + "accuracy": 0.9786821705426356, + "expected_accuracy": 0.9786821705426356, + "expected_risk": 0.02131782945736438 + }, + { + "threshold": 0.99, + "selected": 439, + "coverage": 0.7316666666666667, + "accuracy": 0.9840546697038725, + "expected_accuracy": 0.9840546697038725, + "expected_risk": 0.015945330296127547 + } + ] + } + }, + "painting-geometry-v1": { + "count": 6912, + "successful": 6912, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 6912, + "accuracy_all_rows_failures_incorrect": 0.8051215277777778, + "expected_accuracy_all_rows_failures_zero": 0.8051215277777778, + "coverage": [ + { + "threshold": 0.5, + "selected": 6908, + "total": 6912, + "coverage": 0.9994212962962963 + }, + { + "threshold": 0.7, + "selected": 6876, + "total": 6912, + "coverage": 0.9947916666666666 + }, + { + "threshold": 0.8, + "selected": 6845, + "total": 6912, + "coverage": 0.9903067129629629 + }, + { + "threshold": 0.9, + "selected": 6734, + "total": 6912, + "coverage": 0.9742476851851852 + }, + { + "threshold": 0.95, + "selected": 6551, + "total": 6912, + "coverage": 0.9477719907407407 + }, + { + "threshold": 0.99, + "selected": 5718, + "total": 6912, + "coverage": 0.8272569444444444 + } + ], + "probability_metrics_all_rows": { + "count": 6912, + "hard_count": 6912, + "accuracy": 0.8051215277777778, + "expected_accuracy": 0.8051215277777778, + "nll": 1.4325737213947278, + "brier": 0.3808534813920613, + "expected_brier": 0.3808534813920613, + "ece_bins": 15, + "multiclass_ece": 0.18400414817622343, + "coverage": [ + { + "threshold": 0.5, + "selected": 6908, + "coverage": 0.9994212962962963, + "accuracy": 0.8051534452808338, + "expected_accuracy": 0.8051534452808338, + "expected_risk": 0.19484655471916623 + }, + { + "threshold": 0.7, + "selected": 6876, + "coverage": 0.9947916666666666, + "accuracy": 0.806282722513089, + "expected_accuracy": 0.806282722513089, + "expected_risk": 0.19371727748691103 + }, + { + "threshold": 0.8, + "selected": 6845, + "coverage": 0.9903067129629629, + "accuracy": 0.8081811541271001, + "expected_accuracy": 0.8081811541271001, + "expected_risk": 0.19181884587289988 + }, + { + "threshold": 0.9, + "selected": 6734, + "coverage": 0.9742476851851852, + "accuracy": 0.8091773091773091, + "expected_accuracy": 0.8091773091773091, + "expected_risk": 0.19082269082269088 + }, + { + "threshold": 0.95, + "selected": 6551, + "coverage": 0.9477719907407407, + "accuracy": 0.8087314913753625, + "expected_accuracy": 0.8087314913753625, + "expected_risk": 0.1912685086246375 + }, + { + "threshold": 0.99, + "selected": 5718, + "coverage": 0.8272569444444444, + "accuracy": 0.8121720881427072, + "expected_accuracy": 0.8121720881427072, + "expected_risk": 0.18782791185729275 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 6912, + "hard_count": 6912, + "accuracy": 0.8051215277777778, + "expected_accuracy": 0.8051215277777778, + "nll": 1.4325737213947278, + "brier": 0.3808534813920613, + "expected_brier": 0.3808534813920613, + "ece_bins": 15, + "multiclass_ece": 0.18400414817622343, + "coverage": [ + { + "threshold": 0.5, + "selected": 6908, + "coverage": 0.9994212962962963, + "accuracy": 0.8051534452808338, + "expected_accuracy": 0.8051534452808338, + "expected_risk": 0.19484655471916623 + }, + { + "threshold": 0.7, + "selected": 6876, + "coverage": 0.9947916666666666, + "accuracy": 0.806282722513089, + "expected_accuracy": 0.806282722513089, + "expected_risk": 0.19371727748691103 + }, + { + "threshold": 0.8, + "selected": 6845, + "coverage": 0.9903067129629629, + "accuracy": 0.8081811541271001, + "expected_accuracy": 0.8081811541271001, + "expected_risk": 0.19181884587289988 + }, + { + "threshold": 0.9, + "selected": 6734, + "coverage": 0.9742476851851852, + "accuracy": 0.8091773091773091, + "expected_accuracy": 0.8091773091773091, + "expected_risk": 0.19082269082269088 + }, + { + "threshold": 0.95, + "selected": 6551, + "coverage": 0.9477719907407407, + "accuracy": 0.8087314913753625, + "expected_accuracy": 0.8087314913753625, + "expected_risk": 0.1912685086246375 + }, + { + "threshold": 0.99, + "selected": 5718, + "coverage": 0.8272569444444444, + "accuracy": 0.8121720881427072, + "expected_accuracy": 0.8121720881427072, + "expected_risk": 0.18782791185729275 + } + ] + } + }, + "reasoning-control-v1": { + "count": 750, + "successful": 750, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 750, + "accuracy_all_rows_failures_incorrect": 0.6693333333333333, + "expected_accuracy_all_rows_failures_zero": 0.6693333333333333, + "coverage": [ + { + "threshold": 0.5, + "selected": 617, + "total": 750, + "coverage": 0.8226666666666667 + }, + { + "threshold": 0.7, + "selected": 533, + "total": 750, + "coverage": 0.7106666666666667 + }, + { + "threshold": 0.8, + "selected": 468, + "total": 750, + "coverage": 0.624 + }, + { + "threshold": 0.9, + "selected": 375, + "total": 750, + "coverage": 0.5 + }, + { + "threshold": 0.95, + "selected": 268, + "total": 750, + "coverage": 0.35733333333333334 + }, + { + "threshold": 0.99, + "selected": 113, + "total": 750, + "coverage": 0.15066666666666667 + } + ], + "probability_metrics_all_rows": { + "count": 750, + "hard_count": 750, + "accuracy": 0.6693333333333333, + "expected_accuracy": 0.6693333333333333, + "nll": 0.8174632190648329, + "brier": 0.4497004885456776, + "expected_brier": 0.4497004885456775, + "ece_bins": 15, + "multiclass_ece": 0.12814111983806595, + "coverage": [ + { + "threshold": 0.5, + "selected": 617, + "coverage": 0.8226666666666667, + "accuracy": 0.7439222042139384, + "expected_accuracy": 0.7439222042139384, + "expected_risk": 0.2560777957860616 + }, + { + "threshold": 0.7, + "selected": 533, + "coverage": 0.7106666666666667, + "accuracy": 0.774859287054409, + "expected_accuracy": 0.774859287054409, + "expected_risk": 0.22514071294559101 + }, + { + "threshold": 0.8, + "selected": 468, + "coverage": 0.624, + "accuracy": 0.8141025641025641, + "expected_accuracy": 0.8141025641025641, + "expected_risk": 0.1858974358974359 + }, + { + "threshold": 0.9, + "selected": 375, + "coverage": 0.5, + "accuracy": 0.856, + "expected_accuracy": 0.856, + "expected_risk": 0.14400000000000002 + }, + { + "threshold": 0.95, + "selected": 268, + "coverage": 0.35733333333333334, + "accuracy": 0.9104477611940298, + "expected_accuracy": 0.9104477611940298, + "expected_risk": 0.08955223880597019 + }, + { + "threshold": 0.99, + "selected": 113, + "coverage": 0.15066666666666667, + "accuracy": 0.9557522123893806, + "expected_accuracy": 0.9557522123893806, + "expected_risk": 0.04424778761061943 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 750, + "hard_count": 750, + "accuracy": 0.6693333333333333, + "expected_accuracy": 0.6693333333333333, + "nll": 0.8174632190648329, + "brier": 0.4497004885456776, + "expected_brier": 0.4497004885456775, + "ece_bins": 15, + "multiclass_ece": 0.12814111983806595, + "coverage": [ + { + "threshold": 0.5, + "selected": 617, + "coverage": 0.8226666666666667, + "accuracy": 0.7439222042139384, + "expected_accuracy": 0.7439222042139384, + "expected_risk": 0.2560777957860616 + }, + { + "threshold": 0.7, + "selected": 533, + "coverage": 0.7106666666666667, + "accuracy": 0.774859287054409, + "expected_accuracy": 0.774859287054409, + "expected_risk": 0.22514071294559101 + }, + { + "threshold": 0.8, + "selected": 468, + "coverage": 0.624, + "accuracy": 0.8141025641025641, + "expected_accuracy": 0.8141025641025641, + "expected_risk": 0.1858974358974359 + }, + { + "threshold": 0.9, + "selected": 375, + "coverage": 0.5, + "accuracy": 0.856, + "expected_accuracy": 0.856, + "expected_risk": 0.14400000000000002 + }, + { + "threshold": 0.95, + "selected": 268, + "coverage": 0.35733333333333334, + "accuracy": 0.9104477611940298, + "expected_accuracy": 0.9104477611940298, + "expected_risk": 0.08955223880597019 + }, + { + "threshold": 0.99, + "selected": 113, + "coverage": 0.15066666666666667, + "accuracy": 0.9557522123893806, + "expected_accuracy": 0.9557522123893806, + "expected_risk": 0.04424778761061943 + } + ] + } + }, + "snake-v1": { + "count": 3464, + "successful": 3464, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 3192, + "accuracy_all_rows_failures_incorrect": 0.868421052631579, + "expected_accuracy_all_rows_failures_zero": 0.8391070053887606, + "coverage": [ + { + "threshold": 0.5, + "selected": 3389, + "total": 3464, + "coverage": 0.9783487297921478 + }, + { + "threshold": 0.7, + "selected": 3093, + "total": 3464, + "coverage": 0.8928983833718245 + }, + { + "threshold": 0.8, + "selected": 3011, + "total": 3464, + "coverage": 0.8692263279445728 + }, + { + "threshold": 0.9, + "selected": 2735, + "total": 3464, + "coverage": 0.7895496535796767 + }, + { + "threshold": 0.95, + "selected": 2217, + "total": 3464, + "coverage": 0.6400115473441108 + }, + { + "threshold": 0.99, + "selected": 627, + "total": 3464, + "coverage": 0.18100461893764433 + } + ], + "probability_metrics_all_rows": { + "count": 3464, + "hard_count": 3192, + "accuracy": 0.868421052631579, + "expected_accuracy": 0.8391070053887606, + "nll": 0.40090561564728155, + "brier": 0.20086908723576663, + "expected_brier": 0.24022628508026242, + "ece_bins": 15, + "multiclass_ece": 0.07383729789301127, + "coverage": [ + { + "threshold": 0.5, + "selected": 3389, + "coverage": 0.9783487297921478, + "accuracy": 0.8714060031595576, + "expected_accuracy": 0.8466115865053604, + "expected_risk": 0.1533884134946396 + }, + { + "threshold": 0.7, + "selected": 3093, + "coverage": 0.8928983833718245, + "accuracy": 0.8851063829787233, + "expected_accuracy": 0.8802672701799762, + "expected_risk": 0.11973272982002381 + }, + { + "threshold": 0.8, + "selected": 3011, + "coverage": 0.8692263279445728, + "accuracy": 0.89171974522293, + "expected_accuracy": 0.8880216982176464, + "expected_risk": 0.11197830178235357 + }, + { + "threshold": 0.9, + "selected": 2735, + "coverage": 0.7895496535796767, + "accuracy": 0.9158088235294117, + "expected_accuracy": 0.913528336380256, + "expected_risk": 0.08647166361974401 + }, + { + "threshold": 0.95, + "selected": 2217, + "coverage": 0.6400115473441108, + "accuracy": 0.944018058690745, + "expected_accuracy": 0.94361750112765, + "expected_risk": 0.05638249887235003 + }, + { + "threshold": 0.99, + "selected": 627, + "coverage": 0.18100461893764433, + "accuracy": 0.9936204146730463, + "expected_accuracy": 0.9936204146730463, + "expected_risk": 0.006379585326953707 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 3464, + "hard_count": 3192, + "accuracy": 0.868421052631579, + "expected_accuracy": 0.8391070053887606, + "nll": 0.40090561564728155, + "brier": 0.20086908723576663, + "expected_brier": 0.24022628508026242, + "ece_bins": 15, + "multiclass_ece": 0.07383729789301127, + "coverage": [ + { + "threshold": 0.5, + "selected": 3389, + "coverage": 0.9783487297921478, + "accuracy": 0.8714060031595576, + "expected_accuracy": 0.8466115865053604, + "expected_risk": 0.1533884134946396 + }, + { + "threshold": 0.7, + "selected": 3093, + "coverage": 0.8928983833718245, + "accuracy": 0.8851063829787233, + "expected_accuracy": 0.8802672701799762, + "expected_risk": 0.11973272982002381 + }, + { + "threshold": 0.8, + "selected": 3011, + "coverage": 0.8692263279445728, + "accuracy": 0.89171974522293, + "expected_accuracy": 0.8880216982176464, + "expected_risk": 0.11197830178235357 + }, + { + "threshold": 0.9, + "selected": 2735, + "coverage": 0.7895496535796767, + "accuracy": 0.9158088235294117, + "expected_accuracy": 0.913528336380256, + "expected_risk": 0.08647166361974401 + }, + { + "threshold": 0.95, + "selected": 2217, + "coverage": 0.6400115473441108, + "accuracy": 0.944018058690745, + "expected_accuracy": 0.94361750112765, + "expected_risk": 0.05638249887235003 + }, + { + "threshold": 0.99, + "selected": 627, + "coverage": 0.18100461893764433, + "accuracy": 0.9936204146730463, + "expected_accuracy": 0.9936204146730463, + "expected_risk": 0.006379585326953707 + } + ] + } + }, + "vizdoom-basic-v1": { + "count": 825, + "successful": 825, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 825, + "accuracy_all_rows_failures_incorrect": 0.9309090909090909, + "expected_accuracy_all_rows_failures_zero": 0.9309090909090909, + "coverage": [ + { + "threshold": 0.5, + "selected": 816, + "total": 825, + "coverage": 0.9890909090909091 + }, + { + "threshold": 0.7, + "selected": 788, + "total": 825, + "coverage": 0.9551515151515152 + }, + { + "threshold": 0.8, + "selected": 765, + "total": 825, + "coverage": 0.9272727272727272 + }, + { + "threshold": 0.9, + "selected": 741, + "total": 825, + "coverage": 0.8981818181818182 + }, + { + "threshold": 0.95, + "selected": 693, + "total": 825, + "coverage": 0.84 + }, + { + "threshold": 0.99, + "selected": 457, + "total": 825, + "coverage": 0.553939393939394 + } + ], + "probability_metrics_all_rows": { + "count": 825, + "hard_count": 825, + "accuracy": 0.9309090909090909, + "expected_accuracy": 0.9309090909090909, + "nll": 0.18737228735728542, + "brier": 0.09844207618854224, + "expected_brier": 0.09844207618854224, + "ece_bins": 15, + "multiclass_ece": 0.0250802098487241, + "coverage": [ + { + "threshold": 0.5, + "selected": 816, + "coverage": 0.9890909090909091, + "accuracy": 0.9399509803921569, + "expected_accuracy": 0.9399509803921569, + "expected_risk": 0.060049019607843146 + }, + { + "threshold": 0.7, + "selected": 788, + "coverage": 0.9551515151515152, + "accuracy": 0.9593908629441624, + "expected_accuracy": 0.9593908629441624, + "expected_risk": 0.040609137055837574 + }, + { + "threshold": 0.8, + "selected": 765, + "coverage": 0.9272727272727272, + "accuracy": 0.9751633986928104, + "expected_accuracy": 0.9751633986928104, + "expected_risk": 0.024836601307189565 + }, + { + "threshold": 0.9, + "selected": 741, + "coverage": 0.8981818181818182, + "accuracy": 0.9824561403508771, + "expected_accuracy": 0.9824561403508771, + "expected_risk": 0.01754385964912286 + }, + { + "threshold": 0.95, + "selected": 693, + "coverage": 0.84, + "accuracy": 0.9884559884559885, + "expected_accuracy": 0.9884559884559885, + "expected_risk": 0.011544011544011523 + }, + { + "threshold": 0.99, + "selected": 457, + "coverage": 0.553939393939394, + "accuracy": 0.9978118161925602, + "expected_accuracy": 0.9978118161925602, + "expected_risk": 0.002188183807439792 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 825, + "hard_count": 825, + "accuracy": 0.9309090909090909, + "expected_accuracy": 0.9309090909090909, + "nll": 0.18737228735728542, + "brier": 0.09844207618854224, + "expected_brier": 0.09844207618854224, + "ece_bins": 15, + "multiclass_ece": 0.0250802098487241, + "coverage": [ + { + "threshold": 0.5, + "selected": 816, + "coverage": 0.9890909090909091, + "accuracy": 0.9399509803921569, + "expected_accuracy": 0.9399509803921569, + "expected_risk": 0.060049019607843146 + }, + { + "threshold": 0.7, + "selected": 788, + "coverage": 0.9551515151515152, + "accuracy": 0.9593908629441624, + "expected_accuracy": 0.9593908629441624, + "expected_risk": 0.040609137055837574 + }, + { + "threshold": 0.8, + "selected": 765, + "coverage": 0.9272727272727272, + "accuracy": 0.9751633986928104, + "expected_accuracy": 0.9751633986928104, + "expected_risk": 0.024836601307189565 + }, + { + "threshold": 0.9, + "selected": 741, + "coverage": 0.8981818181818182, + "accuracy": 0.9824561403508771, + "expected_accuracy": 0.9824561403508771, + "expected_risk": 0.01754385964912286 + }, + { + "threshold": 0.95, + "selected": 693, + "coverage": 0.84, + "accuracy": 0.9884559884559885, + "expected_accuracy": 0.9884559884559885, + "expected_risk": 0.011544011544011523 + }, + { + "threshold": 0.99, + "selected": 457, + "coverage": 0.553939393939394, + "accuracy": 0.9978118161925602, + "expected_accuracy": 0.9978118161925602, + "expected_risk": 0.002188183807439792 + } + ] + } + }, + "wikispeedia-v1": { + "count": 219, + "successful": 219, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 57, + "accuracy_all_rows_failures_incorrect": 0.5789473684210527, + "expected_accuracy_all_rows_failures_zero": 0.29281085479715613, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "total": 219, + "coverage": 0.0091324200913242 + }, + { + "threshold": 0.7, + "selected": 0, + "total": 219, + "coverage": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "total": 219, + "coverage": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "total": 219, + "coverage": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "total": 219, + "coverage": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "total": 219, + "coverage": 0.0 + } + ], + "probability_metrics_all_rows": { + "count": 219, + "hard_count": 57, + "accuracy": 0.5789473684210527, + "expected_accuracy": 0.29281085479715613, + "nll": 2.1593282501811903, + "brier": 0.3142609630528501, + "expected_brier": 0.8511196016238175, + "ece_bins": 15, + "multiclass_ece": 0.1249962258708074, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 0.0091324200913242, + "accuracy": null, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 219, + "hard_count": 57, + "accuracy": 0.5789473684210527, + "expected_accuracy": 0.29281085479715613, + "nll": 2.1593282501811903, + "brier": 0.3142609630528501, + "expected_brier": 0.8511196016238175, + "ece_bins": 15, + "multiclass_ece": 0.1249962258708074, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 0.0091324200913242, + "accuracy": null, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + } + }, + "workflow-controls-v1/agent_trace_observability": { + "count": 525, + "successful": 525, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 525, + "accuracy_all_rows_failures_incorrect": 0.9885714285714285, + "expected_accuracy_all_rows_failures_zero": 0.9885714285714285, + "coverage": [ + { + "threshold": 0.5, + "selected": 525, + "total": 525, + "coverage": 1.0 + }, + { + "threshold": 0.7, + "selected": 525, + "total": 525, + "coverage": 1.0 + }, + { + "threshold": 0.8, + "selected": 524, + "total": 525, + "coverage": 0.9980952380952381 + }, + { + "threshold": 0.9, + "selected": 520, + "total": 525, + "coverage": 0.9904761904761905 + }, + { + "threshold": 0.95, + "selected": 519, + "total": 525, + "coverage": 0.9885714285714285 + }, + { + "threshold": 0.99, + "selected": 438, + "total": 525, + "coverage": 0.8342857142857143 + } + ], + "probability_metrics_all_rows": { + "count": 525, + "hard_count": 525, + "accuracy": 0.9885714285714285, + "expected_accuracy": 0.9885714285714285, + "nll": 0.032287053288972606, + "brier": 0.01822595812319716, + "expected_brier": 0.01822595812319716, + "ece_bins": 15, + "multiclass_ece": 0.005162135252793167, + "coverage": [ + { + "threshold": 0.5, + "selected": 525, + "coverage": 1.0, + "accuracy": 0.9885714285714285, + "expected_accuracy": 0.9885714285714285, + "expected_risk": 0.011428571428571455 + }, + { + "threshold": 0.7, + "selected": 525, + "coverage": 1.0, + "accuracy": 0.9885714285714285, + "expected_accuracy": 0.9885714285714285, + "expected_risk": 0.011428571428571455 + }, + { + "threshold": 0.8, + "selected": 524, + "coverage": 0.9980952380952381, + "accuracy": 0.9904580152671756, + "expected_accuracy": 0.9904580152671756, + "expected_risk": 0.009541984732824416 + }, + { + "threshold": 0.9, + "selected": 520, + "coverage": 0.9904761904761905, + "accuracy": 0.9942307692307693, + "expected_accuracy": 0.9942307692307693, + "expected_risk": 0.005769230769230749 + }, + { + "threshold": 0.95, + "selected": 519, + "coverage": 0.9885714285714285, + "accuracy": 0.9942196531791907, + "expected_accuracy": 0.9942196531791907, + "expected_risk": 0.005780346820809301 + }, + { + "threshold": 0.99, + "selected": 438, + "coverage": 0.8342857142857143, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 525, + "hard_count": 525, + "accuracy": 0.9885714285714285, + "expected_accuracy": 0.9885714285714285, + "nll": 0.032287053288972606, + "brier": 0.01822595812319716, + "expected_brier": 0.01822595812319716, + "ece_bins": 15, + "multiclass_ece": 0.005162135252793167, + "coverage": [ + { + "threshold": 0.5, + "selected": 525, + "coverage": 1.0, + "accuracy": 0.9885714285714285, + "expected_accuracy": 0.9885714285714285, + "expected_risk": 0.011428571428571455 + }, + { + "threshold": 0.7, + "selected": 525, + "coverage": 1.0, + "accuracy": 0.9885714285714285, + "expected_accuracy": 0.9885714285714285, + "expected_risk": 0.011428571428571455 + }, + { + "threshold": 0.8, + "selected": 524, + "coverage": 0.9980952380952381, + "accuracy": 0.9904580152671756, + "expected_accuracy": 0.9904580152671756, + "expected_risk": 0.009541984732824416 + }, + { + "threshold": 0.9, + "selected": 520, + "coverage": 0.9904761904761905, + "accuracy": 0.9942307692307693, + "expected_accuracy": 0.9942307692307693, + "expected_risk": 0.005769230769230749 + }, + { + "threshold": 0.95, + "selected": 519, + "coverage": 0.9885714285714285, + "accuracy": 0.9942196531791907, + "expected_accuracy": 0.9942196531791907, + "expected_risk": 0.005780346820809301 + }, + { + "threshold": 0.99, + "selected": 438, + "coverage": 0.8342857142857143, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "workflow-controls-v1/customer_service": { + "count": 600, + "successful": 600, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 600, + "accuracy_all_rows_failures_incorrect": 0.9466666666666667, + "expected_accuracy_all_rows_failures_zero": 0.9466666666666667, + "coverage": [ + { + "threshold": 0.5, + "selected": 600, + "total": 600, + "coverage": 1.0 + }, + { + "threshold": 0.7, + "selected": 596, + "total": 600, + "coverage": 0.9933333333333333 + }, + { + "threshold": 0.8, + "selected": 592, + "total": 600, + "coverage": 0.9866666666666667 + }, + { + "threshold": 0.9, + "selected": 575, + "total": 600, + "coverage": 0.9583333333333334 + }, + { + "threshold": 0.95, + "selected": 559, + "total": 600, + "coverage": 0.9316666666666666 + }, + { + "threshold": 0.99, + "selected": 458, + "total": 600, + "coverage": 0.7633333333333333 + } + ], + "probability_metrics_all_rows": { + "count": 600, + "hard_count": 600, + "accuracy": 0.9466666666666667, + "expected_accuracy": 0.9466666666666667, + "nll": 0.14410110761968423, + "brier": 0.08122778878087651, + "expected_brier": 0.08122778878087651, + "ece_bins": 15, + "multiclass_ece": 0.037479888817243255, + "coverage": [ + { + "threshold": 0.5, + "selected": 600, + "coverage": 1.0, + "accuracy": 0.9466666666666667, + "expected_accuracy": 0.9466666666666667, + "expected_risk": 0.053333333333333344 + }, + { + "threshold": 0.7, + "selected": 596, + "coverage": 0.9933333333333333, + "accuracy": 0.9530201342281879, + "expected_accuracy": 0.9530201342281879, + "expected_risk": 0.046979865771812124 + }, + { + "threshold": 0.8, + "selected": 592, + "coverage": 0.9866666666666667, + "accuracy": 0.9594594594594594, + "expected_accuracy": 0.9594594594594594, + "expected_risk": 0.04054054054054057 + }, + { + "threshold": 0.9, + "selected": 575, + "coverage": 0.9583333333333334, + "accuracy": 0.9739130434782609, + "expected_accuracy": 0.9739130434782609, + "expected_risk": 0.02608695652173909 + }, + { + "threshold": 0.95, + "selected": 559, + "coverage": 0.9316666666666666, + "accuracy": 0.9803220035778175, + "expected_accuracy": 0.9803220035778175, + "expected_risk": 0.019677996422182487 + }, + { + "threshold": 0.99, + "selected": 458, + "coverage": 0.7633333333333333, + "accuracy": 0.9912663755458515, + "expected_accuracy": 0.9912663755458515, + "expected_risk": 0.008733624454148492 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 600, + "hard_count": 600, + "accuracy": 0.9466666666666667, + "expected_accuracy": 0.9466666666666667, + "nll": 0.14410110761968423, + "brier": 0.08122778878087651, + "expected_brier": 0.08122778878087651, + "ece_bins": 15, + "multiclass_ece": 0.037479888817243255, + "coverage": [ + { + "threshold": 0.5, + "selected": 600, + "coverage": 1.0, + "accuracy": 0.9466666666666667, + "expected_accuracy": 0.9466666666666667, + "expected_risk": 0.053333333333333344 + }, + { + "threshold": 0.7, + "selected": 596, + "coverage": 0.9933333333333333, + "accuracy": 0.9530201342281879, + "expected_accuracy": 0.9530201342281879, + "expected_risk": 0.046979865771812124 + }, + { + "threshold": 0.8, + "selected": 592, + "coverage": 0.9866666666666667, + "accuracy": 0.9594594594594594, + "expected_accuracy": 0.9594594594594594, + "expected_risk": 0.04054054054054057 + }, + { + "threshold": 0.9, + "selected": 575, + "coverage": 0.9583333333333334, + "accuracy": 0.9739130434782609, + "expected_accuracy": 0.9739130434782609, + "expected_risk": 0.02608695652173909 + }, + { + "threshold": 0.95, + "selected": 559, + "coverage": 0.9316666666666666, + "accuracy": 0.9803220035778175, + "expected_accuracy": 0.9803220035778175, + "expected_risk": 0.019677996422182487 + }, + { + "threshold": 0.99, + "selected": 458, + "coverage": 0.7633333333333333, + "accuracy": 0.9912663755458515, + "expected_accuracy": 0.9912663755458515, + "expected_risk": 0.008733624454148492 + } + ] + } + }, + "workflow-controls-v1/invoice_processing": { + "count": 750, + "successful": 750, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 750, + "accuracy_all_rows_failures_incorrect": 0.9973333333333333, + "expected_accuracy_all_rows_failures_zero": 0.9973333333333333, + "coverage": [ + { + "threshold": 0.5, + "selected": 750, + "total": 750, + "coverage": 1.0 + }, + { + "threshold": 0.7, + "selected": 746, + "total": 750, + "coverage": 0.9946666666666667 + }, + { + "threshold": 0.8, + "selected": 741, + "total": 750, + "coverage": 0.988 + }, + { + "threshold": 0.9, + "selected": 739, + "total": 750, + "coverage": 0.9853333333333333 + }, + { + "threshold": 0.95, + "selected": 723, + "total": 750, + "coverage": 0.964 + }, + { + "threshold": 0.99, + "selected": 565, + "total": 750, + "coverage": 0.7533333333333333 + } + ], + "probability_metrics_all_rows": { + "count": 750, + "hard_count": 750, + "accuracy": 0.9973333333333333, + "expected_accuracy": 0.9973333333333333, + "nll": 0.013733993789974948, + "brier": 0.005134315183105472, + "expected_brier": 0.005134315183105474, + "ece_bins": 15, + "multiclass_ece": 0.008916569978317768, + "coverage": [ + { + "threshold": 0.5, + "selected": 750, + "coverage": 1.0, + "accuracy": 0.9973333333333333, + "expected_accuracy": 0.9973333333333333, + "expected_risk": 0.002666666666666706 + }, + { + "threshold": 0.7, + "selected": 746, + "coverage": 0.9946666666666667, + "accuracy": 0.9973190348525469, + "expected_accuracy": 0.9973190348525469, + "expected_risk": 0.002680965147453085 + }, + { + "threshold": 0.8, + "selected": 741, + "coverage": 0.988, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 739, + "coverage": 0.9853333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 723, + "coverage": 0.964, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 565, + "coverage": 0.7533333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 750, + "hard_count": 750, + "accuracy": 0.9973333333333333, + "expected_accuracy": 0.9973333333333333, + "nll": 0.013733993789974948, + "brier": 0.005134315183105472, + "expected_brier": 0.005134315183105474, + "ece_bins": 15, + "multiclass_ece": 0.008916569978317768, + "coverage": [ + { + "threshold": 0.5, + "selected": 750, + "coverage": 1.0, + "accuracy": 0.9973333333333333, + "expected_accuracy": 0.9973333333333333, + "expected_risk": 0.002666666666666706 + }, + { + "threshold": 0.7, + "selected": 746, + "coverage": 0.9946666666666667, + "accuracy": 0.9973190348525469, + "expected_accuracy": 0.9973190348525469, + "expected_risk": 0.002680965147453085 + }, + { + "threshold": 0.8, + "selected": 741, + "coverage": 0.988, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 739, + "coverage": 0.9853333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 723, + "coverage": 0.964, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 565, + "coverage": 0.7533333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "workflow-controls-v1/security_incidents": { + "count": 1275, + "successful": 1275, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 1275, + "accuracy_all_rows_failures_incorrect": 0.9937254901960785, + "expected_accuracy_all_rows_failures_zero": 0.9937254901960785, + "coverage": [ + { + "threshold": 0.5, + "selected": 1275, + "total": 1275, + "coverage": 1.0 + }, + { + "threshold": 0.7, + "selected": 1267, + "total": 1275, + "coverage": 0.9937254901960785 + }, + { + "threshold": 0.8, + "selected": 1265, + "total": 1275, + "coverage": 0.9921568627450981 + }, + { + "threshold": 0.9, + "selected": 1259, + "total": 1275, + "coverage": 0.9874509803921568 + }, + { + "threshold": 0.95, + "selected": 1239, + "total": 1275, + "coverage": 0.971764705882353 + }, + { + "threshold": 0.99, + "selected": 1181, + "total": 1275, + "coverage": 0.9262745098039216 + } + ], + "probability_metrics_all_rows": { + "count": 1275, + "hard_count": 1275, + "accuracy": 0.9937254901960785, + "expected_accuracy": 0.9937254901960785, + "nll": 0.01963776236994051, + "brier": 0.010570254728764788, + "expected_brier": 0.010570254728764786, + "ece_bins": 15, + "multiclass_ece": 0.002970435231356543, + "coverage": [ + { + "threshold": 0.5, + "selected": 1275, + "coverage": 1.0, + "accuracy": 0.9937254901960785, + "expected_accuracy": 0.9937254901960785, + "expected_risk": 0.006274509803921524 + }, + { + "threshold": 0.7, + "selected": 1267, + "coverage": 0.9937254901960785, + "accuracy": 0.9960536700868192, + "expected_accuracy": 0.9960536700868192, + "expected_risk": 0.003946329913180779 + }, + { + "threshold": 0.8, + "selected": 1265, + "coverage": 0.9921568627450981, + "accuracy": 0.9960474308300395, + "expected_accuracy": 0.9960474308300395, + "expected_risk": 0.0039525691699604515 + }, + { + "threshold": 0.9, + "selected": 1259, + "coverage": 0.9874509803921568, + "accuracy": 0.9968228752978554, + "expected_accuracy": 0.9968228752978554, + "expected_risk": 0.003177124702144596 + }, + { + "threshold": 0.95, + "selected": 1239, + "coverage": 0.971764705882353, + "accuracy": 0.9983857949959645, + "expected_accuracy": 0.9983857949959645, + "expected_risk": 0.0016142050040355294 + }, + { + "threshold": 0.99, + "selected": 1181, + "coverage": 0.9262745098039216, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 1275, + "hard_count": 1275, + "accuracy": 0.9937254901960785, + "expected_accuracy": 0.9937254901960785, + "nll": 0.01963776236994051, + "brier": 0.010570254728764788, + "expected_brier": 0.010570254728764786, + "ece_bins": 15, + "multiclass_ece": 0.002970435231356543, + "coverage": [ + { + "threshold": 0.5, + "selected": 1275, + "coverage": 1.0, + "accuracy": 0.9937254901960785, + "expected_accuracy": 0.9937254901960785, + "expected_risk": 0.006274509803921524 + }, + { + "threshold": 0.7, + "selected": 1267, + "coverage": 0.9937254901960785, + "accuracy": 0.9960536700868192, + "expected_accuracy": 0.9960536700868192, + "expected_risk": 0.003946329913180779 + }, + { + "threshold": 0.8, + "selected": 1265, + "coverage": 0.9921568627450981, + "accuracy": 0.9960474308300395, + "expected_accuracy": 0.9960474308300395, + "expected_risk": 0.0039525691699604515 + }, + { + "threshold": 0.9, + "selected": 1259, + "coverage": 0.9874509803921568, + "accuracy": 0.9968228752978554, + "expected_accuracy": 0.9968228752978554, + "expected_risk": 0.003177124702144596 + }, + { + "threshold": 0.95, + "selected": 1239, + "coverage": 0.971764705882353, + "accuracy": 0.9983857949959645, + "expected_accuracy": 0.9983857949959645, + "expected_risk": 0.0016142050040355294 + }, + { + "threshold": 0.99, + "selected": 1181, + "coverage": 0.9262745098039216, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + } + }, + "by_kind": { + "choice": { + "count": 3438, + "successful": 3438, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 2993, + "accuracy_all_rows_failures_incorrect": 0.7885065152021383, + "expected_accuracy_all_rows_failures_zero": 0.736268831840385, + "coverage": [ + { + "threshold": 0.5, + "selected": 3067, + "total": 3438, + "coverage": 0.8920884235020361 + }, + { + "threshold": 0.7, + "selected": 2755, + "total": 3438, + "coverage": 0.8013379872018616 + }, + { + "threshold": 0.8, + "selected": 2665, + "total": 3438, + "coverage": 0.7751599767306574 + }, + { + "threshold": 0.9, + "selected": 2505, + "total": 3438, + "coverage": 0.7286212914485166 + }, + { + "threshold": 0.95, + "selected": 2207, + "total": 3438, + "coverage": 0.6419429901105294 + }, + { + "threshold": 0.99, + "selected": 1672, + "total": 3438, + "coverage": 0.48632926119837117 + } + ], + "probability_metrics_all_rows": { + "count": 3438, + "hard_count": 2993, + "accuracy": 0.7885065152021383, + "expected_accuracy": 0.736268831840385, + "nll": 1.4384722301984725, + "brier": 0.33992539289134077, + "expected_brier": 0.41537779986643536, + "ece_bins": 15, + "multiclass_ece": 0.13811597781941512, + "coverage": [ + { + "threshold": 0.5, + "selected": 3067, + "coverage": 0.8920884235020361, + "accuracy": 0.8078417520310844, + "expected_accuracy": 0.7838821867188348, + "expected_risk": 0.2161178132811652 + }, + { + "threshold": 0.7, + "selected": 2755, + "coverage": 0.8013379872018616, + "accuracy": 0.8220338983050848, + "expected_accuracy": 0.8171203871748336, + "expected_risk": 0.1828796128251664 + }, + { + "threshold": 0.8, + "selected": 2665, + "coverage": 0.7751599767306574, + "accuracy": 0.828907435508346, + "expected_accuracy": 0.8252657911194498, + "expected_risk": 0.17473420888055025 + }, + { + "threshold": 0.9, + "selected": 2505, + "coverage": 0.7286212914485166, + "accuracy": 0.8285140562248996, + "expected_accuracy": 0.8265469061876247, + "expected_risk": 0.17345309381237528 + }, + { + "threshold": 0.95, + "selected": 2207, + "coverage": 0.6419429901105294, + "accuracy": 0.8145124716553288, + "expected_accuracy": 0.8142274580879021, + "expected_risk": 0.1857725419120979 + }, + { + "threshold": 0.99, + "selected": 1672, + "coverage": 0.48632926119837117, + "accuracy": 0.7816985645933014, + "expected_accuracy": 0.7816985645933014, + "expected_risk": 0.21830143540669855 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 3438, + "hard_count": 2993, + "accuracy": 0.7885065152021383, + "expected_accuracy": 0.736268831840385, + "nll": 1.4384722301984725, + "brier": 0.33992539289134077, + "expected_brier": 0.41537779986643536, + "ece_bins": 15, + "multiclass_ece": 0.13811597781941512, + "coverage": [ + { + "threshold": 0.5, + "selected": 3067, + "coverage": 0.8920884235020361, + "accuracy": 0.8078417520310844, + "expected_accuracy": 0.7838821867188348, + "expected_risk": 0.2161178132811652 + }, + { + "threshold": 0.7, + "selected": 2755, + "coverage": 0.8013379872018616, + "accuracy": 0.8220338983050848, + "expected_accuracy": 0.8171203871748336, + "expected_risk": 0.1828796128251664 + }, + { + "threshold": 0.8, + "selected": 2665, + "coverage": 0.7751599767306574, + "accuracy": 0.828907435508346, + "expected_accuracy": 0.8252657911194498, + "expected_risk": 0.17473420888055025 + }, + { + "threshold": 0.9, + "selected": 2505, + "coverage": 0.7286212914485166, + "accuracy": 0.8285140562248996, + "expected_accuracy": 0.8265469061876247, + "expected_risk": 0.17345309381237528 + }, + { + "threshold": 0.95, + "selected": 2207, + "coverage": 0.6419429901105294, + "accuracy": 0.8145124716553288, + "expected_accuracy": 0.8142274580879021, + "expected_risk": 0.1857725419120979 + }, + { + "threshold": 0.99, + "selected": 1672, + "coverage": 0.48632926119837117, + "accuracy": 0.7816985645933014, + "expected_accuracy": 0.7816985645933014, + "expected_risk": 0.21830143540669855 + } + ] + } + }, + "noul": { + "count": 9929, + "successful": 9929, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 9929, + "accuracy_all_rows_failures_incorrect": 0.8819619297008762, + "expected_accuracy_all_rows_failures_zero": 0.8819619297008762, + "coverage": [ + { + "threshold": 0.5, + "selected": 9929, + "total": 9929, + "coverage": 1.0 + }, + { + "threshold": 0.7, + "selected": 9795, + "total": 9929, + "coverage": 0.9865041796756975 + }, + { + "threshold": 0.8, + "selected": 9687, + "total": 9929, + "coverage": 0.9756269513546177 + }, + { + "threshold": 0.9, + "selected": 9400, + "total": 9929, + "coverage": 0.9467217242421191 + }, + { + "threshold": 0.95, + "selected": 8945, + "total": 9929, + "coverage": 0.9008963641857186 + }, + { + "threshold": 0.99, + "selected": 6768, + "total": 9929, + "coverage": 0.6816396414543258 + } + ], + "probability_metrics_all_rows": { + "count": 9929, + "hard_count": 9929, + "accuracy": 0.8819619297008762, + "expected_accuracy": 0.8819619297008762, + "nll": 0.5310062525703106, + "brier": 0.21147263775707414, + "expected_brier": 0.21147263775707414, + "ece_bins": 15, + "multiclass_ece": 0.09634430608269225, + "coverage": [ + { + "threshold": 0.5, + "selected": 9929, + "coverage": 1.0, + "accuracy": 0.8819619297008762, + "expected_accuracy": 0.8819619297008762, + "expected_risk": 0.11803807029912383 + }, + { + "threshold": 0.7, + "selected": 9795, + "coverage": 0.9865041796756975, + "accuracy": 0.8870852475752935, + "expected_accuracy": 0.8870852475752935, + "expected_risk": 0.1129147524247065 + }, + { + "threshold": 0.8, + "selected": 9687, + "coverage": 0.9756269513546177, + "accuracy": 0.8924331578404047, + "expected_accuracy": 0.8924331578404047, + "expected_risk": 0.10756684215959533 + }, + { + "threshold": 0.9, + "selected": 9400, + "coverage": 0.9467217242421191, + "accuracy": 0.9038297872340425, + "expected_accuracy": 0.9038297872340425, + "expected_risk": 0.0961702127659575 + }, + { + "threshold": 0.95, + "selected": 8945, + "coverage": 0.9008963641857186, + "accuracy": 0.9149245388485188, + "expected_accuracy": 0.9149245388485188, + "expected_risk": 0.08507546115148124 + }, + { + "threshold": 0.99, + "selected": 6768, + "coverage": 0.6816396414543258, + "accuracy": 0.9206560283687943, + "expected_accuracy": 0.9206560283687943, + "expected_risk": 0.07934397163120566 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 9929, + "hard_count": 9929, + "accuracy": 0.8819619297008762, + "expected_accuracy": 0.8819619297008762, + "nll": 0.5310062525703106, + "brier": 0.21147263775707414, + "expected_brier": 0.21147263775707414, + "ece_bins": 15, + "multiclass_ece": 0.09634430608269225, + "coverage": [ + { + "threshold": 0.5, + "selected": 9929, + "coverage": 1.0, + "accuracy": 0.8819619297008762, + "expected_accuracy": 0.8819619297008762, + "expected_risk": 0.11803807029912383 + }, + { + "threshold": 0.7, + "selected": 9795, + "coverage": 0.9865041796756975, + "accuracy": 0.8870852475752935, + "expected_accuracy": 0.8870852475752935, + "expected_risk": 0.1129147524247065 + }, + { + "threshold": 0.8, + "selected": 9687, + "coverage": 0.9756269513546177, + "accuracy": 0.8924331578404047, + "expected_accuracy": 0.8924331578404047, + "expected_risk": 0.10756684215959533 + }, + { + "threshold": 0.9, + "selected": 9400, + "coverage": 0.9467217242421191, + "accuracy": 0.9038297872340425, + "expected_accuracy": 0.9038297872340425, + "expected_risk": 0.0961702127659575 + }, + { + "threshold": 0.95, + "selected": 8945, + "coverage": 0.9008963641857186, + "accuracy": 0.9149245388485188, + "expected_accuracy": 0.9149245388485188, + "expected_risk": 0.08507546115148124 + }, + { + "threshold": 0.99, + "selected": 6768, + "coverage": 0.6816396414543258, + "accuracy": 0.9206560283687943, + "expected_accuracy": 0.9206560283687943, + "expected_risk": 0.07934397163120566 + } + ] + } + }, + "score": { + "count": 2553, + "successful": 2553, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 2524, + "accuracy_all_rows_failures_incorrect": 0.8597464342313788, + "expected_accuracy_all_rows_failures_zero": 0.854876615746181, + "coverage": [ + { + "threshold": 0.5, + "selected": 2471, + "total": 2553, + "coverage": 0.9678809244026635 + }, + { + "threshold": 0.7, + "selected": 2424, + "total": 2553, + "coverage": 0.9494712103407755 + }, + { + "threshold": 0.8, + "selected": 2403, + "total": 2553, + "coverage": 0.9412455934195064 + }, + { + "threshold": 0.9, + "selected": 2307, + "total": 2553, + "coverage": 0.9036427732079906 + }, + { + "threshold": 0.95, + "selected": 2133, + "total": 2553, + "coverage": 0.8354876615746181 + }, + { + "threshold": 0.99, + "selected": 1556, + "total": 2553, + "coverage": 0.609479044261653 + } + ], + "probability_metrics_all_rows": { + "count": 2553, + "hard_count": 2524, + "accuracy": 0.8597464342313788, + "expected_accuracy": 0.854876615746181, + "nll": 1.0050022837630184, + "brier": 0.25660594897422545, + "expected_brier": 0.26306893369807977, + "ece_bins": 15, + "multiclass_ece": 0.10397632298121842, + "coverage": [ + { + "threshold": 0.5, + "selected": 2471, + "coverage": 0.9678809244026635, + "accuracy": 0.874949083503055, + "expected_accuracy": 0.8725212464589235, + "expected_risk": 0.12747875354107652 + }, + { + "threshold": 0.7, + "selected": 2424, + "coverage": 0.9494712103407755, + "accuracy": 0.8758250825082509, + "expected_accuracy": 0.8758250825082509, + "expected_risk": 0.12417491749174914 + }, + { + "threshold": 0.8, + "selected": 2403, + "coverage": 0.9412455934195064, + "accuracy": 0.8768206408655846, + "expected_accuracy": 0.8768206408655846, + "expected_risk": 0.12317935913441536 + }, + { + "threshold": 0.9, + "selected": 2307, + "coverage": 0.9036427732079906, + "accuracy": 0.8764629388816645, + "expected_accuracy": 0.8764629388816645, + "expected_risk": 0.12353706111833551 + }, + { + "threshold": 0.95, + "selected": 2133, + "coverage": 0.8354876615746181, + "accuracy": 0.8752930145335208, + "expected_accuracy": 0.8752930145335208, + "expected_risk": 0.12470698546647918 + }, + { + "threshold": 0.99, + "selected": 1556, + "coverage": 0.609479044261653, + "accuracy": 0.8759640102827764, + "expected_accuracy": 0.8759640102827764, + "expected_risk": 0.12403598971722363 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 2553, + "hard_count": 2524, + "accuracy": 0.8597464342313788, + "expected_accuracy": 0.854876615746181, + "nll": 1.0050022837630184, + "brier": 0.25660594897422545, + "expected_brier": 0.26306893369807977, + "ece_bins": 15, + "multiclass_ece": 0.10397632298121842, + "coverage": [ + { + "threshold": 0.5, + "selected": 2471, + "coverage": 0.9678809244026635, + "accuracy": 0.874949083503055, + "expected_accuracy": 0.8725212464589235, + "expected_risk": 0.12747875354107652 + }, + { + "threshold": 0.7, + "selected": 2424, + "coverage": 0.9494712103407755, + "accuracy": 0.8758250825082509, + "expected_accuracy": 0.8758250825082509, + "expected_risk": 0.12417491749174914 + }, + { + "threshold": 0.8, + "selected": 2403, + "coverage": 0.9412455934195064, + "accuracy": 0.8768206408655846, + "expected_accuracy": 0.8768206408655846, + "expected_risk": 0.12317935913441536 + }, + { + "threshold": 0.9, + "selected": 2307, + "coverage": 0.9036427732079906, + "accuracy": 0.8764629388816645, + "expected_accuracy": 0.8764629388816645, + "expected_risk": 0.12353706111833551 + }, + { + "threshold": 0.95, + "selected": 2133, + "coverage": 0.8354876615746181, + "accuracy": 0.8752930145335208, + "expected_accuracy": 0.8752930145335208, + "expected_risk": 0.12470698546647918 + }, + { + "threshold": 0.99, + "selected": 1556, + "coverage": 0.609479044261653, + "accuracy": 0.8759640102827764, + "expected_accuracy": 0.8759640102827764, + "expected_risk": 0.12403598971722363 + } + ] + } + } + } + } + }, + "full_data_baseline_evaluated": false, + "full_data_training_gain": null, + "limitations": [ + "No full-data baseline exists in this run. Earlier 512-row-per-split baseline results cannot establish full-data training gain.", + "Per-group metrics were recomputed and compared for every group; compact report retains source/kind metrics and group counts/hashes. Original complete summary remains in ignored evidence.", + "Snapshot/checkpoint hashes bind this evaluation to the packaged inference files; the trainer's historical reload lacked a contemporaneous weight digest.", + "Synthetic held-out decision accuracy is not end-to-end game, browser or flight performance. Four-replica wall time is not single-GPU latency." + ], + "schema_version": 1, + "status": "passed", + "source_audit_sha256": "e8b0583b12ee00e8031e7ea5388d935389c3ce9681e3099da0cafe30329ff472", + "source_audit_path": "reports/full-data-eval-n1-v1/2b-audit.json", + "scope": "Existing completed full release-v2 held-out evaluation, bound to the unchanged five packaged inference files. No new model inference was run for this Hugging Face upload." +}