| { |
| "model": "Qwen/Qwen3.5-2B", |
| "revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", |
| "checkpoint": { |
| "model": "Qwen/Qwen3.5-2B", |
| "revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", |
| "checkpoint_sha256": "8c37b393c27d2b58009463a89dd1a873b7c010ecd1b8c184c7ada5efe13f8bfb", |
| "temperature": 1.518796342858676, |
| "calibration": { |
| "temperature": 1.518796342858676, |
| "split": "calibration", |
| "n": 512, |
| "ids_sha256": "6ff7db59c1ae2485179b80316b3cb4f0161df77704692dcab48871a295ab94f1" |
| }, |
| "max_length": 4096, |
| "training_completion": { |
| "steps": 20204, |
| "trained_rows_consumed": 80816, |
| "checkpoint_reload_max_error": 0.0, |
| "run_sha256": "8ee935b52c0ee1796a25d8d8b0fcfc8aa41eddee815c454016c956f204b1416a", |
| "summary_sha256": "ea61d4b1cc28c57a826c57986323abe4bbf715ff28985d7a6fa89fec2119c8bf" |
| } |
| }, |
| "package_bound_inference_files": { |
| "checkpoint/adapter/adapter_config.json": { |
| "sha256": "d27624bdbbe984cf175e10c7fa3d9beaea9876231ce943a248ad40e12742fe74", |
| "bytes": 1170 |
| }, |
| "checkpoint/adapter/adapter_model.safetensors": { |
| "sha256": "2d23935b1a7380db444abac572c04646918ba794e59002d1588236182a3ca18f", |
| "bytes": 10042480 |
| }, |
| "checkpoint/head.pt": { |
| "sha256": "3532cd576c58d5ad5bf17c3e9f2df4be8c70e08c07fa6bb7fa673dcd7b401f2a", |
| "bytes": 9997 |
| }, |
| "checkpoint/model.json": { |
| "sha256": "d985eafa4d635113ee8cc912e0512a95aac0bed3426e9256931235b75cb3d6c5", |
| "bytes": 186 |
| }, |
| "checkpoint/temperature.json": { |
| "sha256": "090fb330a616338210a924d1c370c934e934f30f53f92c8a52a5cc7852cadd99", |
| "bytes": 161 |
| } |
| }, |
| "data_identity": { |
| "manifest_sha256": "56105dc9fc89ef74919f5beb60bb6ae8c6e17bb95699dab59205f67d8b338d97", |
| "split_sha256": { |
| "train": "80a9ec19da9c4065b72f7a9e5ff8d67e62c1d1dd2572dfd296e46bfe2ea5a182", |
| "calibration": "dd21b26912a6b26a09c3b00ae66a7985ee760e737b94ada4fcc6320406d4db63", |
| "validation": "1dc97079d4fc034a1ba16315f801b0a4715b30c2a3874ae05aeb60dd1e40e8bb", |
| "test": "fa8d62775b96c1514f238c2d8a8f8b66515dada67d490978376545296593e29b", |
| "ood": "9152bff7f3d9c83cce3f1423cc6e0e6b33286b3092b3f544cb18dcc37b35a7ee" |
| }, |
| "counts": { |
| "test": 10532, |
| "ood": 15920 |
| }, |
| "ordered_ids_sha256": { |
| "test": "98e1af223471c08d84e44c8864e51cefa1bc671687c00bfbf64b7e5a22e8bf52", |
| "ood": "52fa6cc83c0a172516609257b6048aaece9a3fef75f3f5f694bc6213e5c30a42" |
| } |
| }, |
| "coverage": { |
| "expected_rows": 26452, |
| "unique_ids": 26452, |
| "shard_rows": [ |
| 6613, |
| 6613, |
| 6613, |
| 6613 |
| ], |
| "missing": 0, |
| "duplicate": 0, |
| "failed": 0, |
| "merged_original_order_verified": true |
| }, |
| "metrics": { |
| "test": { |
| "count": 10532, |
| "successful": 10532, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 10046, |
| "accuracy_all_rows_failures_incorrect": 0.9471431415488751, |
| "expected_accuracy_all_rows_failures_zero": 0.9195253563797051, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 9830, |
| "total": 10532, |
| "coverage": 0.9333459931636916 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 9500, |
| "total": 10532, |
| "coverage": 0.9020129130269654 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 9388, |
| "total": 10532, |
| "coverage": 0.8913786555260159 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 9116, |
| "total": 10532, |
| "coverage": 0.8655526015951386 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 8634, |
| "total": 10532, |
| "coverage": 0.819787314849981 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 6742, |
| "total": 10532, |
| "coverage": 0.6401443220660843 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 10532, |
| "hard_count": 10046, |
| "accuracy": 0.9471431415488751, |
| "expected_accuracy": 0.9195253563797051, |
| "nll": 0.19538003409263777, |
| "brier": 0.07611697556114595, |
| "expected_brier": 0.10416793987302937, |
| "ece_bins": 15, |
| "multiclass_ece": 0.011605590146725192, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 9830, |
| "coverage": 0.9333459931636916, |
| "accuracy": 0.9686659772492244, |
| "expected_accuracy": 0.9607494065785012, |
| "expected_risk": 0.039250593421498836 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 9500, |
| "coverage": 0.9020129130269654, |
| "accuracy": 0.976356343677433, |
| "expected_accuracy": 0.9749649122807017, |
| "expected_risk": 0.0250350877192983 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 9388, |
| "coverage": 0.8913786555260159, |
| "accuracy": 0.9796136193830718, |
| "expected_accuracy": 0.9785719358045732, |
| "expected_risk": 0.021428064195426755 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 9116, |
| "coverage": 0.8655526015951386, |
| "accuracy": 0.9880377524143986, |
| "expected_accuracy": 0.9878236068451075, |
| "expected_risk": 0.012176393154892495 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 8634, |
| "coverage": 0.819787314849981, |
| "accuracy": 0.9950196895992588, |
| "expected_accuracy": 0.9950196895992588, |
| "expected_risk": 0.004980310400741206 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 6742, |
| "coverage": 0.6401443220660843, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 10532, |
| "hard_count": 10046, |
| "accuracy": 0.9471431415488751, |
| "expected_accuracy": 0.9195253563797051, |
| "nll": 0.19538003409263777, |
| "brier": 0.07611697556114595, |
| "expected_brier": 0.10416793987302937, |
| "ece_bins": 15, |
| "multiclass_ece": 0.011605590146725192, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 9830, |
| "coverage": 0.9333459931636916, |
| "accuracy": 0.9686659772492244, |
| "expected_accuracy": 0.9607494065785012, |
| "expected_risk": 0.039250593421498836 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 9500, |
| "coverage": 0.9020129130269654, |
| "accuracy": 0.976356343677433, |
| "expected_accuracy": 0.9749649122807017, |
| "expected_risk": 0.0250350877192983 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 9388, |
| "coverage": 0.8913786555260159, |
| "accuracy": 0.9796136193830718, |
| "expected_accuracy": 0.9785719358045732, |
| "expected_risk": 0.021428064195426755 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 9116, |
| "coverage": 0.8655526015951386, |
| "accuracy": 0.9880377524143986, |
| "expected_accuracy": 0.9878236068451075, |
| "expected_risk": 0.012176393154892495 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 8634, |
| "coverage": 0.819787314849981, |
| "accuracy": 0.9950196895992588, |
| "expected_accuracy": 0.9950196895992588, |
| "expected_risk": 0.004980310400741206 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 6742, |
| "coverage": 0.6401443220660843, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "by_source": { |
| "customer-control-v1": { |
| "count": 606, |
| "successful": 606, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 574, |
| "accuracy_all_rows_failures_incorrect": 1.0, |
| "expected_accuracy_all_rows_failures_zero": 0.9722222222222222, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 598, |
| "total": 606, |
| "coverage": 0.9867986798679867 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 576, |
| "total": 606, |
| "coverage": 0.9504950495049505 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 576, |
| "total": 606, |
| "coverage": 0.9504950495049505 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 573, |
| "total": 606, |
| "coverage": 0.9455445544554455 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 565, |
| "total": 606, |
| "coverage": 0.9323432343234324 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 504, |
| "total": 606, |
| "coverage": 0.8316831683168316 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 606, |
| "hard_count": 574, |
| "accuracy": 1.0, |
| "expected_accuracy": 0.9722222222222222, |
| "nll": 0.04670156852740247, |
| "brier": 0.0013588176050983829, |
| "expected_brier": 0.029136595382876163, |
| "ece_bins": 15, |
| "multiclass_ece": 0.005803372421616342, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 598, |
| "coverage": 0.9867986798679867, |
| "accuracy": 1.0, |
| "expected_accuracy": 0.979933110367893, |
| "expected_risk": 0.02006688963210701 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 576, |
| "coverage": 0.9504950495049505, |
| "accuracy": 1.0, |
| "expected_accuracy": 0.9982638888888888, |
| "expected_risk": 0.0017361111111111605 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 576, |
| "coverage": 0.9504950495049505, |
| "accuracy": 1.0, |
| "expected_accuracy": 0.9982638888888888, |
| "expected_risk": 0.0017361111111111605 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 573, |
| "coverage": 0.9455445544554455, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 565, |
| "coverage": 0.9323432343234324, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 504, |
| "coverage": 0.8316831683168316, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 606, |
| "hard_count": 574, |
| "accuracy": 1.0, |
| "expected_accuracy": 0.9722222222222222, |
| "nll": 0.04670156852740247, |
| "brier": 0.0013588176050983829, |
| "expected_brier": 0.029136595382876163, |
| "ece_bins": 15, |
| "multiclass_ece": 0.005803372421616342, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 598, |
| "coverage": 0.9867986798679867, |
| "accuracy": 1.0, |
| "expected_accuracy": 0.979933110367893, |
| "expected_risk": 0.02006688963210701 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 576, |
| "coverage": 0.9504950495049505, |
| "accuracy": 1.0, |
| "expected_accuracy": 0.9982638888888888, |
| "expected_risk": 0.0017361111111111605 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 576, |
| "coverage": 0.9504950495049505, |
| "accuracy": 1.0, |
| "expected_accuracy": 0.9982638888888888, |
| "expected_risk": 0.0017361111111111605 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 573, |
| "coverage": 0.9455445544554455, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 565, |
| "coverage": 0.9323432343234324, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 504, |
| "coverage": 0.8316831683168316, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| }, |
| "painting-geometry-v1": { |
| "count": 3072, |
| "successful": 3072, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 3072, |
| "accuracy_all_rows_failures_incorrect": 0.9964192708333334, |
| "expected_accuracy_all_rows_failures_zero": 0.9964192708333334, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 3072, |
| "total": 3072, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 3064, |
| "total": 3072, |
| "coverage": 0.9973958333333334 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 3060, |
| "total": 3072, |
| "coverage": 0.99609375 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 3058, |
| "total": 3072, |
| "coverage": 0.9954427083333334 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 3039, |
| "total": 3072, |
| "coverage": 0.9892578125 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 2768, |
| "total": 3072, |
| "coverage": 0.9010416666666666 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 3072, |
| "hard_count": 3072, |
| "accuracy": 0.9964192708333334, |
| "expected_accuracy": 0.9964192708333334, |
| "nll": 0.012696533729082774, |
| "brier": 0.005503376394520538, |
| "expected_brier": 0.0055033763945205386, |
| "ece_bins": 15, |
| "multiclass_ece": 0.0037695271810740086, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 3072, |
| "coverage": 1.0, |
| "accuracy": 0.9964192708333334, |
| "expected_accuracy": 0.9964192708333334, |
| "expected_risk": 0.0035807291666666297 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 3064, |
| "coverage": 0.9973958333333334, |
| "accuracy": 0.9977154046997389, |
| "expected_accuracy": 0.9977154046997389, |
| "expected_risk": 0.0022845953002611497 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 3060, |
| "coverage": 0.99609375, |
| "accuracy": 0.9980392156862745, |
| "expected_accuracy": 0.9980392156862745, |
| "expected_risk": 0.0019607843137254832 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 3058, |
| "coverage": 0.9954427083333334, |
| "accuracy": 0.9983649444081099, |
| "expected_accuracy": 0.9983649444081099, |
| "expected_risk": 0.0016350555918901444 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 3039, |
| "coverage": 0.9892578125, |
| "accuracy": 0.9983547219480092, |
| "expected_accuracy": 0.9983547219480092, |
| "expected_risk": 0.0016452780519907728 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 2768, |
| "coverage": 0.9010416666666666, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 3072, |
| "hard_count": 3072, |
| "accuracy": 0.9964192708333334, |
| "expected_accuracy": 0.9964192708333334, |
| "nll": 0.012696533729082774, |
| "brier": 0.005503376394520538, |
| "expected_brier": 0.0055033763945205386, |
| "ece_bins": 15, |
| "multiclass_ece": 0.0037695271810740086, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 3072, |
| "coverage": 1.0, |
| "accuracy": 0.9964192708333334, |
| "expected_accuracy": 0.9964192708333334, |
| "expected_risk": 0.0035807291666666297 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 3064, |
| "coverage": 0.9973958333333334, |
| "accuracy": 0.9977154046997389, |
| "expected_accuracy": 0.9977154046997389, |
| "expected_risk": 0.0022845953002611497 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 3060, |
| "coverage": 0.99609375, |
| "accuracy": 0.9980392156862745, |
| "expected_accuracy": 0.9980392156862745, |
| "expected_risk": 0.0019607843137254832 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 3058, |
| "coverage": 0.9954427083333334, |
| "accuracy": 0.9983649444081099, |
| "expected_accuracy": 0.9983649444081099, |
| "expected_risk": 0.0016350555918901444 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 3039, |
| "coverage": 0.9892578125, |
| "accuracy": 0.9983547219480092, |
| "expected_accuracy": 0.9983547219480092, |
| "expected_risk": 0.0016452780519907728 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 2768, |
| "coverage": 0.9010416666666666, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| }, |
| "reasoning-control-v1": { |
| "count": 681, |
| "successful": 681, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 681, |
| "accuracy_all_rows_failures_incorrect": 0.7797356828193832, |
| "expected_accuracy_all_rows_failures_zero": 0.7797356828193832, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 572, |
| "total": 681, |
| "coverage": 0.8399412628487518 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 515, |
| "total": 681, |
| "coverage": 0.7562408223201175 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 487, |
| "total": 681, |
| "coverage": 0.7151248164464024 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 424, |
| "total": 681, |
| "coverage": 0.6226138032305433 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 351, |
| "total": 681, |
| "coverage": 0.5154185022026432 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 180, |
| "total": 681, |
| "coverage": 0.2643171806167401 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 681, |
| "hard_count": 681, |
| "accuracy": 0.7797356828193832, |
| "expected_accuracy": 0.7797356828193832, |
| "nll": 0.5276862396629047, |
| "brier": 0.30043397912430997, |
| "expected_brier": 0.30043397912430997, |
| "ece_bins": 15, |
| "multiclass_ece": 0.06116577188210845, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 572, |
| "coverage": 0.8399412628487518, |
| "accuracy": 0.8513986013986014, |
| "expected_accuracy": 0.8513986013986014, |
| "expected_risk": 0.14860139860139865 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 515, |
| "coverage": 0.7562408223201175, |
| "accuracy": 0.883495145631068, |
| "expected_accuracy": 0.883495145631068, |
| "expected_risk": 0.11650485436893199 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 487, |
| "coverage": 0.7151248164464024, |
| "accuracy": 0.9055441478439425, |
| "expected_accuracy": 0.9055441478439425, |
| "expected_risk": 0.09445585215605745 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 424, |
| "coverage": 0.6226138032305433, |
| "accuracy": 0.9410377358490566, |
| "expected_accuracy": 0.9410377358490566, |
| "expected_risk": 0.05896226415094341 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 351, |
| "coverage": 0.5154185022026432, |
| "accuracy": 0.9886039886039886, |
| "expected_accuracy": 0.9886039886039886, |
| "expected_risk": 0.01139601139601143 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 180, |
| "coverage": 0.2643171806167401, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 681, |
| "hard_count": 681, |
| "accuracy": 0.7797356828193832, |
| "expected_accuracy": 0.7797356828193832, |
| "nll": 0.5276862396629047, |
| "brier": 0.30043397912430997, |
| "expected_brier": 0.30043397912430997, |
| "ece_bins": 15, |
| "multiclass_ece": 0.06116577188210845, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 572, |
| "coverage": 0.8399412628487518, |
| "accuracy": 0.8513986013986014, |
| "expected_accuracy": 0.8513986013986014, |
| "expected_risk": 0.14860139860139865 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 515, |
| "coverage": 0.7562408223201175, |
| "accuracy": 0.883495145631068, |
| "expected_accuracy": 0.883495145631068, |
| "expected_risk": 0.11650485436893199 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 487, |
| "coverage": 0.7151248164464024, |
| "accuracy": 0.9055441478439425, |
| "expected_accuracy": 0.9055441478439425, |
| "expected_risk": 0.09445585215605745 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 424, |
| "coverage": 0.6226138032305433, |
| "accuracy": 0.9410377358490566, |
| "expected_accuracy": 0.9410377358490566, |
| "expected_risk": 0.05896226415094341 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 351, |
| "coverage": 0.5154185022026432, |
| "accuracy": 0.9886039886039886, |
| "expected_accuracy": 0.9886039886039886, |
| "expected_risk": 0.01139601139601143 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 180, |
| "coverage": 0.2643171806167401, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| }, |
| "snake-v1": { |
| "count": 1392, |
| "successful": 1392, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 1279, |
| "accuracy_all_rows_failures_incorrect": 0.8780297107114934, |
| "expected_accuracy_all_rows_failures_zero": 0.8461446360153256, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1352, |
| "total": 1392, |
| "coverage": 0.9712643678160919 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1237, |
| "total": 1392, |
| "coverage": 0.8886494252873564 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1186, |
| "total": 1392, |
| "coverage": 0.8520114942528736 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1014, |
| "total": 1392, |
| "coverage": 0.728448275862069 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 743, |
| "total": 1392, |
| "coverage": 0.5337643678160919 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 159, |
| "total": 1392, |
| "coverage": 0.11422413793103449 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 1392, |
| "hard_count": 1279, |
| "accuracy": 0.8780297107114934, |
| "expected_accuracy": 0.8461446360153256, |
| "nll": 0.3775572072892607, |
| "brier": 0.18597837736131362, |
| "expected_brier": 0.22704638502414884, |
| "ece_bins": 15, |
| "multiclass_ece": 0.0526435483183463, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1352, |
| "coverage": 0.9712643678160919, |
| "accuracy": 0.8844022169437846, |
| "expected_accuracy": 0.8583579881656804, |
| "expected_risk": 0.14164201183431957 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1237, |
| "coverage": 0.8886494252873564, |
| "accuracy": 0.8962108731466227, |
| "expected_accuracy": 0.8881703045001348, |
| "expected_risk": 0.11182969549986521 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1186, |
| "coverage": 0.8520114942528736, |
| "accuracy": 0.9024807527801539, |
| "expected_accuracy": 0.8961495222034851, |
| "expected_risk": 0.1038504777965149 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1014, |
| "coverage": 0.728448275862069, |
| "accuracy": 0.9376237623762376, |
| "expected_accuracy": 0.9358974358974359, |
| "expected_risk": 0.0641025641025641 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 743, |
| "coverage": 0.5337643678160919, |
| "accuracy": 0.9663526244952894, |
| "expected_accuracy": 0.9663526244952894, |
| "expected_risk": 0.03364737550471064 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 159, |
| "coverage": 0.11422413793103449, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 1392, |
| "hard_count": 1279, |
| "accuracy": 0.8780297107114934, |
| "expected_accuracy": 0.8461446360153256, |
| "nll": 0.3775572072892607, |
| "brier": 0.18597837736131362, |
| "expected_brier": 0.22704638502414884, |
| "ece_bins": 15, |
| "multiclass_ece": 0.0526435483183463, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1352, |
| "coverage": 0.9712643678160919, |
| "accuracy": 0.8844022169437846, |
| "expected_accuracy": 0.8583579881656804, |
| "expected_risk": 0.14164201183431957 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1237, |
| "coverage": 0.8886494252873564, |
| "accuracy": 0.8962108731466227, |
| "expected_accuracy": 0.8881703045001348, |
| "expected_risk": 0.11182969549986521 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1186, |
| "coverage": 0.8520114942528736, |
| "accuracy": 0.9024807527801539, |
| "expected_accuracy": 0.8961495222034851, |
| "expected_risk": 0.1038504777965149 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1014, |
| "coverage": 0.728448275862069, |
| "accuracy": 0.9376237623762376, |
| "expected_accuracy": 0.9358974358974359, |
| "expected_risk": 0.0641025641025641 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 743, |
| "coverage": 0.5337643678160919, |
| "accuracy": 0.9663526244952894, |
| "expected_accuracy": 0.9663526244952894, |
| "expected_risk": 0.03364737550471064 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 159, |
| "coverage": 0.11422413793103449, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| }, |
| "tic_tac_toe-v1": { |
| "count": 472, |
| "successful": 472, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 256, |
| "accuracy_all_rows_failures_incorrect": 0.4765625, |
| "expected_accuracy_all_rows_failures_zero": 0.41646489104116224, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 107, |
| "total": 472, |
| "coverage": 0.2266949152542373 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 10, |
| "total": 472, |
| "coverage": 0.0211864406779661 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1, |
| "total": 472, |
| "coverage": 0.00211864406779661 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 0, |
| "total": 472, |
| "coverage": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 0, |
| "total": 472, |
| "coverage": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 0, |
| "total": 472, |
| "coverage": 0.0 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 472, |
| "hard_count": 256, |
| "accuracy": 0.4765625, |
| "expected_accuracy": 0.41646489104116224, |
| "nll": 1.1856874570601634, |
| "brier": 0.3986960943836407, |
| "expected_brier": 0.6686718813085801, |
| "ece_bins": 15, |
| "multiclass_ece": 0.05099574700428666, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 107, |
| "coverage": 0.2266949152542373, |
| "accuracy": 0.75, |
| "expected_accuracy": 0.6230529595015577, |
| "expected_risk": 0.3769470404984423 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 10, |
| "coverage": 0.0211864406779661, |
| "accuracy": 0.8888888888888888, |
| "expected_accuracy": 0.85, |
| "expected_risk": 0.15000000000000002 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1, |
| "coverage": 0.00211864406779661, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 472, |
| "hard_count": 256, |
| "accuracy": 0.4765625, |
| "expected_accuracy": 0.41646489104116224, |
| "nll": 1.1856874570601634, |
| "brier": 0.3986960943836407, |
| "expected_brier": 0.6686718813085801, |
| "ece_bins": 15, |
| "multiclass_ece": 0.05099574700428666, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 107, |
| "coverage": 0.2266949152542373, |
| "accuracy": 0.75, |
| "expected_accuracy": 0.6230529595015577, |
| "expected_risk": 0.3769470404984423 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 10, |
| "coverage": 0.0211864406779661, |
| "accuracy": 0.8888888888888888, |
| "expected_accuracy": 0.85, |
| "expected_risk": 0.15000000000000002 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1, |
| "coverage": 0.00211864406779661, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| } |
| ] |
| } |
| }, |
| "tile_platformer-v1": { |
| "count": 202, |
| "successful": 202, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 202, |
| "accuracy_all_rows_failures_incorrect": 0.9752475247524752, |
| "expected_accuracy_all_rows_failures_zero": 0.9752475247524752, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 202, |
| "total": 202, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 202, |
| "total": 202, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 200, |
| "total": 202, |
| "coverage": 0.9900990099009901 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 198, |
| "total": 202, |
| "coverage": 0.9801980198019802 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 183, |
| "total": 202, |
| "coverage": 0.905940594059406 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 8, |
| "total": 202, |
| "coverage": 0.039603960396039604 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 202, |
| "hard_count": 202, |
| "accuracy": 0.9752475247524752, |
| "expected_accuracy": 0.9752475247524752, |
| "nll": 0.13723787641738036, |
| "brier": 0.048090580988872436, |
| "expected_brier": 0.048090580988872436, |
| "ece_bins": 15, |
| "multiclass_ece": 0.01462437849252733, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 202, |
| "coverage": 1.0, |
| "accuracy": 0.9752475247524752, |
| "expected_accuracy": 0.9752475247524752, |
| "expected_risk": 0.024752475247524774 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 202, |
| "coverage": 1.0, |
| "accuracy": 0.9752475247524752, |
| "expected_accuracy": 0.9752475247524752, |
| "expected_risk": 0.024752475247524774 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 200, |
| "coverage": 0.9900990099009901, |
| "accuracy": 0.975, |
| "expected_accuracy": 0.975, |
| "expected_risk": 0.025000000000000022 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 198, |
| "coverage": 0.9801980198019802, |
| "accuracy": 0.9797979797979798, |
| "expected_accuracy": 0.9797979797979798, |
| "expected_risk": 0.02020202020202022 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 183, |
| "coverage": 0.905940594059406, |
| "accuracy": 0.9781420765027322, |
| "expected_accuracy": 0.9781420765027322, |
| "expected_risk": 0.021857923497267784 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 8, |
| "coverage": 0.039603960396039604, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 202, |
| "hard_count": 202, |
| "accuracy": 0.9752475247524752, |
| "expected_accuracy": 0.9752475247524752, |
| "nll": 0.13723787641738036, |
| "brier": 0.048090580988872436, |
| "expected_brier": 0.048090580988872436, |
| "ece_bins": 15, |
| "multiclass_ece": 0.01462437849252733, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 202, |
| "coverage": 1.0, |
| "accuracy": 0.9752475247524752, |
| "expected_accuracy": 0.9752475247524752, |
| "expected_risk": 0.024752475247524774 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 202, |
| "coverage": 1.0, |
| "accuracy": 0.9752475247524752, |
| "expected_accuracy": 0.9752475247524752, |
| "expected_risk": 0.024752475247524774 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 200, |
| "coverage": 0.9900990099009901, |
| "accuracy": 0.975, |
| "expected_accuracy": 0.975, |
| "expected_risk": 0.025000000000000022 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 198, |
| "coverage": 0.9801980198019802, |
| "accuracy": 0.9797979797979798, |
| "expected_accuracy": 0.9797979797979798, |
| "expected_risk": 0.02020202020202022 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 183, |
| "coverage": 0.905940594059406, |
| "accuracy": 0.9781420765027322, |
| "expected_accuracy": 0.9781420765027322, |
| "expected_risk": 0.021857923497267784 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 8, |
| "coverage": 0.039603960396039604, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| }, |
| "trex_runner-v1": { |
| "count": 4, |
| "successful": 4, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 4, |
| "accuracy_all_rows_failures_incorrect": 0.25, |
| "expected_accuracy_all_rows_failures_zero": 0.25, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 4, |
| "total": 4, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 2, |
| "total": 4, |
| "coverage": 0.5 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 2, |
| "total": 4, |
| "coverage": 0.5 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1, |
| "total": 4, |
| "coverage": 0.25 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 0, |
| "total": 4, |
| "coverage": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 0, |
| "total": 4, |
| "coverage": 0.0 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 4, |
| "hard_count": 4, |
| "accuracy": 0.25, |
| "expected_accuracy": 0.25, |
| "nll": 1.380181719997761, |
| "brier": 0.8197811047574941, |
| "expected_brier": 0.8197811047574941, |
| "ece_bins": 15, |
| "multiclass_ece": 0.47413478912915286, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 4, |
| "coverage": 1.0, |
| "accuracy": 0.25, |
| "expected_accuracy": 0.25, |
| "expected_risk": 0.75 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 2, |
| "coverage": 0.5, |
| "accuracy": 0.5, |
| "expected_accuracy": 0.5, |
| "expected_risk": 0.5 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 2, |
| "coverage": 0.5, |
| "accuracy": 0.5, |
| "expected_accuracy": 0.5, |
| "expected_risk": 0.5 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1, |
| "coverage": 0.25, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 4, |
| "hard_count": 4, |
| "accuracy": 0.25, |
| "expected_accuracy": 0.25, |
| "nll": 1.380181719997761, |
| "brier": 0.8197811047574941, |
| "expected_brier": 0.8197811047574941, |
| "ece_bins": 15, |
| "multiclass_ece": 0.47413478912915286, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 4, |
| "coverage": 1.0, |
| "accuracy": 0.25, |
| "expected_accuracy": 0.25, |
| "expected_risk": 0.75 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 2, |
| "coverage": 0.5, |
| "accuracy": 0.5, |
| "expected_accuracy": 0.5, |
| "expected_risk": 0.5 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 2, |
| "coverage": 0.5, |
| "accuracy": 0.5, |
| "expected_accuracy": 0.5, |
| "expected_risk": 0.5 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1, |
| "coverage": 0.25, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| } |
| ] |
| } |
| }, |
| "vizdoom-basic-v1": { |
| "count": 858, |
| "successful": 858, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 858, |
| "accuracy_all_rows_failures_incorrect": 0.9685314685314685, |
| "expected_accuracy_all_rows_failures_zero": 0.9685314685314685, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 853, |
| "total": 858, |
| "coverage": 0.9941724941724942 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 834, |
| "total": 858, |
| "coverage": 0.972027972027972 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 821, |
| "total": 858, |
| "coverage": 0.9568764568764568 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 800, |
| "total": 858, |
| "coverage": 0.9324009324009324 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 752, |
| "total": 858, |
| "coverage": 0.8764568764568764 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 480, |
| "total": 858, |
| "coverage": 0.5594405594405595 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 858, |
| "hard_count": 858, |
| "accuracy": 0.9685314685314685, |
| "expected_accuracy": 0.9685314685314685, |
| "nll": 0.08747268287742492, |
| "brier": 0.04462544178795084, |
| "expected_brier": 0.04462544178795084, |
| "ece_bins": 15, |
| "multiclass_ece": 0.0214576175261959, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 853, |
| "coverage": 0.9941724941724942, |
| "accuracy": 0.9718640093786636, |
| "expected_accuracy": 0.9718640093786636, |
| "expected_risk": 0.028135990621336426 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 834, |
| "coverage": 0.972027972027972, |
| "accuracy": 0.9844124700239808, |
| "expected_accuracy": 0.9844124700239808, |
| "expected_risk": 0.015587529976019199 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 821, |
| "coverage": 0.9568764568764568, |
| "accuracy": 0.9866017052375152, |
| "expected_accuracy": 0.9866017052375152, |
| "expected_risk": 0.013398294762484775 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 800, |
| "coverage": 0.9324009324009324, |
| "accuracy": 0.99375, |
| "expected_accuracy": 0.99375, |
| "expected_risk": 0.006249999999999978 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 752, |
| "coverage": 0.8764568764568764, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 480, |
| "coverage": 0.5594405594405595, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 858, |
| "hard_count": 858, |
| "accuracy": 0.9685314685314685, |
| "expected_accuracy": 0.9685314685314685, |
| "nll": 0.08747268287742492, |
| "brier": 0.04462544178795084, |
| "expected_brier": 0.04462544178795084, |
| "ece_bins": 15, |
| "multiclass_ece": 0.0214576175261959, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 853, |
| "coverage": 0.9941724941724942, |
| "accuracy": 0.9718640093786636, |
| "expected_accuracy": 0.9718640093786636, |
| "expected_risk": 0.028135990621336426 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 834, |
| "coverage": 0.972027972027972, |
| "accuracy": 0.9844124700239808, |
| "expected_accuracy": 0.9844124700239808, |
| "expected_risk": 0.015587529976019199 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 821, |
| "coverage": 0.9568764568764568, |
| "accuracy": 0.9866017052375152, |
| "expected_accuracy": 0.9866017052375152, |
| "expected_risk": 0.013398294762484775 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 800, |
| "coverage": 0.9324009324009324, |
| "accuracy": 0.99375, |
| "expected_accuracy": 0.99375, |
| "expected_risk": 0.006249999999999978 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 752, |
| "coverage": 0.8764568764568764, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 480, |
| "coverage": 0.5594405594405595, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| }, |
| "wikispeedia-v1": { |
| "count": 176, |
| "successful": 176, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 51, |
| "accuracy_all_rows_failures_incorrect": 0.39215686274509803, |
| "expected_accuracy_all_rows_failures_zero": 0.25494105011150464, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1, |
| "total": 176, |
| "coverage": 0.005681818181818182 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 0, |
| "total": 176, |
| "coverage": 0.0 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 0, |
| "total": 176, |
| "coverage": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 0, |
| "total": 176, |
| "coverage": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 0, |
| "total": 176, |
| "coverage": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 0, |
| "total": 176, |
| "coverage": 0.0 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 176, |
| "hard_count": 51, |
| "accuracy": 0.39215686274509803, |
| "expected_accuracy": 0.25494105011150464, |
| "nll": 2.2057266068313712, |
| "brier": 0.32864256792900676, |
| "expected_brier": 0.862757253748238, |
| "ece_bins": 15, |
| "multiclass_ece": 0.09196286653058698, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1, |
| "coverage": 0.005681818181818182, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 176, |
| "hard_count": 51, |
| "accuracy": 0.39215686274509803, |
| "expected_accuracy": 0.25494105011150464, |
| "nll": 2.2057266068313712, |
| "brier": 0.32864256792900676, |
| "expected_brier": 0.862757253748238, |
| "ece_bins": 15, |
| "multiclass_ece": 0.09196286653058698, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1, |
| "coverage": 0.005681818181818182, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| } |
| ] |
| } |
| }, |
| "workflow-controls-v1/agent_trace_observability": { |
| "count": 357, |
| "successful": 357, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 357, |
| "accuracy_all_rows_failures_incorrect": 0.988795518207283, |
| "expected_accuracy_all_rows_failures_zero": 0.988795518207283, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 357, |
| "total": 357, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 355, |
| "total": 357, |
| "coverage": 0.9943977591036415 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 354, |
| "total": 357, |
| "coverage": 0.9915966386554622 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 353, |
| "total": 357, |
| "coverage": 0.988795518207283 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 352, |
| "total": 357, |
| "coverage": 0.9859943977591037 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 301, |
| "total": 357, |
| "coverage": 0.8431372549019608 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 357, |
| "hard_count": 357, |
| "accuracy": 0.988795518207283, |
| "expected_accuracy": 0.988795518207283, |
| "nll": 0.026771587416750012, |
| "brier": 0.0157428306754681, |
| "expected_brier": 0.0157428306754681, |
| "ece_bins": 15, |
| "multiclass_ece": 0.006613012775607079, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 357, |
| "coverage": 1.0, |
| "accuracy": 0.988795518207283, |
| "expected_accuracy": 0.988795518207283, |
| "expected_risk": 0.011204481792717047 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 355, |
| "coverage": 0.9943977591036415, |
| "accuracy": 0.9915492957746479, |
| "expected_accuracy": 0.9915492957746479, |
| "expected_risk": 0.008450704225352101 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 354, |
| "coverage": 0.9915966386554622, |
| "accuracy": 0.9943502824858758, |
| "expected_accuracy": 0.9943502824858758, |
| "expected_risk": 0.005649717514124242 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 353, |
| "coverage": 0.988795518207283, |
| "accuracy": 0.9971671388101983, |
| "expected_accuracy": 0.9971671388101983, |
| "expected_risk": 0.0028328611898017497 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 352, |
| "coverage": 0.9859943977591037, |
| "accuracy": 0.9971590909090909, |
| "expected_accuracy": 0.9971590909090909, |
| "expected_risk": 0.0028409090909090606 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 301, |
| "coverage": 0.8431372549019608, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 357, |
| "hard_count": 357, |
| "accuracy": 0.988795518207283, |
| "expected_accuracy": 0.988795518207283, |
| "nll": 0.026771587416750012, |
| "brier": 0.0157428306754681, |
| "expected_brier": 0.0157428306754681, |
| "ece_bins": 15, |
| "multiclass_ece": 0.006613012775607079, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 357, |
| "coverage": 1.0, |
| "accuracy": 0.988795518207283, |
| "expected_accuracy": 0.988795518207283, |
| "expected_risk": 0.011204481792717047 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 355, |
| "coverage": 0.9943977591036415, |
| "accuracy": 0.9915492957746479, |
| "expected_accuracy": 0.9915492957746479, |
| "expected_risk": 0.008450704225352101 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 354, |
| "coverage": 0.9915966386554622, |
| "accuracy": 0.9943502824858758, |
| "expected_accuracy": 0.9943502824858758, |
| "expected_risk": 0.005649717514124242 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 353, |
| "coverage": 0.988795518207283, |
| "accuracy": 0.9971671388101983, |
| "expected_accuracy": 0.9971671388101983, |
| "expected_risk": 0.0028328611898017497 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 352, |
| "coverage": 0.9859943977591037, |
| "accuracy": 0.9971590909090909, |
| "expected_accuracy": 0.9971590909090909, |
| "expected_risk": 0.0028409090909090606 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 301, |
| "coverage": 0.8431372549019608, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| }, |
| "workflow-controls-v1/customer_service": { |
| "count": 456, |
| "successful": 456, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 456, |
| "accuracy_all_rows_failures_incorrect": 0.9912280701754386, |
| "expected_accuracy_all_rows_failures_zero": 0.9912280701754386, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 456, |
| "total": 456, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 455, |
| "total": 456, |
| "coverage": 0.9978070175438597 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 454, |
| "total": 456, |
| "coverage": 0.9956140350877193 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 454, |
| "total": 456, |
| "coverage": 0.9956140350877193 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 452, |
| "total": 456, |
| "coverage": 0.9912280701754386 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 382, |
| "total": 456, |
| "coverage": 0.8377192982456141 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 456, |
| "hard_count": 456, |
| "accuracy": 0.9912280701754386, |
| "expected_accuracy": 0.9912280701754386, |
| "nll": 0.024863508782455843, |
| "brier": 0.012699999145872375, |
| "expected_brier": 0.012699999145872375, |
| "ece_bins": 15, |
| "multiclass_ece": 0.004145846144954706, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 456, |
| "coverage": 1.0, |
| "accuracy": 0.9912280701754386, |
| "expected_accuracy": 0.9912280701754386, |
| "expected_risk": 0.00877192982456143 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 455, |
| "coverage": 0.9978070175438597, |
| "accuracy": 0.9934065934065934, |
| "expected_accuracy": 0.9934065934065934, |
| "expected_risk": 0.00659340659340657 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 454, |
| "coverage": 0.9956140350877193, |
| "accuracy": 0.9955947136563876, |
| "expected_accuracy": 0.9955947136563876, |
| "expected_risk": 0.004405286343612369 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 454, |
| "coverage": 0.9956140350877193, |
| "accuracy": 0.9955947136563876, |
| "expected_accuracy": 0.9955947136563876, |
| "expected_risk": 0.004405286343612369 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 452, |
| "coverage": 0.9912280701754386, |
| "accuracy": 0.995575221238938, |
| "expected_accuracy": 0.995575221238938, |
| "expected_risk": 0.004424778761061954 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 382, |
| "coverage": 0.8377192982456141, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 456, |
| "hard_count": 456, |
| "accuracy": 0.9912280701754386, |
| "expected_accuracy": 0.9912280701754386, |
| "nll": 0.024863508782455843, |
| "brier": 0.012699999145872375, |
| "expected_brier": 0.012699999145872375, |
| "ece_bins": 15, |
| "multiclass_ece": 0.004145846144954706, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 456, |
| "coverage": 1.0, |
| "accuracy": 0.9912280701754386, |
| "expected_accuracy": 0.9912280701754386, |
| "expected_risk": 0.00877192982456143 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 455, |
| "coverage": 0.9978070175438597, |
| "accuracy": 0.9934065934065934, |
| "expected_accuracy": 0.9934065934065934, |
| "expected_risk": 0.00659340659340657 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 454, |
| "coverage": 0.9956140350877193, |
| "accuracy": 0.9955947136563876, |
| "expected_accuracy": 0.9955947136563876, |
| "expected_risk": 0.004405286343612369 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 454, |
| "coverage": 0.9956140350877193, |
| "accuracy": 0.9955947136563876, |
| "expected_accuracy": 0.9955947136563876, |
| "expected_risk": 0.004405286343612369 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 452, |
| "coverage": 0.9912280701754386, |
| "accuracy": 0.995575221238938, |
| "expected_accuracy": 0.995575221238938, |
| "expected_risk": 0.004424778761061954 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 382, |
| "coverage": 0.8377192982456141, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| }, |
| "workflow-controls-v1/invoice_processing": { |
| "count": 930, |
| "successful": 930, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 930, |
| "accuracy_all_rows_failures_incorrect": 0.9989247311827957, |
| "expected_accuracy_all_rows_failures_zero": 0.9989247311827957, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 930, |
| "total": 930, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 926, |
| "total": 930, |
| "coverage": 0.9956989247311828 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 924, |
| "total": 930, |
| "coverage": 0.9935483870967742 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 919, |
| "total": 930, |
| "coverage": 0.9881720430107527 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 902, |
| "total": 930, |
| "coverage": 0.9698924731182795 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 732, |
| "total": 930, |
| "coverage": 0.7870967741935484 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 930, |
| "hard_count": 930, |
| "accuracy": 0.9989247311827957, |
| "expected_accuracy": 0.9989247311827957, |
| "nll": 0.010428173350270523, |
| "brier": 0.0031761278683533505, |
| "expected_brier": 0.0031761278683533475, |
| "ece_bins": 15, |
| "multiclass_ece": 0.009015233488700234, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 930, |
| "coverage": 1.0, |
| "accuracy": 0.9989247311827957, |
| "expected_accuracy": 0.9989247311827957, |
| "expected_risk": 0.0010752688172043223 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 926, |
| "coverage": 0.9956989247311828, |
| "accuracy": 0.9989200863930886, |
| "expected_accuracy": 0.9989200863930886, |
| "expected_risk": 0.001079913606911398 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 924, |
| "coverage": 0.9935483870967742, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 919, |
| "coverage": 0.9881720430107527, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 902, |
| "coverage": 0.9698924731182795, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 732, |
| "coverage": 0.7870967741935484, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 930, |
| "hard_count": 930, |
| "accuracy": 0.9989247311827957, |
| "expected_accuracy": 0.9989247311827957, |
| "nll": 0.010428173350270523, |
| "brier": 0.0031761278683533505, |
| "expected_brier": 0.0031761278683533475, |
| "ece_bins": 15, |
| "multiclass_ece": 0.009015233488700234, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 930, |
| "coverage": 1.0, |
| "accuracy": 0.9989247311827957, |
| "expected_accuracy": 0.9989247311827957, |
| "expected_risk": 0.0010752688172043223 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 926, |
| "coverage": 0.9956989247311828, |
| "accuracy": 0.9989200863930886, |
| "expected_accuracy": 0.9989200863930886, |
| "expected_risk": 0.001079913606911398 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 924, |
| "coverage": 0.9935483870967742, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 919, |
| "coverage": 0.9881720430107527, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 902, |
| "coverage": 0.9698924731182795, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 732, |
| "coverage": 0.7870967741935484, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| }, |
| "workflow-controls-v1/security_incidents": { |
| "count": 1326, |
| "successful": 1326, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 1326, |
| "accuracy_all_rows_failures_incorrect": 0.9962292609351433, |
| "expected_accuracy_all_rows_failures_zero": 0.9962292609351433, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1326, |
| "total": 1326, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1324, |
| "total": 1326, |
| "coverage": 0.9984917043740573 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1323, |
| "total": 1326, |
| "coverage": 0.997737556561086 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1322, |
| "total": 1326, |
| "coverage": 0.9969834087481146 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1295, |
| "total": 1326, |
| "coverage": 0.9766214177978884 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1228, |
| "total": 1326, |
| "coverage": 0.9260935143288085 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 1326, |
| "hard_count": 1326, |
| "accuracy": 0.9962292609351433, |
| "expected_accuracy": 0.9962292609351433, |
| "nll": 0.014165561745883562, |
| "brier": 0.0066243147587275175, |
| "expected_brier": 0.006624314758727516, |
| "ece_bins": 15, |
| "multiclass_ece": 0.001806122714016116, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1326, |
| "coverage": 1.0, |
| "accuracy": 0.9962292609351433, |
| "expected_accuracy": 0.9962292609351433, |
| "expected_risk": 0.0037707390648566985 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1324, |
| "coverage": 0.9984917043740573, |
| "accuracy": 0.9969788519637462, |
| "expected_accuracy": 0.9969788519637462, |
| "expected_risk": 0.0030211480362537513 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1323, |
| "coverage": 0.997737556561086, |
| "accuracy": 0.9969765684051398, |
| "expected_accuracy": 0.9969765684051398, |
| "expected_risk": 0.003023431594860182 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1322, |
| "coverage": 0.9969834087481146, |
| "accuracy": 0.9969742813918305, |
| "expected_accuracy": 0.9969742813918305, |
| "expected_risk": 0.00302571860816947 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1295, |
| "coverage": 0.9766214177978884, |
| "accuracy": 0.9984555984555985, |
| "expected_accuracy": 0.9984555984555985, |
| "expected_risk": 0.001544401544401497 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1228, |
| "coverage": 0.9260935143288085, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 1326, |
| "hard_count": 1326, |
| "accuracy": 0.9962292609351433, |
| "expected_accuracy": 0.9962292609351433, |
| "nll": 0.014165561745883562, |
| "brier": 0.0066243147587275175, |
| "expected_brier": 0.006624314758727516, |
| "ece_bins": 15, |
| "multiclass_ece": 0.001806122714016116, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1326, |
| "coverage": 1.0, |
| "accuracy": 0.9962292609351433, |
| "expected_accuracy": 0.9962292609351433, |
| "expected_risk": 0.0037707390648566985 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1324, |
| "coverage": 0.9984917043740573, |
| "accuracy": 0.9969788519637462, |
| "expected_accuracy": 0.9969788519637462, |
| "expected_risk": 0.0030211480362537513 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1323, |
| "coverage": 0.997737556561086, |
| "accuracy": 0.9969765684051398, |
| "expected_accuracy": 0.9969765684051398, |
| "expected_risk": 0.003023431594860182 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1322, |
| "coverage": 0.9969834087481146, |
| "accuracy": 0.9969742813918305, |
| "expected_accuracy": 0.9969742813918305, |
| "expected_risk": 0.00302571860816947 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1295, |
| "coverage": 0.9766214177978884, |
| "accuracy": 0.9984555984555985, |
| "expected_accuracy": 0.9984555984555985, |
| "expected_risk": 0.001544401544401497 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1228, |
| "coverage": 0.9260935143288085, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| } |
| }, |
| "by_kind": { |
| "choice": { |
| "count": 2584, |
| "successful": 2584, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 2119, |
| "accuracy_all_rows_failures_incorrect": 0.8744690891930156, |
| "expected_accuracy_all_rows_failures_zero": 0.7789374561626884, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1948, |
| "total": 2584, |
| "coverage": 0.7538699690402477 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1703, |
| "total": 2584, |
| "coverage": 0.6590557275541795 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1648, |
| "total": 2584, |
| "coverage": 0.6377708978328174 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1550, |
| "total": 2584, |
| "coverage": 0.5998452012383901 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1399, |
| "total": 2584, |
| "coverage": 0.5414086687306502 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 936, |
| "total": 2584, |
| "coverage": 0.3622291021671827 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 2584, |
| "hard_count": 2119, |
| "accuracy": 0.8744690891930156, |
| "expected_accuracy": 0.7789374561626884, |
| "nll": 0.5182847492916147, |
| "brier": 0.1546043322635334, |
| "expected_brier": 0.26454992932213356, |
| "ece_bins": 15, |
| "multiclass_ece": 0.02196788032076633, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1948, |
| "coverage": 0.7538699690402477, |
| "accuracy": 0.9578480310593456, |
| "expected_accuracy": 0.9223134839151267, |
| "expected_risk": 0.07768651608487331 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1703, |
| "coverage": 0.6590557275541795, |
| "accuracy": 0.9815146094215862, |
| "expected_accuracy": 0.9736739087884126, |
| "expected_risk": 0.026326091211587355 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1648, |
| "coverage": 0.6377708978328174, |
| "accuracy": 0.9852670349907919, |
| "expected_accuracy": 0.9792677993527508, |
| "expected_risk": 0.020732200647249233 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1550, |
| "coverage": 0.5998452012383901, |
| "accuracy": 0.9922380336351876, |
| "expected_accuracy": 0.9909677419354839, |
| "expected_risk": 0.00903225806451613 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1399, |
| "coverage": 0.5414086687306502, |
| "accuracy": 0.9964260185847034, |
| "expected_accuracy": 0.9964260185847034, |
| "expected_risk": 0.0035739814152966343 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 936, |
| "coverage": 0.3622291021671827, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 2584, |
| "hard_count": 2119, |
| "accuracy": 0.8744690891930156, |
| "expected_accuracy": 0.7789374561626884, |
| "nll": 0.5182847492916147, |
| "brier": 0.1546043322635334, |
| "expected_brier": 0.26454992932213356, |
| "ece_bins": 15, |
| "multiclass_ece": 0.02196788032076633, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1948, |
| "coverage": 0.7538699690402477, |
| "accuracy": 0.9578480310593456, |
| "expected_accuracy": 0.9223134839151267, |
| "expected_risk": 0.07768651608487331 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1703, |
| "coverage": 0.6590557275541795, |
| "accuracy": 0.9815146094215862, |
| "expected_accuracy": 0.9736739087884126, |
| "expected_risk": 0.026326091211587355 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1648, |
| "coverage": 0.6377708978328174, |
| "accuracy": 0.9852670349907919, |
| "expected_accuracy": 0.9792677993527508, |
| "expected_risk": 0.020732200647249233 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1550, |
| "coverage": 0.5998452012383901, |
| "accuracy": 0.9922380336351876, |
| "expected_accuracy": 0.9909677419354839, |
| "expected_risk": 0.00903225806451613 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1399, |
| "coverage": 0.5414086687306502, |
| "accuracy": 0.9964260185847034, |
| "expected_accuracy": 0.9964260185847034, |
| "expected_risk": 0.0035739814152966343 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 936, |
| "coverage": 0.3622291021671827, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| }, |
| "noul": { |
| "count": 6364, |
| "successful": 6364, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 6364, |
| "accuracy_all_rows_failures_incorrect": 0.9693588937774984, |
| "expected_accuracy_all_rows_failures_zero": 0.9693588937774984, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 6364, |
| "total": 6364, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 6302, |
| "total": 6364, |
| "coverage": 0.9902576995600252 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 6255, |
| "total": 6364, |
| "coverage": 0.982872407291012 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 6091, |
| "total": 6364, |
| "coverage": 0.9571024512884978 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 5801, |
| "total": 6364, |
| "coverage": 0.9115336266499057 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 4545, |
| "total": 6364, |
| "coverage": 0.7141734758013828 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 6364, |
| "hard_count": 6364, |
| "accuracy": 0.9693588937774984, |
| "expected_accuracy": 0.9693588937774984, |
| "nll": 0.08299025396923065, |
| "brier": 0.04881816041081464, |
| "expected_brier": 0.04881816041081464, |
| "ece_bins": 15, |
| "multiclass_ece": 0.01363401934594279, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 6364, |
| "coverage": 1.0, |
| "accuracy": 0.9693588937774984, |
| "expected_accuracy": 0.9693588937774984, |
| "expected_risk": 0.030641106222501602 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 6302, |
| "coverage": 0.9902576995600252, |
| "accuracy": 0.973341796255157, |
| "expected_accuracy": 0.973341796255157, |
| "expected_risk": 0.026658203744842957 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 6255, |
| "coverage": 0.982872407291012, |
| "accuracy": 0.9761790567545964, |
| "expected_accuracy": 0.9761790567545964, |
| "expected_risk": 0.023820943245403625 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 6091, |
| "coverage": 0.9571024512884978, |
| "accuracy": 0.9860449844032179, |
| "expected_accuracy": 0.9860449844032179, |
| "expected_risk": 0.013955015596782139 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 5801, |
| "coverage": 0.9115336266499057, |
| "accuracy": 0.9937941734183762, |
| "expected_accuracy": 0.9937941734183762, |
| "expected_risk": 0.00620582658162383 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 4545, |
| "coverage": 0.7141734758013828, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 6364, |
| "hard_count": 6364, |
| "accuracy": 0.9693588937774984, |
| "expected_accuracy": 0.9693588937774984, |
| "nll": 0.08299025396923065, |
| "brier": 0.04881816041081464, |
| "expected_brier": 0.04881816041081464, |
| "ece_bins": 15, |
| "multiclass_ece": 0.01363401934594279, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 6364, |
| "coverage": 1.0, |
| "accuracy": 0.9693588937774984, |
| "expected_accuracy": 0.9693588937774984, |
| "expected_risk": 0.030641106222501602 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 6302, |
| "coverage": 0.9902576995600252, |
| "accuracy": 0.973341796255157, |
| "expected_accuracy": 0.973341796255157, |
| "expected_risk": 0.026658203744842957 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 6255, |
| "coverage": 0.982872407291012, |
| "accuracy": 0.9761790567545964, |
| "expected_accuracy": 0.9761790567545964, |
| "expected_risk": 0.023820943245403625 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 6091, |
| "coverage": 0.9571024512884978, |
| "accuracy": 0.9860449844032179, |
| "expected_accuracy": 0.9860449844032179, |
| "expected_risk": 0.013955015596782139 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 5801, |
| "coverage": 0.9115336266499057, |
| "accuracy": 0.9937941734183762, |
| "expected_accuracy": 0.9937941734183762, |
| "expected_risk": 0.00620582658162383 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 4545, |
| "coverage": 0.7141734758013828, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| }, |
| "score": { |
| "count": 1584, |
| "successful": 1584, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 1563, |
| "accuracy_all_rows_failures_incorrect": 0.9552143314139475, |
| "expected_accuracy_all_rows_failures_zero": 0.9486531986531987, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1518, |
| "total": 1584, |
| "coverage": 0.9583333333333334 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1495, |
| "total": 1584, |
| "coverage": 0.9438131313131313 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1485, |
| "total": 1584, |
| "coverage": 0.9375 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1475, |
| "total": 1584, |
| "coverage": 0.9311868686868687 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1434, |
| "total": 1584, |
| "coverage": 0.9053030303030303 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1261, |
| "total": 1584, |
| "coverage": 0.7960858585858586 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 1584, |
| "hard_count": 1563, |
| "accuracy": 0.9552143314139475, |
| "expected_accuracy": 0.9486531986531987, |
| "nll": 0.1201671405517328, |
| "brier": 0.05775733534507227, |
| "expected_brier": 0.06491221749995442, |
| "ece_bins": 15, |
| "multiclass_ece": 0.014709357196524655, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1518, |
| "coverage": 0.9583333333333334, |
| "accuracy": 0.978709248170326, |
| "expected_accuracy": 0.9739789196310935, |
| "expected_risk": 0.02602108036890649 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1495, |
| "coverage": 0.9438131313131313, |
| "accuracy": 0.9832775919732442, |
| "expected_accuracy": 0.9832775919732442, |
| "expected_risk": 0.01672240802675584 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1485, |
| "coverage": 0.9375, |
| "accuracy": 0.9878787878787879, |
| "expected_accuracy": 0.9878787878787879, |
| "expected_risk": 0.012121212121212088 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1475, |
| "coverage": 0.9311868686868687, |
| "accuracy": 0.991864406779661, |
| "expected_accuracy": 0.991864406779661, |
| "expected_risk": 0.008135593220338966 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1434, |
| "coverage": 0.9053030303030303, |
| "accuracy": 0.99860529986053, |
| "expected_accuracy": 0.99860529986053, |
| "expected_risk": 0.001394700139469962 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1261, |
| "coverage": 0.7960858585858586, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 1584, |
| "hard_count": 1563, |
| "accuracy": 0.9552143314139475, |
| "expected_accuracy": 0.9486531986531987, |
| "nll": 0.1201671405517328, |
| "brier": 0.05775733534507227, |
| "expected_brier": 0.06491221749995442, |
| "ece_bins": 15, |
| "multiclass_ece": 0.014709357196524655, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1518, |
| "coverage": 0.9583333333333334, |
| "accuracy": 0.978709248170326, |
| "expected_accuracy": 0.9739789196310935, |
| "expected_risk": 0.02602108036890649 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1495, |
| "coverage": 0.9438131313131313, |
| "accuracy": 0.9832775919732442, |
| "expected_accuracy": 0.9832775919732442, |
| "expected_risk": 0.01672240802675584 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1485, |
| "coverage": 0.9375, |
| "accuracy": 0.9878787878787879, |
| "expected_accuracy": 0.9878787878787879, |
| "expected_risk": 0.012121212121212088 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1475, |
| "coverage": 0.9311868686868687, |
| "accuracy": 0.991864406779661, |
| "expected_accuracy": 0.991864406779661, |
| "expected_risk": 0.008135593220338966 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1434, |
| "coverage": 0.9053030303030303, |
| "accuracy": 0.99860529986053, |
| "expected_accuracy": 0.99860529986053, |
| "expected_risk": 0.001394700139469962 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1261, |
| "coverage": 0.7960858585858586, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| } |
| } |
| }, |
| "ood": { |
| "count": 15920, |
| "successful": 15920, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 15446, |
| "accuracy_all_rows_failures_incorrect": 0.8602227113815875, |
| "expected_accuracy_all_rows_failures_zero": 0.8461552917002038, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 15467, |
| "total": 15920, |
| "coverage": 0.9715452261306533 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 14974, |
| "total": 15920, |
| "coverage": 0.9405778894472362 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 14755, |
| "total": 15920, |
| "coverage": 0.926821608040201 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 14212, |
| "total": 15920, |
| "coverage": 0.892713567839196 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 13285, |
| "total": 15920, |
| "coverage": 0.8344849246231156 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 9996, |
| "total": 15920, |
| "coverage": 0.6278894472361809 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 15920, |
| "hard_count": 15446, |
| "accuracy": 0.8602227113815875, |
| "expected_accuracy": 0.8461552917002038, |
| "nll": 0.8029899145502479, |
| "brier": 0.24645039628025225, |
| "expected_brier": 0.26378113592726077, |
| "ece_bins": 15, |
| "multiclass_ece": 0.10559429485648715, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 15467, |
| "coverage": 0.9715452261306533, |
| "accuracy": 0.8670391061452514, |
| "expected_accuracy": 0.8610051507510613, |
| "expected_risk": 0.13899484924893868 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 14974, |
| "coverage": 0.9405778894472362, |
| "accuracy": 0.8734346748811357, |
| "expected_accuracy": 0.8723899203063087, |
| "expected_risk": 0.12761007969369131 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 14755, |
| "coverage": 0.926821608040201, |
| "accuracy": 0.8785141925845443, |
| "expected_accuracy": 0.8777589517677624, |
| "expected_risk": 0.12224104823223758 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 14212, |
| "coverage": 0.892713567839196, |
| "accuracy": 0.886173135169402, |
| "expected_accuracy": 0.8857655502392344, |
| "expected_risk": 0.11423444976076558 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 13285, |
| "coverage": 0.8344849246231156, |
| "accuracy": 0.8918918918918919, |
| "expected_accuracy": 0.8918328942416259, |
| "expected_risk": 0.10816710575837407 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 9996, |
| "coverage": 0.6278894472361809, |
| "accuracy": 0.8904561824729892, |
| "expected_accuracy": 0.8904561824729892, |
| "expected_risk": 0.10954381752701081 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 15920, |
| "hard_count": 15446, |
| "accuracy": 0.8602227113815875, |
| "expected_accuracy": 0.8461552917002038, |
| "nll": 0.8029899145502479, |
| "brier": 0.24645039628025225, |
| "expected_brier": 0.26378113592726077, |
| "ece_bins": 15, |
| "multiclass_ece": 0.10559429485648715, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 15467, |
| "coverage": 0.9715452261306533, |
| "accuracy": 0.8670391061452514, |
| "expected_accuracy": 0.8610051507510613, |
| "expected_risk": 0.13899484924893868 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 14974, |
| "coverage": 0.9405778894472362, |
| "accuracy": 0.8734346748811357, |
| "expected_accuracy": 0.8723899203063087, |
| "expected_risk": 0.12761007969369131 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 14755, |
| "coverage": 0.926821608040201, |
| "accuracy": 0.8785141925845443, |
| "expected_accuracy": 0.8777589517677624, |
| "expected_risk": 0.12224104823223758 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 14212, |
| "coverage": 0.892713567839196, |
| "accuracy": 0.886173135169402, |
| "expected_accuracy": 0.8857655502392344, |
| "expected_risk": 0.11423444976076558 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 13285, |
| "coverage": 0.8344849246231156, |
| "accuracy": 0.8918918918918919, |
| "expected_accuracy": 0.8918328942416259, |
| "expected_risk": 0.10816710575837407 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 9996, |
| "coverage": 0.6278894472361809, |
| "accuracy": 0.8904561824729892, |
| "expected_accuracy": 0.8904561824729892, |
| "expected_risk": 0.10954381752701081 |
| } |
| ] |
| }, |
| "by_source": { |
| "customer-control-v1": { |
| "count": 600, |
| "successful": 600, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 560, |
| "accuracy_all_rows_failures_incorrect": 0.9732142857142857, |
| "expected_accuracy_all_rows_failures_zero": 0.9383333333333334, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 585, |
| "total": 600, |
| "coverage": 0.975 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 550, |
| "total": 600, |
| "coverage": 0.9166666666666666 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 544, |
| "total": 600, |
| "coverage": 0.9066666666666666 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 534, |
| "total": 600, |
| "coverage": 0.89 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 516, |
| "total": 600, |
| "coverage": 0.86 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 439, |
| "total": 600, |
| "coverage": 0.7316666666666667 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 600, |
| "hard_count": 560, |
| "accuracy": 0.9732142857142857, |
| "expected_accuracy": 0.9383333333333334, |
| "nll": 0.18931729447777135, |
| "brier": 0.053790307653173525, |
| "expected_brier": 0.09045697431984019, |
| "ece_bins": 15, |
| "multiclass_ece": 0.02661442402817384, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 585, |
| "coverage": 0.975, |
| "accuracy": 0.9731663685152058, |
| "expected_accuracy": 0.9521367521367521, |
| "expected_risk": 0.047863247863247915 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 550, |
| "coverage": 0.9166666666666666, |
| "accuracy": 0.9744058500914077, |
| "expected_accuracy": 0.9718181818181818, |
| "expected_risk": 0.028181818181818197 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 544, |
| "coverage": 0.9066666666666666, |
| "accuracy": 0.9760589318600368, |
| "expected_accuracy": 0.9751838235294118, |
| "expected_risk": 0.024816176470588203 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 534, |
| "coverage": 0.89, |
| "accuracy": 0.9756554307116105, |
| "expected_accuracy": 0.9756554307116105, |
| "expected_risk": 0.02434456928838946 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 516, |
| "coverage": 0.86, |
| "accuracy": 0.9786821705426356, |
| "expected_accuracy": 0.9786821705426356, |
| "expected_risk": 0.02131782945736438 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 439, |
| "coverage": 0.7316666666666667, |
| "accuracy": 0.9840546697038725, |
| "expected_accuracy": 0.9840546697038725, |
| "expected_risk": 0.015945330296127547 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 600, |
| "hard_count": 560, |
| "accuracy": 0.9732142857142857, |
| "expected_accuracy": 0.9383333333333334, |
| "nll": 0.18931729447777135, |
| "brier": 0.053790307653173525, |
| "expected_brier": 0.09045697431984019, |
| "ece_bins": 15, |
| "multiclass_ece": 0.02661442402817384, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 585, |
| "coverage": 0.975, |
| "accuracy": 0.9731663685152058, |
| "expected_accuracy": 0.9521367521367521, |
| "expected_risk": 0.047863247863247915 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 550, |
| "coverage": 0.9166666666666666, |
| "accuracy": 0.9744058500914077, |
| "expected_accuracy": 0.9718181818181818, |
| "expected_risk": 0.028181818181818197 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 544, |
| "coverage": 0.9066666666666666, |
| "accuracy": 0.9760589318600368, |
| "expected_accuracy": 0.9751838235294118, |
| "expected_risk": 0.024816176470588203 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 534, |
| "coverage": 0.89, |
| "accuracy": 0.9756554307116105, |
| "expected_accuracy": 0.9756554307116105, |
| "expected_risk": 0.02434456928838946 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 516, |
| "coverage": 0.86, |
| "accuracy": 0.9786821705426356, |
| "expected_accuracy": 0.9786821705426356, |
| "expected_risk": 0.02131782945736438 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 439, |
| "coverage": 0.7316666666666667, |
| "accuracy": 0.9840546697038725, |
| "expected_accuracy": 0.9840546697038725, |
| "expected_risk": 0.015945330296127547 |
| } |
| ] |
| } |
| }, |
| "painting-geometry-v1": { |
| "count": 6912, |
| "successful": 6912, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 6912, |
| "accuracy_all_rows_failures_incorrect": 0.8051215277777778, |
| "expected_accuracy_all_rows_failures_zero": 0.8051215277777778, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 6908, |
| "total": 6912, |
| "coverage": 0.9994212962962963 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 6876, |
| "total": 6912, |
| "coverage": 0.9947916666666666 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 6845, |
| "total": 6912, |
| "coverage": 0.9903067129629629 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 6734, |
| "total": 6912, |
| "coverage": 0.9742476851851852 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 6551, |
| "total": 6912, |
| "coverage": 0.9477719907407407 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 5718, |
| "total": 6912, |
| "coverage": 0.8272569444444444 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 6912, |
| "hard_count": 6912, |
| "accuracy": 0.8051215277777778, |
| "expected_accuracy": 0.8051215277777778, |
| "nll": 1.4325737213947278, |
| "brier": 0.3808534813920613, |
| "expected_brier": 0.3808534813920613, |
| "ece_bins": 15, |
| "multiclass_ece": 0.18400414817622343, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 6908, |
| "coverage": 0.9994212962962963, |
| "accuracy": 0.8051534452808338, |
| "expected_accuracy": 0.8051534452808338, |
| "expected_risk": 0.19484655471916623 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 6876, |
| "coverage": 0.9947916666666666, |
| "accuracy": 0.806282722513089, |
| "expected_accuracy": 0.806282722513089, |
| "expected_risk": 0.19371727748691103 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 6845, |
| "coverage": 0.9903067129629629, |
| "accuracy": 0.8081811541271001, |
| "expected_accuracy": 0.8081811541271001, |
| "expected_risk": 0.19181884587289988 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 6734, |
| "coverage": 0.9742476851851852, |
| "accuracy": 0.8091773091773091, |
| "expected_accuracy": 0.8091773091773091, |
| "expected_risk": 0.19082269082269088 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 6551, |
| "coverage": 0.9477719907407407, |
| "accuracy": 0.8087314913753625, |
| "expected_accuracy": 0.8087314913753625, |
| "expected_risk": 0.1912685086246375 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 5718, |
| "coverage": 0.8272569444444444, |
| "accuracy": 0.8121720881427072, |
| "expected_accuracy": 0.8121720881427072, |
| "expected_risk": 0.18782791185729275 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 6912, |
| "hard_count": 6912, |
| "accuracy": 0.8051215277777778, |
| "expected_accuracy": 0.8051215277777778, |
| "nll": 1.4325737213947278, |
| "brier": 0.3808534813920613, |
| "expected_brier": 0.3808534813920613, |
| "ece_bins": 15, |
| "multiclass_ece": 0.18400414817622343, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 6908, |
| "coverage": 0.9994212962962963, |
| "accuracy": 0.8051534452808338, |
| "expected_accuracy": 0.8051534452808338, |
| "expected_risk": 0.19484655471916623 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 6876, |
| "coverage": 0.9947916666666666, |
| "accuracy": 0.806282722513089, |
| "expected_accuracy": 0.806282722513089, |
| "expected_risk": 0.19371727748691103 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 6845, |
| "coverage": 0.9903067129629629, |
| "accuracy": 0.8081811541271001, |
| "expected_accuracy": 0.8081811541271001, |
| "expected_risk": 0.19181884587289988 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 6734, |
| "coverage": 0.9742476851851852, |
| "accuracy": 0.8091773091773091, |
| "expected_accuracy": 0.8091773091773091, |
| "expected_risk": 0.19082269082269088 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 6551, |
| "coverage": 0.9477719907407407, |
| "accuracy": 0.8087314913753625, |
| "expected_accuracy": 0.8087314913753625, |
| "expected_risk": 0.1912685086246375 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 5718, |
| "coverage": 0.8272569444444444, |
| "accuracy": 0.8121720881427072, |
| "expected_accuracy": 0.8121720881427072, |
| "expected_risk": 0.18782791185729275 |
| } |
| ] |
| } |
| }, |
| "reasoning-control-v1": { |
| "count": 750, |
| "successful": 750, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 750, |
| "accuracy_all_rows_failures_incorrect": 0.6693333333333333, |
| "expected_accuracy_all_rows_failures_zero": 0.6693333333333333, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 617, |
| "total": 750, |
| "coverage": 0.8226666666666667 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 533, |
| "total": 750, |
| "coverage": 0.7106666666666667 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 468, |
| "total": 750, |
| "coverage": 0.624 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 375, |
| "total": 750, |
| "coverage": 0.5 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 268, |
| "total": 750, |
| "coverage": 0.35733333333333334 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 113, |
| "total": 750, |
| "coverage": 0.15066666666666667 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 750, |
| "hard_count": 750, |
| "accuracy": 0.6693333333333333, |
| "expected_accuracy": 0.6693333333333333, |
| "nll": 0.8174632190648329, |
| "brier": 0.4497004885456776, |
| "expected_brier": 0.4497004885456775, |
| "ece_bins": 15, |
| "multiclass_ece": 0.12814111983806595, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 617, |
| "coverage": 0.8226666666666667, |
| "accuracy": 0.7439222042139384, |
| "expected_accuracy": 0.7439222042139384, |
| "expected_risk": 0.2560777957860616 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 533, |
| "coverage": 0.7106666666666667, |
| "accuracy": 0.774859287054409, |
| "expected_accuracy": 0.774859287054409, |
| "expected_risk": 0.22514071294559101 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 468, |
| "coverage": 0.624, |
| "accuracy": 0.8141025641025641, |
| "expected_accuracy": 0.8141025641025641, |
| "expected_risk": 0.1858974358974359 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 375, |
| "coverage": 0.5, |
| "accuracy": 0.856, |
| "expected_accuracy": 0.856, |
| "expected_risk": 0.14400000000000002 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 268, |
| "coverage": 0.35733333333333334, |
| "accuracy": 0.9104477611940298, |
| "expected_accuracy": 0.9104477611940298, |
| "expected_risk": 0.08955223880597019 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 113, |
| "coverage": 0.15066666666666667, |
| "accuracy": 0.9557522123893806, |
| "expected_accuracy": 0.9557522123893806, |
| "expected_risk": 0.04424778761061943 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 750, |
| "hard_count": 750, |
| "accuracy": 0.6693333333333333, |
| "expected_accuracy": 0.6693333333333333, |
| "nll": 0.8174632190648329, |
| "brier": 0.4497004885456776, |
| "expected_brier": 0.4497004885456775, |
| "ece_bins": 15, |
| "multiclass_ece": 0.12814111983806595, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 617, |
| "coverage": 0.8226666666666667, |
| "accuracy": 0.7439222042139384, |
| "expected_accuracy": 0.7439222042139384, |
| "expected_risk": 0.2560777957860616 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 533, |
| "coverage": 0.7106666666666667, |
| "accuracy": 0.774859287054409, |
| "expected_accuracy": 0.774859287054409, |
| "expected_risk": 0.22514071294559101 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 468, |
| "coverage": 0.624, |
| "accuracy": 0.8141025641025641, |
| "expected_accuracy": 0.8141025641025641, |
| "expected_risk": 0.1858974358974359 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 375, |
| "coverage": 0.5, |
| "accuracy": 0.856, |
| "expected_accuracy": 0.856, |
| "expected_risk": 0.14400000000000002 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 268, |
| "coverage": 0.35733333333333334, |
| "accuracy": 0.9104477611940298, |
| "expected_accuracy": 0.9104477611940298, |
| "expected_risk": 0.08955223880597019 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 113, |
| "coverage": 0.15066666666666667, |
| "accuracy": 0.9557522123893806, |
| "expected_accuracy": 0.9557522123893806, |
| "expected_risk": 0.04424778761061943 |
| } |
| ] |
| } |
| }, |
| "snake-v1": { |
| "count": 3464, |
| "successful": 3464, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 3192, |
| "accuracy_all_rows_failures_incorrect": 0.868421052631579, |
| "expected_accuracy_all_rows_failures_zero": 0.8391070053887606, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 3389, |
| "total": 3464, |
| "coverage": 0.9783487297921478 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 3093, |
| "total": 3464, |
| "coverage": 0.8928983833718245 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 3011, |
| "total": 3464, |
| "coverage": 0.8692263279445728 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 2735, |
| "total": 3464, |
| "coverage": 0.7895496535796767 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 2217, |
| "total": 3464, |
| "coverage": 0.6400115473441108 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 627, |
| "total": 3464, |
| "coverage": 0.18100461893764433 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 3464, |
| "hard_count": 3192, |
| "accuracy": 0.868421052631579, |
| "expected_accuracy": 0.8391070053887606, |
| "nll": 0.40090561564728155, |
| "brier": 0.20086908723576663, |
| "expected_brier": 0.24022628508026242, |
| "ece_bins": 15, |
| "multiclass_ece": 0.07383729789301127, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 3389, |
| "coverage": 0.9783487297921478, |
| "accuracy": 0.8714060031595576, |
| "expected_accuracy": 0.8466115865053604, |
| "expected_risk": 0.1533884134946396 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 3093, |
| "coverage": 0.8928983833718245, |
| "accuracy": 0.8851063829787233, |
| "expected_accuracy": 0.8802672701799762, |
| "expected_risk": 0.11973272982002381 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 3011, |
| "coverage": 0.8692263279445728, |
| "accuracy": 0.89171974522293, |
| "expected_accuracy": 0.8880216982176464, |
| "expected_risk": 0.11197830178235357 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 2735, |
| "coverage": 0.7895496535796767, |
| "accuracy": 0.9158088235294117, |
| "expected_accuracy": 0.913528336380256, |
| "expected_risk": 0.08647166361974401 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 2217, |
| "coverage": 0.6400115473441108, |
| "accuracy": 0.944018058690745, |
| "expected_accuracy": 0.94361750112765, |
| "expected_risk": 0.05638249887235003 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 627, |
| "coverage": 0.18100461893764433, |
| "accuracy": 0.9936204146730463, |
| "expected_accuracy": 0.9936204146730463, |
| "expected_risk": 0.006379585326953707 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 3464, |
| "hard_count": 3192, |
| "accuracy": 0.868421052631579, |
| "expected_accuracy": 0.8391070053887606, |
| "nll": 0.40090561564728155, |
| "brier": 0.20086908723576663, |
| "expected_brier": 0.24022628508026242, |
| "ece_bins": 15, |
| "multiclass_ece": 0.07383729789301127, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 3389, |
| "coverage": 0.9783487297921478, |
| "accuracy": 0.8714060031595576, |
| "expected_accuracy": 0.8466115865053604, |
| "expected_risk": 0.1533884134946396 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 3093, |
| "coverage": 0.8928983833718245, |
| "accuracy": 0.8851063829787233, |
| "expected_accuracy": 0.8802672701799762, |
| "expected_risk": 0.11973272982002381 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 3011, |
| "coverage": 0.8692263279445728, |
| "accuracy": 0.89171974522293, |
| "expected_accuracy": 0.8880216982176464, |
| "expected_risk": 0.11197830178235357 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 2735, |
| "coverage": 0.7895496535796767, |
| "accuracy": 0.9158088235294117, |
| "expected_accuracy": 0.913528336380256, |
| "expected_risk": 0.08647166361974401 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 2217, |
| "coverage": 0.6400115473441108, |
| "accuracy": 0.944018058690745, |
| "expected_accuracy": 0.94361750112765, |
| "expected_risk": 0.05638249887235003 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 627, |
| "coverage": 0.18100461893764433, |
| "accuracy": 0.9936204146730463, |
| "expected_accuracy": 0.9936204146730463, |
| "expected_risk": 0.006379585326953707 |
| } |
| ] |
| } |
| }, |
| "vizdoom-basic-v1": { |
| "count": 825, |
| "successful": 825, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 825, |
| "accuracy_all_rows_failures_incorrect": 0.9309090909090909, |
| "expected_accuracy_all_rows_failures_zero": 0.9309090909090909, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 816, |
| "total": 825, |
| "coverage": 0.9890909090909091 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 788, |
| "total": 825, |
| "coverage": 0.9551515151515152 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 765, |
| "total": 825, |
| "coverage": 0.9272727272727272 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 741, |
| "total": 825, |
| "coverage": 0.8981818181818182 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 693, |
| "total": 825, |
| "coverage": 0.84 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 457, |
| "total": 825, |
| "coverage": 0.553939393939394 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 825, |
| "hard_count": 825, |
| "accuracy": 0.9309090909090909, |
| "expected_accuracy": 0.9309090909090909, |
| "nll": 0.18737228735728542, |
| "brier": 0.09844207618854224, |
| "expected_brier": 0.09844207618854224, |
| "ece_bins": 15, |
| "multiclass_ece": 0.0250802098487241, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 816, |
| "coverage": 0.9890909090909091, |
| "accuracy": 0.9399509803921569, |
| "expected_accuracy": 0.9399509803921569, |
| "expected_risk": 0.060049019607843146 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 788, |
| "coverage": 0.9551515151515152, |
| "accuracy": 0.9593908629441624, |
| "expected_accuracy": 0.9593908629441624, |
| "expected_risk": 0.040609137055837574 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 765, |
| "coverage": 0.9272727272727272, |
| "accuracy": 0.9751633986928104, |
| "expected_accuracy": 0.9751633986928104, |
| "expected_risk": 0.024836601307189565 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 741, |
| "coverage": 0.8981818181818182, |
| "accuracy": 0.9824561403508771, |
| "expected_accuracy": 0.9824561403508771, |
| "expected_risk": 0.01754385964912286 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 693, |
| "coverage": 0.84, |
| "accuracy": 0.9884559884559885, |
| "expected_accuracy": 0.9884559884559885, |
| "expected_risk": 0.011544011544011523 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 457, |
| "coverage": 0.553939393939394, |
| "accuracy": 0.9978118161925602, |
| "expected_accuracy": 0.9978118161925602, |
| "expected_risk": 0.002188183807439792 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 825, |
| "hard_count": 825, |
| "accuracy": 0.9309090909090909, |
| "expected_accuracy": 0.9309090909090909, |
| "nll": 0.18737228735728542, |
| "brier": 0.09844207618854224, |
| "expected_brier": 0.09844207618854224, |
| "ece_bins": 15, |
| "multiclass_ece": 0.0250802098487241, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 816, |
| "coverage": 0.9890909090909091, |
| "accuracy": 0.9399509803921569, |
| "expected_accuracy": 0.9399509803921569, |
| "expected_risk": 0.060049019607843146 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 788, |
| "coverage": 0.9551515151515152, |
| "accuracy": 0.9593908629441624, |
| "expected_accuracy": 0.9593908629441624, |
| "expected_risk": 0.040609137055837574 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 765, |
| "coverage": 0.9272727272727272, |
| "accuracy": 0.9751633986928104, |
| "expected_accuracy": 0.9751633986928104, |
| "expected_risk": 0.024836601307189565 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 741, |
| "coverage": 0.8981818181818182, |
| "accuracy": 0.9824561403508771, |
| "expected_accuracy": 0.9824561403508771, |
| "expected_risk": 0.01754385964912286 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 693, |
| "coverage": 0.84, |
| "accuracy": 0.9884559884559885, |
| "expected_accuracy": 0.9884559884559885, |
| "expected_risk": 0.011544011544011523 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 457, |
| "coverage": 0.553939393939394, |
| "accuracy": 0.9978118161925602, |
| "expected_accuracy": 0.9978118161925602, |
| "expected_risk": 0.002188183807439792 |
| } |
| ] |
| } |
| }, |
| "wikispeedia-v1": { |
| "count": 219, |
| "successful": 219, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 57, |
| "accuracy_all_rows_failures_incorrect": 0.5789473684210527, |
| "expected_accuracy_all_rows_failures_zero": 0.29281085479715613, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 2, |
| "total": 219, |
| "coverage": 0.0091324200913242 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 0, |
| "total": 219, |
| "coverage": 0.0 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 0, |
| "total": 219, |
| "coverage": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 0, |
| "total": 219, |
| "coverage": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 0, |
| "total": 219, |
| "coverage": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 0, |
| "total": 219, |
| "coverage": 0.0 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 219, |
| "hard_count": 57, |
| "accuracy": 0.5789473684210527, |
| "expected_accuracy": 0.29281085479715613, |
| "nll": 2.1593282501811903, |
| "brier": 0.3142609630528501, |
| "expected_brier": 0.8511196016238175, |
| "ece_bins": 15, |
| "multiclass_ece": 0.1249962258708074, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 2, |
| "coverage": 0.0091324200913242, |
| "accuracy": null, |
| "expected_accuracy": 0.5, |
| "expected_risk": 0.5 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 219, |
| "hard_count": 57, |
| "accuracy": 0.5789473684210527, |
| "expected_accuracy": 0.29281085479715613, |
| "nll": 2.1593282501811903, |
| "brier": 0.3142609630528501, |
| "expected_brier": 0.8511196016238175, |
| "ece_bins": 15, |
| "multiclass_ece": 0.1249962258708074, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 2, |
| "coverage": 0.0091324200913242, |
| "accuracy": null, |
| "expected_accuracy": 0.5, |
| "expected_risk": 0.5 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| } |
| ] |
| } |
| }, |
| "workflow-controls-v1/agent_trace_observability": { |
| "count": 525, |
| "successful": 525, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 525, |
| "accuracy_all_rows_failures_incorrect": 0.9885714285714285, |
| "expected_accuracy_all_rows_failures_zero": 0.9885714285714285, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 525, |
| "total": 525, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 525, |
| "total": 525, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 524, |
| "total": 525, |
| "coverage": 0.9980952380952381 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 520, |
| "total": 525, |
| "coverage": 0.9904761904761905 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 519, |
| "total": 525, |
| "coverage": 0.9885714285714285 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 438, |
| "total": 525, |
| "coverage": 0.8342857142857143 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 525, |
| "hard_count": 525, |
| "accuracy": 0.9885714285714285, |
| "expected_accuracy": 0.9885714285714285, |
| "nll": 0.032287053288972606, |
| "brier": 0.01822595812319716, |
| "expected_brier": 0.01822595812319716, |
| "ece_bins": 15, |
| "multiclass_ece": 0.005162135252793167, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 525, |
| "coverage": 1.0, |
| "accuracy": 0.9885714285714285, |
| "expected_accuracy": 0.9885714285714285, |
| "expected_risk": 0.011428571428571455 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 525, |
| "coverage": 1.0, |
| "accuracy": 0.9885714285714285, |
| "expected_accuracy": 0.9885714285714285, |
| "expected_risk": 0.011428571428571455 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 524, |
| "coverage": 0.9980952380952381, |
| "accuracy": 0.9904580152671756, |
| "expected_accuracy": 0.9904580152671756, |
| "expected_risk": 0.009541984732824416 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 520, |
| "coverage": 0.9904761904761905, |
| "accuracy": 0.9942307692307693, |
| "expected_accuracy": 0.9942307692307693, |
| "expected_risk": 0.005769230769230749 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 519, |
| "coverage": 0.9885714285714285, |
| "accuracy": 0.9942196531791907, |
| "expected_accuracy": 0.9942196531791907, |
| "expected_risk": 0.005780346820809301 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 438, |
| "coverage": 0.8342857142857143, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 525, |
| "hard_count": 525, |
| "accuracy": 0.9885714285714285, |
| "expected_accuracy": 0.9885714285714285, |
| "nll": 0.032287053288972606, |
| "brier": 0.01822595812319716, |
| "expected_brier": 0.01822595812319716, |
| "ece_bins": 15, |
| "multiclass_ece": 0.005162135252793167, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 525, |
| "coverage": 1.0, |
| "accuracy": 0.9885714285714285, |
| "expected_accuracy": 0.9885714285714285, |
| "expected_risk": 0.011428571428571455 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 525, |
| "coverage": 1.0, |
| "accuracy": 0.9885714285714285, |
| "expected_accuracy": 0.9885714285714285, |
| "expected_risk": 0.011428571428571455 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 524, |
| "coverage": 0.9980952380952381, |
| "accuracy": 0.9904580152671756, |
| "expected_accuracy": 0.9904580152671756, |
| "expected_risk": 0.009541984732824416 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 520, |
| "coverage": 0.9904761904761905, |
| "accuracy": 0.9942307692307693, |
| "expected_accuracy": 0.9942307692307693, |
| "expected_risk": 0.005769230769230749 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 519, |
| "coverage": 0.9885714285714285, |
| "accuracy": 0.9942196531791907, |
| "expected_accuracy": 0.9942196531791907, |
| "expected_risk": 0.005780346820809301 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 438, |
| "coverage": 0.8342857142857143, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| }, |
| "workflow-controls-v1/customer_service": { |
| "count": 600, |
| "successful": 600, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 600, |
| "accuracy_all_rows_failures_incorrect": 0.9466666666666667, |
| "expected_accuracy_all_rows_failures_zero": 0.9466666666666667, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 600, |
| "total": 600, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 596, |
| "total": 600, |
| "coverage": 0.9933333333333333 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 592, |
| "total": 600, |
| "coverage": 0.9866666666666667 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 575, |
| "total": 600, |
| "coverage": 0.9583333333333334 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 559, |
| "total": 600, |
| "coverage": 0.9316666666666666 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 458, |
| "total": 600, |
| "coverage": 0.7633333333333333 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 600, |
| "hard_count": 600, |
| "accuracy": 0.9466666666666667, |
| "expected_accuracy": 0.9466666666666667, |
| "nll": 0.14410110761968423, |
| "brier": 0.08122778878087651, |
| "expected_brier": 0.08122778878087651, |
| "ece_bins": 15, |
| "multiclass_ece": 0.037479888817243255, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 600, |
| "coverage": 1.0, |
| "accuracy": 0.9466666666666667, |
| "expected_accuracy": 0.9466666666666667, |
| "expected_risk": 0.053333333333333344 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 596, |
| "coverage": 0.9933333333333333, |
| "accuracy": 0.9530201342281879, |
| "expected_accuracy": 0.9530201342281879, |
| "expected_risk": 0.046979865771812124 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 592, |
| "coverage": 0.9866666666666667, |
| "accuracy": 0.9594594594594594, |
| "expected_accuracy": 0.9594594594594594, |
| "expected_risk": 0.04054054054054057 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 575, |
| "coverage": 0.9583333333333334, |
| "accuracy": 0.9739130434782609, |
| "expected_accuracy": 0.9739130434782609, |
| "expected_risk": 0.02608695652173909 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 559, |
| "coverage": 0.9316666666666666, |
| "accuracy": 0.9803220035778175, |
| "expected_accuracy": 0.9803220035778175, |
| "expected_risk": 0.019677996422182487 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 458, |
| "coverage": 0.7633333333333333, |
| "accuracy": 0.9912663755458515, |
| "expected_accuracy": 0.9912663755458515, |
| "expected_risk": 0.008733624454148492 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 600, |
| "hard_count": 600, |
| "accuracy": 0.9466666666666667, |
| "expected_accuracy": 0.9466666666666667, |
| "nll": 0.14410110761968423, |
| "brier": 0.08122778878087651, |
| "expected_brier": 0.08122778878087651, |
| "ece_bins": 15, |
| "multiclass_ece": 0.037479888817243255, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 600, |
| "coverage": 1.0, |
| "accuracy": 0.9466666666666667, |
| "expected_accuracy": 0.9466666666666667, |
| "expected_risk": 0.053333333333333344 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 596, |
| "coverage": 0.9933333333333333, |
| "accuracy": 0.9530201342281879, |
| "expected_accuracy": 0.9530201342281879, |
| "expected_risk": 0.046979865771812124 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 592, |
| "coverage": 0.9866666666666667, |
| "accuracy": 0.9594594594594594, |
| "expected_accuracy": 0.9594594594594594, |
| "expected_risk": 0.04054054054054057 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 575, |
| "coverage": 0.9583333333333334, |
| "accuracy": 0.9739130434782609, |
| "expected_accuracy": 0.9739130434782609, |
| "expected_risk": 0.02608695652173909 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 559, |
| "coverage": 0.9316666666666666, |
| "accuracy": 0.9803220035778175, |
| "expected_accuracy": 0.9803220035778175, |
| "expected_risk": 0.019677996422182487 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 458, |
| "coverage": 0.7633333333333333, |
| "accuracy": 0.9912663755458515, |
| "expected_accuracy": 0.9912663755458515, |
| "expected_risk": 0.008733624454148492 |
| } |
| ] |
| } |
| }, |
| "workflow-controls-v1/invoice_processing": { |
| "count": 750, |
| "successful": 750, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 750, |
| "accuracy_all_rows_failures_incorrect": 0.9973333333333333, |
| "expected_accuracy_all_rows_failures_zero": 0.9973333333333333, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 750, |
| "total": 750, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 746, |
| "total": 750, |
| "coverage": 0.9946666666666667 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 741, |
| "total": 750, |
| "coverage": 0.988 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 739, |
| "total": 750, |
| "coverage": 0.9853333333333333 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 723, |
| "total": 750, |
| "coverage": 0.964 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 565, |
| "total": 750, |
| "coverage": 0.7533333333333333 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 750, |
| "hard_count": 750, |
| "accuracy": 0.9973333333333333, |
| "expected_accuracy": 0.9973333333333333, |
| "nll": 0.013733993789974948, |
| "brier": 0.005134315183105472, |
| "expected_brier": 0.005134315183105474, |
| "ece_bins": 15, |
| "multiclass_ece": 0.008916569978317768, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 750, |
| "coverage": 1.0, |
| "accuracy": 0.9973333333333333, |
| "expected_accuracy": 0.9973333333333333, |
| "expected_risk": 0.002666666666666706 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 746, |
| "coverage": 0.9946666666666667, |
| "accuracy": 0.9973190348525469, |
| "expected_accuracy": 0.9973190348525469, |
| "expected_risk": 0.002680965147453085 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 741, |
| "coverage": 0.988, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 739, |
| "coverage": 0.9853333333333333, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 723, |
| "coverage": 0.964, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 565, |
| "coverage": 0.7533333333333333, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 750, |
| "hard_count": 750, |
| "accuracy": 0.9973333333333333, |
| "expected_accuracy": 0.9973333333333333, |
| "nll": 0.013733993789974948, |
| "brier": 0.005134315183105472, |
| "expected_brier": 0.005134315183105474, |
| "ece_bins": 15, |
| "multiclass_ece": 0.008916569978317768, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 750, |
| "coverage": 1.0, |
| "accuracy": 0.9973333333333333, |
| "expected_accuracy": 0.9973333333333333, |
| "expected_risk": 0.002666666666666706 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 746, |
| "coverage": 0.9946666666666667, |
| "accuracy": 0.9973190348525469, |
| "expected_accuracy": 0.9973190348525469, |
| "expected_risk": 0.002680965147453085 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 741, |
| "coverage": 0.988, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 739, |
| "coverage": 0.9853333333333333, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 723, |
| "coverage": 0.964, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 565, |
| "coverage": 0.7533333333333333, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| }, |
| "workflow-controls-v1/security_incidents": { |
| "count": 1275, |
| "successful": 1275, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 1275, |
| "accuracy_all_rows_failures_incorrect": 0.9937254901960785, |
| "expected_accuracy_all_rows_failures_zero": 0.9937254901960785, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1275, |
| "total": 1275, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1267, |
| "total": 1275, |
| "coverage": 0.9937254901960785 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1265, |
| "total": 1275, |
| "coverage": 0.9921568627450981 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1259, |
| "total": 1275, |
| "coverage": 0.9874509803921568 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1239, |
| "total": 1275, |
| "coverage": 0.971764705882353 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1181, |
| "total": 1275, |
| "coverage": 0.9262745098039216 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 1275, |
| "hard_count": 1275, |
| "accuracy": 0.9937254901960785, |
| "expected_accuracy": 0.9937254901960785, |
| "nll": 0.01963776236994051, |
| "brier": 0.010570254728764788, |
| "expected_brier": 0.010570254728764786, |
| "ece_bins": 15, |
| "multiclass_ece": 0.002970435231356543, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1275, |
| "coverage": 1.0, |
| "accuracy": 0.9937254901960785, |
| "expected_accuracy": 0.9937254901960785, |
| "expected_risk": 0.006274509803921524 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1267, |
| "coverage": 0.9937254901960785, |
| "accuracy": 0.9960536700868192, |
| "expected_accuracy": 0.9960536700868192, |
| "expected_risk": 0.003946329913180779 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1265, |
| "coverage": 0.9921568627450981, |
| "accuracy": 0.9960474308300395, |
| "expected_accuracy": 0.9960474308300395, |
| "expected_risk": 0.0039525691699604515 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1259, |
| "coverage": 0.9874509803921568, |
| "accuracy": 0.9968228752978554, |
| "expected_accuracy": 0.9968228752978554, |
| "expected_risk": 0.003177124702144596 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1239, |
| "coverage": 0.971764705882353, |
| "accuracy": 0.9983857949959645, |
| "expected_accuracy": 0.9983857949959645, |
| "expected_risk": 0.0016142050040355294 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1181, |
| "coverage": 0.9262745098039216, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 1275, |
| "hard_count": 1275, |
| "accuracy": 0.9937254901960785, |
| "expected_accuracy": 0.9937254901960785, |
| "nll": 0.01963776236994051, |
| "brier": 0.010570254728764788, |
| "expected_brier": 0.010570254728764786, |
| "ece_bins": 15, |
| "multiclass_ece": 0.002970435231356543, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1275, |
| "coverage": 1.0, |
| "accuracy": 0.9937254901960785, |
| "expected_accuracy": 0.9937254901960785, |
| "expected_risk": 0.006274509803921524 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1267, |
| "coverage": 0.9937254901960785, |
| "accuracy": 0.9960536700868192, |
| "expected_accuracy": 0.9960536700868192, |
| "expected_risk": 0.003946329913180779 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1265, |
| "coverage": 0.9921568627450981, |
| "accuracy": 0.9960474308300395, |
| "expected_accuracy": 0.9960474308300395, |
| "expected_risk": 0.0039525691699604515 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1259, |
| "coverage": 0.9874509803921568, |
| "accuracy": 0.9968228752978554, |
| "expected_accuracy": 0.9968228752978554, |
| "expected_risk": 0.003177124702144596 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1239, |
| "coverage": 0.971764705882353, |
| "accuracy": 0.9983857949959645, |
| "expected_accuracy": 0.9983857949959645, |
| "expected_risk": 0.0016142050040355294 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1181, |
| "coverage": 0.9262745098039216, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| } |
| }, |
| "by_kind": { |
| "choice": { |
| "count": 3438, |
| "successful": 3438, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 2993, |
| "accuracy_all_rows_failures_incorrect": 0.7885065152021383, |
| "expected_accuracy_all_rows_failures_zero": 0.736268831840385, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 3067, |
| "total": 3438, |
| "coverage": 0.8920884235020361 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 2755, |
| "total": 3438, |
| "coverage": 0.8013379872018616 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 2665, |
| "total": 3438, |
| "coverage": 0.7751599767306574 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 2505, |
| "total": 3438, |
| "coverage": 0.7286212914485166 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 2207, |
| "total": 3438, |
| "coverage": 0.6419429901105294 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1672, |
| "total": 3438, |
| "coverage": 0.48632926119837117 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 3438, |
| "hard_count": 2993, |
| "accuracy": 0.7885065152021383, |
| "expected_accuracy": 0.736268831840385, |
| "nll": 1.4384722301984725, |
| "brier": 0.33992539289134077, |
| "expected_brier": 0.41537779986643536, |
| "ece_bins": 15, |
| "multiclass_ece": 0.13811597781941512, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 3067, |
| "coverage": 0.8920884235020361, |
| "accuracy": 0.8078417520310844, |
| "expected_accuracy": 0.7838821867188348, |
| "expected_risk": 0.2161178132811652 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 2755, |
| "coverage": 0.8013379872018616, |
| "accuracy": 0.8220338983050848, |
| "expected_accuracy": 0.8171203871748336, |
| "expected_risk": 0.1828796128251664 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 2665, |
| "coverage": 0.7751599767306574, |
| "accuracy": 0.828907435508346, |
| "expected_accuracy": 0.8252657911194498, |
| "expected_risk": 0.17473420888055025 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 2505, |
| "coverage": 0.7286212914485166, |
| "accuracy": 0.8285140562248996, |
| "expected_accuracy": 0.8265469061876247, |
| "expected_risk": 0.17345309381237528 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 2207, |
| "coverage": 0.6419429901105294, |
| "accuracy": 0.8145124716553288, |
| "expected_accuracy": 0.8142274580879021, |
| "expected_risk": 0.1857725419120979 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1672, |
| "coverage": 0.48632926119837117, |
| "accuracy": 0.7816985645933014, |
| "expected_accuracy": 0.7816985645933014, |
| "expected_risk": 0.21830143540669855 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 3438, |
| "hard_count": 2993, |
| "accuracy": 0.7885065152021383, |
| "expected_accuracy": 0.736268831840385, |
| "nll": 1.4384722301984725, |
| "brier": 0.33992539289134077, |
| "expected_brier": 0.41537779986643536, |
| "ece_bins": 15, |
| "multiclass_ece": 0.13811597781941512, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 3067, |
| "coverage": 0.8920884235020361, |
| "accuracy": 0.8078417520310844, |
| "expected_accuracy": 0.7838821867188348, |
| "expected_risk": 0.2161178132811652 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 2755, |
| "coverage": 0.8013379872018616, |
| "accuracy": 0.8220338983050848, |
| "expected_accuracy": 0.8171203871748336, |
| "expected_risk": 0.1828796128251664 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 2665, |
| "coverage": 0.7751599767306574, |
| "accuracy": 0.828907435508346, |
| "expected_accuracy": 0.8252657911194498, |
| "expected_risk": 0.17473420888055025 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 2505, |
| "coverage": 0.7286212914485166, |
| "accuracy": 0.8285140562248996, |
| "expected_accuracy": 0.8265469061876247, |
| "expected_risk": 0.17345309381237528 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 2207, |
| "coverage": 0.6419429901105294, |
| "accuracy": 0.8145124716553288, |
| "expected_accuracy": 0.8142274580879021, |
| "expected_risk": 0.1857725419120979 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1672, |
| "coverage": 0.48632926119837117, |
| "accuracy": 0.7816985645933014, |
| "expected_accuracy": 0.7816985645933014, |
| "expected_risk": 0.21830143540669855 |
| } |
| ] |
| } |
| }, |
| "noul": { |
| "count": 9929, |
| "successful": 9929, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 9929, |
| "accuracy_all_rows_failures_incorrect": 0.8819619297008762, |
| "expected_accuracy_all_rows_failures_zero": 0.8819619297008762, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 9929, |
| "total": 9929, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 9795, |
| "total": 9929, |
| "coverage": 0.9865041796756975 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 9687, |
| "total": 9929, |
| "coverage": 0.9756269513546177 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 9400, |
| "total": 9929, |
| "coverage": 0.9467217242421191 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 8945, |
| "total": 9929, |
| "coverage": 0.9008963641857186 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 6768, |
| "total": 9929, |
| "coverage": 0.6816396414543258 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 9929, |
| "hard_count": 9929, |
| "accuracy": 0.8819619297008762, |
| "expected_accuracy": 0.8819619297008762, |
| "nll": 0.5310062525703106, |
| "brier": 0.21147263775707414, |
| "expected_brier": 0.21147263775707414, |
| "ece_bins": 15, |
| "multiclass_ece": 0.09634430608269225, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 9929, |
| "coverage": 1.0, |
| "accuracy": 0.8819619297008762, |
| "expected_accuracy": 0.8819619297008762, |
| "expected_risk": 0.11803807029912383 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 9795, |
| "coverage": 0.9865041796756975, |
| "accuracy": 0.8870852475752935, |
| "expected_accuracy": 0.8870852475752935, |
| "expected_risk": 0.1129147524247065 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 9687, |
| "coverage": 0.9756269513546177, |
| "accuracy": 0.8924331578404047, |
| "expected_accuracy": 0.8924331578404047, |
| "expected_risk": 0.10756684215959533 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 9400, |
| "coverage": 0.9467217242421191, |
| "accuracy": 0.9038297872340425, |
| "expected_accuracy": 0.9038297872340425, |
| "expected_risk": 0.0961702127659575 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 8945, |
| "coverage": 0.9008963641857186, |
| "accuracy": 0.9149245388485188, |
| "expected_accuracy": 0.9149245388485188, |
| "expected_risk": 0.08507546115148124 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 6768, |
| "coverage": 0.6816396414543258, |
| "accuracy": 0.9206560283687943, |
| "expected_accuracy": 0.9206560283687943, |
| "expected_risk": 0.07934397163120566 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 9929, |
| "hard_count": 9929, |
| "accuracy": 0.8819619297008762, |
| "expected_accuracy": 0.8819619297008762, |
| "nll": 0.5310062525703106, |
| "brier": 0.21147263775707414, |
| "expected_brier": 0.21147263775707414, |
| "ece_bins": 15, |
| "multiclass_ece": 0.09634430608269225, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 9929, |
| "coverage": 1.0, |
| "accuracy": 0.8819619297008762, |
| "expected_accuracy": 0.8819619297008762, |
| "expected_risk": 0.11803807029912383 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 9795, |
| "coverage": 0.9865041796756975, |
| "accuracy": 0.8870852475752935, |
| "expected_accuracy": 0.8870852475752935, |
| "expected_risk": 0.1129147524247065 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 9687, |
| "coverage": 0.9756269513546177, |
| "accuracy": 0.8924331578404047, |
| "expected_accuracy": 0.8924331578404047, |
| "expected_risk": 0.10756684215959533 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 9400, |
| "coverage": 0.9467217242421191, |
| "accuracy": 0.9038297872340425, |
| "expected_accuracy": 0.9038297872340425, |
| "expected_risk": 0.0961702127659575 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 8945, |
| "coverage": 0.9008963641857186, |
| "accuracy": 0.9149245388485188, |
| "expected_accuracy": 0.9149245388485188, |
| "expected_risk": 0.08507546115148124 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 6768, |
| "coverage": 0.6816396414543258, |
| "accuracy": 0.9206560283687943, |
| "expected_accuracy": 0.9206560283687943, |
| "expected_risk": 0.07934397163120566 |
| } |
| ] |
| } |
| }, |
| "score": { |
| "count": 2553, |
| "successful": 2553, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 2524, |
| "accuracy_all_rows_failures_incorrect": 0.8597464342313788, |
| "expected_accuracy_all_rows_failures_zero": 0.854876615746181, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 2471, |
| "total": 2553, |
| "coverage": 0.9678809244026635 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 2424, |
| "total": 2553, |
| "coverage": 0.9494712103407755 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 2403, |
| "total": 2553, |
| "coverage": 0.9412455934195064 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 2307, |
| "total": 2553, |
| "coverage": 0.9036427732079906 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 2133, |
| "total": 2553, |
| "coverage": 0.8354876615746181 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1556, |
| "total": 2553, |
| "coverage": 0.609479044261653 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 2553, |
| "hard_count": 2524, |
| "accuracy": 0.8597464342313788, |
| "expected_accuracy": 0.854876615746181, |
| "nll": 1.0050022837630184, |
| "brier": 0.25660594897422545, |
| "expected_brier": 0.26306893369807977, |
| "ece_bins": 15, |
| "multiclass_ece": 0.10397632298121842, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 2471, |
| "coverage": 0.9678809244026635, |
| "accuracy": 0.874949083503055, |
| "expected_accuracy": 0.8725212464589235, |
| "expected_risk": 0.12747875354107652 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 2424, |
| "coverage": 0.9494712103407755, |
| "accuracy": 0.8758250825082509, |
| "expected_accuracy": 0.8758250825082509, |
| "expected_risk": 0.12417491749174914 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 2403, |
| "coverage": 0.9412455934195064, |
| "accuracy": 0.8768206408655846, |
| "expected_accuracy": 0.8768206408655846, |
| "expected_risk": 0.12317935913441536 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 2307, |
| "coverage": 0.9036427732079906, |
| "accuracy": 0.8764629388816645, |
| "expected_accuracy": 0.8764629388816645, |
| "expected_risk": 0.12353706111833551 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 2133, |
| "coverage": 0.8354876615746181, |
| "accuracy": 0.8752930145335208, |
| "expected_accuracy": 0.8752930145335208, |
| "expected_risk": 0.12470698546647918 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1556, |
| "coverage": 0.609479044261653, |
| "accuracy": 0.8759640102827764, |
| "expected_accuracy": 0.8759640102827764, |
| "expected_risk": 0.12403598971722363 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 2553, |
| "hard_count": 2524, |
| "accuracy": 0.8597464342313788, |
| "expected_accuracy": 0.854876615746181, |
| "nll": 1.0050022837630184, |
| "brier": 0.25660594897422545, |
| "expected_brier": 0.26306893369807977, |
| "ece_bins": 15, |
| "multiclass_ece": 0.10397632298121842, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 2471, |
| "coverage": 0.9678809244026635, |
| "accuracy": 0.874949083503055, |
| "expected_accuracy": 0.8725212464589235, |
| "expected_risk": 0.12747875354107652 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 2424, |
| "coverage": 0.9494712103407755, |
| "accuracy": 0.8758250825082509, |
| "expected_accuracy": 0.8758250825082509, |
| "expected_risk": 0.12417491749174914 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 2403, |
| "coverage": 0.9412455934195064, |
| "accuracy": 0.8768206408655846, |
| "expected_accuracy": 0.8768206408655846, |
| "expected_risk": 0.12317935913441536 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 2307, |
| "coverage": 0.9036427732079906, |
| "accuracy": 0.8764629388816645, |
| "expected_accuracy": 0.8764629388816645, |
| "expected_risk": 0.12353706111833551 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 2133, |
| "coverage": 0.8354876615746181, |
| "accuracy": 0.8752930145335208, |
| "expected_accuracy": 0.8752930145335208, |
| "expected_risk": 0.12470698546647918 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1556, |
| "coverage": 0.609479044261653, |
| "accuracy": 0.8759640102827764, |
| "expected_accuracy": 0.8759640102827764, |
| "expected_risk": 0.12403598971722363 |
| } |
| ] |
| } |
| } |
| } |
| } |
| }, |
| "full_data_baseline_evaluated": false, |
| "full_data_training_gain": null, |
| "limitations": [ |
| "No full-data baseline exists in this run. Earlier 512-row-per-split baseline results cannot establish full-data training gain.", |
| "Per-group metrics were recomputed and compared for every group; compact report retains source/kind metrics and group counts/hashes. Original complete summary remains in ignored evidence.", |
| "Snapshot/checkpoint hashes bind this evaluation to the packaged inference files; the trainer's historical reload lacked a contemporaneous weight digest.", |
| "Synthetic held-out decision accuracy is not end-to-end game, browser or flight performance. Four-replica wall time is not single-GPU latency." |
| ], |
| "schema_version": 1, |
| "status": "passed", |
| "source_audit_sha256": "e8b0583b12ee00e8031e7ea5388d935389c3ce9681e3099da0cafe30329ff472", |
| "source_audit_path": "reports/full-data-eval-n1-v1/2b-audit.json", |
| "scope": "Existing completed full release-v2 held-out evaluation, bound to the unchanged five packaged inference files. No new model inference was run for this Hugging Face upload." |
| } |
|
|
