diff --git "a/package/metrics.json" "b/package/metrics.json" new file mode 100644--- /dev/null +++ "b/package/metrics.json" @@ -0,0 +1,69622 @@ +{ + "baseline_test": { + "count": 512, + "hard_count": 470, + "accuracy": 0.5468085106382978, + "expected_accuracy": 0.5218850756448413, + "nll": 1.027023313476401, + "brier": 0.5120175078226842, + "expected_brier": 0.5633095155111765, + "multiclass_ece": 0.08855853617844331, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 314, + "coverage": 0.61328125, + "accuracy": 0.66996699669967, + "expected_accuracy": 0.6634819532908706, + "expected_risk": 0.33651804670912944 + }, + { + "threshold": 0.7, + "selected": 169, + "coverage": 0.330078125, + "accuracy": 0.7865853658536586, + "expected_accuracy": 0.7781065088757396, + "expected_risk": 0.22189349112426038 + }, + { + "threshold": 0.8, + "selected": 103, + "coverage": 0.201171875, + "accuracy": 0.8415841584158416, + "expected_accuracy": 0.8349514563106796, + "expected_risk": 0.16504854368932043 + }, + { + "threshold": 0.9, + "selected": 52, + "coverage": 0.1015625, + "accuracy": 0.7884615384615384, + "expected_accuracy": 0.7884615384615384, + "expected_risk": 0.21153846153846156 + }, + { + "threshold": 0.95, + "selected": 16, + "coverage": 0.03125, + "accuracy": 0.625, + "expected_accuracy": 0.625, + "expected_risk": 0.375 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 0.001953125, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ], + "mean_latency_seconds": 0.054588349696132354, + "by_kind": { + "choice": { + "count": 197, + "hard_count": 158, + "accuracy": 0.45569620253164556, + "expected_accuracy": 0.40967085649826773, + "nll": 1.35243572091123, + "brier": 0.5469692678193017, + "expected_brier": 0.6726622015071594, + "multiclass_ece": 0.12394110093102934, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 64, + "coverage": 0.3248730964467005, + "accuracy": 0.6964285714285714, + "expected_accuracy": 0.6692708333333334, + "expected_risk": 0.33072916666666663 + }, + { + "threshold": 0.7, + "selected": 31, + "coverage": 0.15736040609137056, + "accuracy": 0.7931034482758621, + "expected_accuracy": 0.7741935483870968, + "expected_risk": 0.22580645161290325 + }, + { + "threshold": 0.8, + "selected": 15, + "coverage": 0.07614213197969544, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 4, + "coverage": 0.02030456852791878, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 0.005076142131979695, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "noul": { + "count": 219, + "hard_count": 219, + "accuracy": 0.7077625570776256, + "expected_accuracy": 0.7077625570776256, + "nll": 0.6364033980795772, + "brier": 0.4041040276282712, + "expected_brier": 0.4041040276282713, + "multiclass_ece": 0.08481249139611502, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 219, + "coverage": 1.0, + "accuracy": 0.7077625570776256, + "expected_accuracy": 0.7077625570776256, + "expected_risk": 0.2922374429223744 + }, + { + "threshold": 0.7, + "selected": 131, + "coverage": 0.5981735159817352, + "accuracy": 0.7862595419847328, + "expected_accuracy": 0.7862595419847328, + "expected_risk": 0.2137404580152672 + }, + { + "threshold": 0.8, + "selected": 83, + "coverage": 0.3789954337899543, + "accuracy": 0.8072289156626506, + "expected_accuracy": 0.8072289156626506, + "expected_risk": 0.19277108433734935 + }, + { + "threshold": 0.9, + "selected": 48, + "coverage": 0.2191780821917808, + "accuracy": 0.7708333333333334, + "expected_accuracy": 0.7708333333333334, + "expected_risk": 0.22916666666666663 + }, + { + "threshold": 0.95, + "selected": 15, + "coverage": 0.0684931506849315, + "accuracy": 0.6, + "expected_accuracy": 0.6, + "expected_risk": 0.4 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 0.0045662100456621, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "score": { + "count": 96, + "hard_count": 93, + "accuracy": 0.3225806451612903, + "expected_accuracy": 0.328125, + "nll": 1.2503516177185172, + "brier": 0.6864712103564635, + "expected_brier": 0.7020962103564635, + "multiclass_ece": 0.17062461370978052, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 31, + "coverage": 0.3229166666666667, + "accuracy": 0.32142857142857145, + "expected_accuracy": 0.3387096774193548, + "expected_risk": 0.6612903225806452 + }, + { + "threshold": 0.7, + "selected": 7, + "coverage": 0.07291666666666667, + "accuracy": 0.75, + "expected_accuracy": 0.6428571428571429, + "expected_risk": 0.3571428571428571 + }, + { + "threshold": 0.8, + "selected": 5, + "coverage": 0.052083333333333336, + "accuracy": 1.0, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ], + "ordinal_mae": 0.7839660316551388 + } + }, + "by_source": { + "customer-control-v1": { + "count": 72, + "hard_count": 68, + "accuracy": 0.6323529411764706, + "expected_accuracy": 0.625, + "nll": 0.6191557236780617, + "brier": 0.3591527103888658, + "expected_brier": 0.3869304881666437, + "multiclass_ece": 0.1805459658304426, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 65, + "coverage": 0.9027777777777778, + "accuracy": 0.6721311475409836, + "expected_accuracy": 0.6615384615384615, + "expected_risk": 0.3384615384615385 + }, + { + "threshold": 0.7, + "selected": 37, + "coverage": 0.5138888888888888, + "accuracy": 0.9705882352941176, + "expected_accuracy": 0.9324324324324325, + "expected_risk": 0.06756756756756754 + }, + { + "threshold": 0.8, + "selected": 29, + "coverage": 0.4027777777777778, + "accuracy": 1.0, + "expected_accuracy": 0.9655172413793104, + "expected_risk": 0.03448275862068961 + }, + { + "threshold": 0.9, + "selected": 13, + "coverage": 0.18055555555555555, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 0.027777777777777776, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1": { + "count": 74, + "hard_count": 74, + "accuracy": 0.4864864864864865, + "expected_accuracy": 0.4864864864864865, + "nll": 1.3396155379983727, + "brier": 0.6904198858152533, + "expected_brier": 0.6904198858152534, + "multiclass_ece": 0.2359836148655567, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 34, + "coverage": 0.4594594594594595, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.7, + "selected": 19, + "coverage": 0.25675675675675674, + "accuracy": 0.47368421052631576, + "expected_accuracy": 0.47368421052631576, + "expected_risk": 0.5263157894736843 + }, + { + "threshold": 0.8, + "selected": 14, + "coverage": 0.1891891891891892, + "accuracy": 0.5714285714285714, + "expected_accuracy": 0.5714285714285714, + "expected_risk": 0.4285714285714286 + }, + { + "threshold": 0.9, + "selected": 11, + "coverage": 0.14864864864864866, + "accuracy": 0.5454545454545454, + "expected_accuracy": 0.5454545454545454, + "expected_risk": 0.4545454545454546 + }, + { + "threshold": 0.95, + "selected": 9, + "coverage": 0.12162162162162163, + "accuracy": 0.4444444444444444, + "expected_accuracy": 0.4444444444444444, + "expected_risk": 0.5555555555555556 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 0.013513513513513514, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "reasoning-control-v1": { + "count": 72, + "hard_count": 72, + "accuracy": 0.5416666666666666, + "expected_accuracy": 0.5416666666666666, + "nll": 0.9690737492233605, + "brier": 0.5642634886999491, + "expected_brier": 0.5642634886999491, + "multiclass_ece": 0.1390017584829721, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 39, + "coverage": 0.5416666666666666, + "accuracy": 0.5897435897435898, + "expected_accuracy": 0.5897435897435898, + "expected_risk": 0.41025641025641024 + }, + { + "threshold": 0.7, + "selected": 11, + "coverage": 0.1527777777777778, + "accuracy": 0.9090909090909091, + "expected_accuracy": 0.9090909090909091, + "expected_risk": 0.09090909090909094 + }, + { + "threshold": 0.8, + "selected": 6, + "coverage": 0.08333333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 0.041666666666666664, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 0.027777777777777776, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "snake-v1": { + "count": 48, + "hard_count": 43, + "accuracy": 0.627906976744186, + "expected_accuracy": 0.6076388888888888, + "nll": 0.9546125403291542, + "brier": 0.5161581087394461, + "expected_brier": 0.5751858865172239, + "multiclass_ece": 0.22471715426109293, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 45, + "coverage": 0.9375, + "accuracy": 0.6428571428571429, + "expected_accuracy": 0.6296296296296297, + "expected_risk": 0.37037037037037035 + }, + { + "threshold": 0.7, + "selected": 37, + "coverage": 0.7708333333333334, + "accuracy": 0.6571428571428571, + "expected_accuracy": 0.6486486486486487, + "expected_risk": 0.3513513513513513 + }, + { + "threshold": 0.8, + "selected": 24, + "coverage": 0.5, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.9, + "selected": 24, + "coverage": 0.5, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 0.0625, + "accuracy": 0.6666666666666666, + "expected_accuracy": 0.6666666666666666, + "expected_risk": 0.33333333333333337 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "tic_tac_toe-v1": { + "count": 24, + "hard_count": 7, + "accuracy": 0.5714285714285714, + "expected_accuracy": 0.2767857142857143, + "nll": 1.304323480548012, + "brier": 0.29733120471959634, + "expected_brier": 0.7149899348783265, + "multiclass_ece": 0.1572187744510727, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 0.20833333333333334, + "accuracy": 1.0, + "expected_accuracy": 0.7, + "expected_risk": 0.30000000000000004 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "tile_platformer-v1": { + "count": 24, + "hard_count": 24, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.9909957729024932, + "brier": 0.9130921692916404, + "expected_brier": 0.9130921692916404, + "multiclass_ece": 0.2792539473683758, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "trex_runner-v1": { + "count": 4, + "hard_count": 4, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 1.113003902721162, + "brier": 0.5930013921090308, + "expected_brier": 0.5930013921090307, + "multiclass_ece": 0.1658166315815956, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 0.75, + "accuracy": 0.6666666666666666, + "expected_accuracy": 0.6666666666666666, + "expected_risk": 0.33333333333333337 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "vizdoom-basic-v1": { + "count": 73, + "hard_count": 73, + "accuracy": 0.3972602739726027, + "expected_accuracy": 0.3972602739726027, + "nll": 1.0735621246379115, + "brier": 0.6253440986354848, + "expected_brier": 0.6253440986354848, + "multiclass_ece": 0.1446888656586181, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 25, + "coverage": 0.3424657534246575, + "accuracy": 0.64, + "expected_accuracy": 0.64, + "expected_risk": 0.36 + }, + { + "threshold": 0.7, + "selected": 12, + "coverage": 0.1643835616438356, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 0.0273972602739726, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "wikispeedia-v1": { + "count": 24, + "hard_count": 8, + "accuracy": 0.25, + "expected_accuracy": 0.22481812169312174, + "nll": 2.248698265794124, + "brier": 0.39996020810020355, + "expected_brier": 0.8751420864070817, + "multiclass_ece": 0.07621955438195471, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 0.041666666666666664, + "accuracy": null, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ], + "optimal_action_hit": 0.625, + "optimal_set_probability_mass": 0.4222930006932841 + }, + "workflow-controls-v1/agent_trace_observability": { + "count": 24, + "hard_count": 24, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "nll": 0.44955972522315363, + "brier": 0.2777851077449246, + "expected_brier": 0.27778510774492454, + "multiclass_ece": 0.02430369399972469, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.7, + "selected": 23, + "coverage": 0.9583333333333334, + "accuracy": 0.8260869565217391, + "expected_accuracy": 0.8260869565217391, + "expected_risk": 0.17391304347826086 + }, + { + "threshold": 0.8, + "selected": 15, + "coverage": 0.625, + "accuracy": 0.8666666666666667, + "expected_accuracy": 0.8666666666666667, + "expected_risk": 0.1333333333333333 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 0.041666666666666664, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service": { + "count": 24, + "hard_count": 24, + "accuracy": 0.9166666666666666, + "expected_accuracy": 0.9166666666666666, + "nll": 0.4607518792188455, + "brier": 0.2879953927534112, + "expected_brier": 0.2879953927534112, + "multiclass_ece": 0.247162358229246, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.9166666666666666, + "expected_accuracy": 0.9166666666666666, + "expected_risk": 0.08333333333333337 + }, + { + "threshold": 0.7, + "selected": 8, + "coverage": 0.3333333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 0.16666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing": { + "count": 24, + "hard_count": 24, + "accuracy": 0.7916666666666666, + "expected_accuracy": 0.7916666666666666, + "nll": 0.5502749881296093, + "brier": 0.36163607586042446, + "expected_brier": 0.3616360758604244, + "multiclass_ece": 0.14459212368670463, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.7916666666666666, + "expected_accuracy": 0.7916666666666666, + "expected_risk": 0.20833333333333337 + }, + { + "threshold": 0.7, + "selected": 18, + "coverage": 0.75, + "accuracy": 0.7222222222222222, + "expected_accuracy": 0.7222222222222222, + "expected_risk": 0.2777777777777778 + }, + { + "threshold": 0.8, + "selected": 9, + "coverage": 0.375, + "accuracy": 0.7777777777777778, + "expected_accuracy": 0.7777777777777778, + "expected_risk": 0.2222222222222222 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents": { + "count": 25, + "hard_count": 25, + "accuracy": 0.56, + "expected_accuracy": 0.56, + "nll": 0.6239165324117573, + "brier": 0.4348403331554468, + "expected_brier": 0.4348403331554468, + "multiclass_ece": 0.12364612041073371, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 25, + "coverage": 1.0, + "accuracy": 0.56, + "expected_accuracy": 0.56, + "expected_risk": 0.43999999999999995 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 0.16, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + } + }, + "by_question": { + "customer-control-v1/bug_severity": { + "count": 8, + "hard_count": 8, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.6177958523192897, + "brier": 1.0496759050732705, + "expected_brier": 1.0496759050732702, + "multiclass_ece": 0.6170238394930267, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 8, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "customer-control-v1/category": { + "count": 24, + "hard_count": 23, + "accuracy": 0.782608695652174, + "expected_accuracy": 0.7708333333333334, + "nll": 0.44759880694041954, + "brier": 0.2136550041597672, + "expected_brier": 0.2344883374931006, + "multiclass_ece": 0.23058034834038146, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 21, + "coverage": 0.875, + "accuracy": 0.85, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.7, + "selected": 16, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 14, + "coverage": 0.5833333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 4, + "coverage": 0.16666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 0.041666666666666664, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "customer-control-v1/churn_likelihood_level": { + "count": 7, + "hard_count": 7, + "accuracy": 0.14285714285714285, + "expected_accuracy": 0.14285714285714285, + "nll": 0.8601075605685917, + "brier": 0.6143712014583927, + "expected_brier": 0.6143712014583927, + "multiclass_ece": 0.37557090493868434, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 0.42857142857142855, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "customer-control-v1/frustration": { + "count": 9, + "hard_count": 6, + "accuracy": 0.16666666666666666, + "expected_accuracy": 0.2777777777777778, + "nll": 0.9611915559756642, + "brier": 0.5320741898525286, + "expected_brier": 0.6987408565191955, + "multiclass_ece": 0.4322101250032314, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 9, + "coverage": 1.0, + "accuracy": 0.16666666666666666, + "expected_accuracy": 0.2777777777777778, + "expected_risk": 0.7222222222222222 + }, + { + "threshold": 0.7, + "selected": 5, + "coverage": 0.5555555555555556, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 0.3333333333333333, + "accuracy": 1.0, + "expected_accuracy": 0.6666666666666666, + "expected_risk": 0.33333333333333337 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "customer-control-v1/has_reproducible_steps": { + "count": 14, + "hard_count": 14, + "accuracy": 0.9285714285714286, + "expected_accuracy": 0.9285714285714286, + "nll": 0.38739217181446384, + "brier": 0.2219629224999957, + "expected_brier": 0.22196292249999566, + "multiclass_ece": 0.23000571493799157, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 14, + "coverage": 1.0, + "accuracy": 0.9285714285714286, + "expected_accuracy": 0.9285714285714286, + "expected_risk": 0.0714285714285714 + }, + { + "threshold": 0.7, + "selected": 6, + "coverage": 0.42857142857142855, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 0.14285714285714285, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 0.07142857142857142, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "customer-control-v1/refund_requested": { + "count": 10, + "hard_count": 10, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.07995065865324405, + "brier": 0.013712077369631889, + "expected_brier": 0.013712077369631892, + "multiclass_ece": 0.07626333377350314, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 10, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 10, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 10, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 8, + "coverage": 0.8, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 0.1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y0_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.5010417638919521, + "brier": 0.2404162175085465, + "expected_brier": 0.24041621750854647, + "multiclass_ece": 0.39410087301690455, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y1_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.195619446471602, + "brier": 0.7312683548795447, + "expected_brier": 0.7312683548795447, + "multiclass_ece": 0.3765222516517983, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y2": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.929067572035733, + "brier": 0.8564560917082965, + "expected_brier": 0.8564560917082966, + "multiclass_ece": 0.30721272924018006, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y2_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.8182474205926693, + "brier": 0.4004938374771207, + "expected_brier": 0.40049383747712075, + "multiclass_ece": 0.558795777250127, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y2_red": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.3181862180305068, + "brier": 1.0727601470949715, + "expected_brier": 1.0727601470949715, + "multiclass_ece": 0.7323797331627123, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y3_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.0851266095036043, + "brier": 1.5337311648294845, + "expected_brier": 1.5337311648294847, + "multiclass_ece": 0.8757086173007219, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y3_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.7435003346943878, + "brier": 0.35424871047032874, + "expected_brier": 0.3542487104703287, + "multiclass_ece": 0.5245532233963466, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y5": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.11118634200521557, + "brier": 0.022145864058631656, + "expected_brier": 0.022145864058631615, + "multiclass_ece": 0.10522800021532208, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y5_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.3669517685994785, + "brier": 0.8549014693905652, + "expected_brier": 0.8549014693905652, + "multiclass_ece": 0.4776819583777261, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y6_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.2161756138958812, + "brier": 0.7452284066074157, + "expected_brier": 0.7452284066074157, + "multiclass_ece": 0.3876487102164534, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y7_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.07460526767657721, + "brier": 0.010336410638690282, + "expected_brier": 0.010336410638690285, + "multiclass_ece": 0.07189023104250769, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y7_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.816316042395182, + "brier": 0.4001448126431099, + "expected_brier": 0.40014481264310997, + "multiclass_ece": 0.5579428216088468, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y0": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.4530956689074417, + "brier": 0.6961209273939908, + "expected_brier": 0.6961209273939908, + "multiclass_ece": 0.2888464214801114, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y0_lightness": { + "count": 2, + "hard_count": 2, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 1.8686309234791338, + "brier": 0.808876015274612, + "expected_brier": 0.808876015274612, + "multiclass_ece": 0.5264110674529598, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y1_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.1426189251531867, + "brier": 0.03533309749064205, + "expected_brier": 0.03533309749064206, + "multiclass_ece": 0.13291556998832388, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y2": { + "count": 3, + "hard_count": 3, + "accuracy": 0.3333333333333333, + "expected_accuracy": 0.3333333333333333, + "nll": 1.527258369757191, + "brier": 0.7046609644093547, + "expected_brier": 0.7046609644093547, + "multiclass_ece": 0.20907956922632176, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 0.3333333333333333, + "expected_accuracy": 0.3333333333333333, + "expected_risk": 0.6666666666666667 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y4_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.8459140228170392, + "brier": 0.9049520746339219, + "expected_brier": 0.9049520746339219, + "multiclass_ece": 0.35917888110611906, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y4_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.0358361766252582, + "brier": 0.6541443344044562, + "expected_brier": 0.6541443344044562, + "multiclass_ece": 0.444949227844012, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y5_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.7441034179925564, + "brier": 0.36578530759341166, + "expected_brier": 0.36578530759341166, + "multiclass_ece": 0.5248398709616167, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y6_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.930926344221573, + "brier": 0.471004440694748, + "expected_brier": 0.471004440694748, + "multiclass_ece": 0.6058116129444063, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y6_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.43831922172605, + "brier": 0.8726217620637207, + "expected_brier": 0.8726217620637207, + "multiclass_ece": 0.3889210670844514, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y0": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 0.7635682037121088, + "brier": 0.5703105655193376, + "expected_brier": 0.5703105655193377, + "multiclass_ece": 0.5339993284262339, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y0_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.869403601102126, + "brier": 1.1971570571305128, + "expected_brier": 1.1971570571305126, + "multiclass_ece": 0.6903322037400696, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y1": { + "count": 2, + "hard_count": 2, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 1.3426409647838842, + "brier": 0.5061187108109384, + "expected_brier": 0.5061187108109383, + "multiclass_ece": 0.1764788064019109, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y1_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 3.0558062789050733, + "brier": 1.2424425463234843, + "expected_brier": 1.2424425463234843, + "multiclass_ece": 0.5058350448091703, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y3_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.2031280870587246, + "brier": 0.6226711262705125, + "expected_brier": 0.6226711262705125, + "multiclass_ece": 0.32602023349605896, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y3_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.9101135818231864, + "brier": 0.5428706955886708, + "expected_brier": 0.5428706955886708, + "multiclass_ece": 0.5975214928052563, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y4_red": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 4.445892744918699, + "brier": 1.9533684979580173, + "expected_brier": 1.9533684979580173, + "multiclass_ece": 0.9882733675350199, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y5": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.6138640427889492, + "brier": 0.4208932359013566, + "expected_brier": 0.42089323590135663, + "multiclass_ece": 0.45874461081377105, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y5_green": { + "count": 2, + "hard_count": 2, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.294105517333984, + "brier": 1.0529283699925314, + "expected_brier": 1.0529283699925314, + "multiclass_ece": 0.7254091318592547, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y6": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 0.718816766000994, + "brier": 0.5256640545660693, + "expected_brier": 0.5256640545660693, + "multiclass_ece": 0.512671461350283, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y6_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.7551191861742355, + "brier": 0.8342293112282068, + "expected_brier": 0.8342293112282069, + "multiclass_ece": 0.33491552620183757, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x3_y2_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.1828687040379144, + "brier": 0.9621660538486306, + "expected_brier": 0.9621660538486307, + "multiclass_ece": 0.6936014899957433, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x3_y4": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.132082250723196, + "brier": 0.8346896343844711, + "expected_brier": 0.8346896343844712, + "multiclass_ece": 0.8814098977419054, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x3_y5_red": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 3.960924181380544, + "brier": 1.9245434438757911, + "expected_brier": 1.9245434438757911, + "multiclass_ece": 0.9809544953451692, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x3_y6_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.7646224088841964, + "brier": 0.8570425813007164, + "expected_brier": 0.8570425813007163, + "multiclass_ece": 0.3726535751547344, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x3_y7_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.9241943348002006, + "brier": 0.42240309601009246, + "expected_brier": 0.4224030960100924, + "multiclass_ece": 0.6031489806319019, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y0_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.3625799736829079, + "brier": 0.1849795795266861, + "expected_brier": 0.18497957952668598, + "multiclass_ece": 0.30412134052601936, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y0_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.9588091550604292, + "brier": 0.5769641600992839, + "expected_brier": 0.5769641600992839, + "multiclass_ece": 0.6166508763689913, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y2": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.6733227272462897, + "brier": 0.2742584161462368, + "expected_brier": 0.2742584161462367, + "multiclass_ece": 0.4899888686096108, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y4_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.6619695281149951, + "brier": 0.31084473498985193, + "expected_brier": 0.310844734989852, + "multiclass_ece": 0.4841656169568871, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y5_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.5628314763414861, + "brier": 0.7693777541136484, + "expected_brier": 0.7693777541136484, + "multiclass_ece": 0.3364040446469659, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y6_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.6236275207587441, + "brier": 0.2904772500378621, + "expected_brier": 0.29047725003786207, + "multiclass_ece": 0.4640034319152242, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y7_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.103540453734151, + "brier": 0.6719481323401812, + "expected_brier": 0.6719481323401812, + "multiclass_ece": 0.3657760566115965, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y2": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.8454830421014222, + "brier": 0.36432123391293847, + "expected_brier": 0.36432123391293847, + "multiclass_ece": 0.5706500859720818, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y2_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.3201481730091522, + "brier": 1.0742973594962906, + "expected_brier": 1.0742973594962906, + "multiclass_ece": 0.7329042773433275, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y3_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.315188230300974, + "brier": 0.14617660255565004, + "expected_brier": 0.14617660255565013, + "multiclass_ece": 0.2703484811827598, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y5_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.3114278906890142, + "brier": 0.8006041689771006, + "expected_brier": 0.8006041689771006, + "multiclass_ece": 0.3681518576219529, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y6_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.02643002787109389, + "brier": 0.0013607304509454126, + "expected_brier": 0.0013607304509453133, + "multiclass_ece": 0.026083811559522996, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y6_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.7115371252744245, + "brier": 0.34291463415340623, + "expected_brier": 0.3429146341534062, + "multiclass_ece": 0.5091109407412843, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y7_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.8021364856391814, + "brier": 0.4019855614864363, + "expected_brier": 0.40198556148643627, + "multiclass_ece": 0.551629995994859, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y7_red": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 4.378184231886405, + "brier": 1.9501224210389085, + "expected_brier": 1.9501224210389085, + "multiclass_ece": 0.9874518775714867, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y7_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.218711239987377, + "brier": 0.7709968141069251, + "expected_brier": 0.7709968141069252, + "multiclass_ece": 0.46784660295582964, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y1_red": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.022853475584988046, + "brier": 0.0010210059528201342, + "expected_brier": 0.0010210059528201754, + "multiclass_ece": 0.022594312921840887, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y2_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.05267124362745212, + "brier": 0.005265052572069336, + "expected_brier": 0.00526505257206944, + "multiclass_ece": 0.05130815028857172, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y2_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.0181577043499472, + "brier": 0.568068051044359, + "expected_brier": 0.568068051044359, + "multiclass_ece": 0.6387401250220291, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y2_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.5896934025949698, + "brier": 0.268477238344988, + "expected_brier": 0.26847723834498805, + "multiclass_ece": 0.445502733901865, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y4": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.885762583277562, + "brier": 0.7801427135417779, + "expected_brier": 0.7801427135417779, + "multiclass_ece": 0.848278780608426, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y5": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.857050288981189, + "brier": 0.7719076147021074, + "expected_brier": 0.7719076147021076, + "multiclass_ece": 0.843867502454673, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y6_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.4207972758016434, + "brier": 0.7624785717937471, + "expected_brier": 0.7624785717937472, + "multiclass_ece": 0.41094623703433103, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y6_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.1138547336962583, + "brier": 0.6819257099988381, + "expected_brier": 0.6819257099988381, + "multiclass_ece": 0.3865315355299282, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y0_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.7383974928966336, + "brier": 0.4229909511835375, + "expected_brier": 0.42299095118353747, + "multiclass_ece": 0.5221208930916642, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y1_green": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 3.956799210566227, + "brier": 1.9242345560264724, + "expected_brier": 1.9242345560264722, + "multiclass_ece": 0.9808757709380104, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y2_green": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 3.778578501557127, + "brier": 1.9096240887361153, + "expected_brier": 1.9096240887361153, + "multiclass_ece": 0.9771448430852294, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y5_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.5838771188083263, + "brier": 0.822416530728612, + "expected_brier": 0.822416530728612, + "multiclass_ece": 0.40981591865580547, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y5_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.0512265646610441, + "brier": 0.536173091584925, + "expected_brier": 0.536173091584925, + "multiclass_ece": 0.6504912090392363, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y7": { + "count": 2, + "hard_count": 2, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 1.3201839402444873, + "brier": 0.47109984513627706, + "expected_brier": 0.4710998451362771, + "multiclass_ece": 0.09564924146432739, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "reasoning-control-v1/decision": { + "count": 24, + "hard_count": 24, + "accuracy": 0.6666666666666666, + "expected_accuracy": 0.6666666666666666, + "nll": 1.028136136696422, + "brier": 0.5642064474803004, + "expected_brier": 0.5642064474803005, + "multiclass_ece": 0.2137625148874377, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 8, + "coverage": 0.3333333333333333, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 0.08333333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 0.041666666666666664, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "reasoning-control-v1/ordinal": { + "count": 24, + "hard_count": 24, + "accuracy": 0.4583333333333333, + "expected_accuracy": 0.4583333333333333, + "nll": 1.2102182551782819, + "brier": 0.6382652373044065, + "expected_brier": 0.6382652373044065, + "multiclass_ece": 0.18944660408235311, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 7, + "coverage": 0.2916666666666667, + "accuracy": 0.7142857142857143, + "expected_accuracy": 0.7142857142857143, + "expected_risk": 0.2857142857142857 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 0.08333333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 0.08333333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "reasoning-control-v1/verification": { + "count": 24, + "hard_count": 24, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 0.6688668557953767, + "brier": 0.490318781315141, + "expected_brier": 0.4903187813151409, + "multiclass_ece": 0.23291217383750948, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.7, + "selected": 7, + "coverage": 0.2916666666666667, + "accuracy": 0.8571428571428571, + "expected_accuracy": 0.8571428571428571, + "expected_risk": 0.1428571428571429 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 0.125, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 0.125, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 0.08333333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "snake-v1/choice": { + "count": 24, + "hard_count": 19, + "accuracy": 0.47368421052631576, + "expected_accuracy": 0.46527777777777773, + "nll": 1.1515063455585253, + "brier": 0.5836142898522181, + "expected_brier": 0.7016698454077738, + "multiclass_ece": 0.25736887356981936, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 21, + "coverage": 0.875, + "accuracy": 0.5, + "expected_accuracy": 0.4920634920634921, + "expected_risk": 0.5079365079365079 + }, + { + "threshold": 0.7, + "selected": 13, + "coverage": 0.5416666666666666, + "accuracy": 0.45454545454545453, + "expected_accuracy": 0.46153846153846156, + "expected_risk": 0.5384615384615384 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "snake-v1/noul": { + "count": 24, + "hard_count": 24, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "nll": 0.7577187350997839, + "brier": 0.44870192762667455, + "expected_brier": 0.44870192762667455, + "multiclass_ece": 0.1920654349523665, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.7, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.8, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.9, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 0.125, + "accuracy": 0.6666666666666666, + "expected_accuracy": 0.6666666666666666, + "expected_risk": 0.33333333333333337 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "tic_tac_toe-v1/choice": { + "count": 24, + "hard_count": 7, + "accuracy": 0.5714285714285714, + "expected_accuracy": 0.2767857142857143, + "nll": 1.304323480548012, + "brier": 0.29733120471959634, + "expected_brier": 0.7149899348783265, + "multiclass_ece": 0.1572187744510727, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 0.20833333333333334, + "accuracy": 1.0, + "expected_accuracy": 0.7, + "expected_risk": 0.30000000000000004 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "tile_platformer-v1/choice": { + "count": 24, + "hard_count": 24, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.9909957729024932, + "brier": 0.9130921692916404, + "expected_brier": 0.9130921692916404, + "multiclass_ece": 0.2792539473683758, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "trex_runner-v1/choice": { + "count": 4, + "hard_count": 4, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 1.113003902721162, + "brier": 0.5930013921090308, + "expected_brier": 0.5930013921090307, + "multiclass_ece": 0.1658166315815956, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 0.75, + "accuracy": 0.6666666666666666, + "expected_accuracy": 0.6666666666666666, + "expected_risk": 0.33333333333333337 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "vizdoom-basic-v1/choice": { + "count": 24, + "hard_count": 24, + "accuracy": 0.375, + "expected_accuracy": 0.375, + "nll": 1.1247964681867277, + "brier": 0.6837702554578765, + "expected_brier": 0.6837702554578765, + "multiclass_ece": 0.049191069708482214, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "vizdoom-basic-v1/noul": { + "count": 25, + "hard_count": 25, + "accuracy": 0.64, + "expected_accuracy": 0.64, + "nll": 0.6064517075940102, + "brier": 0.42398620335049003, + "expected_brier": 0.4239862033504902, + "multiclass_ece": 0.24960668263794836, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 25, + "coverage": 1.0, + "accuracy": 0.64, + "expected_accuracy": 0.64, + "expected_risk": 0.36 + }, + { + "threshold": 0.7, + "selected": 12, + "coverage": 0.48, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 0.08, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "vizdoom-basic-v1/score": { + "count": 24, + "hard_count": 24, + "accuracy": 0.16666666666666666, + "expected_accuracy": 0.16666666666666666, + "nll": 1.5089011321764934, + "brier": 0.7766657494016291, + "expected_brier": 0.7766657494016291, + "multiclass_ece": 0.1800474215556728, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "wikispeedia-v1/choice": { + "count": 24, + "hard_count": 8, + "accuracy": 0.25, + "expected_accuracy": 0.22481812169312174, + "nll": 2.248698265794124, + "brier": 0.39996020810020355, + "expected_brier": 0.8751420864070817, + "multiclass_ece": 0.07621955438195471, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 0.041666666666666664, + "accuracy": null, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/AUTO-CLOSE": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.18089308854668307, + "brier": 0.06072774960009216, + "expected_brier": 0.06072774960009212, + "multiclass_ece": 0.16316525980888064, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/COUNT ONLY": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.20744734770670215, + "brier": 0.0703951587109352, + "expected_brier": 0.07039515871093527, + "multiclass_ece": 0.18726447210249564, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/FILE ISSUE ยท ROUTE": { + "count": 5, + "hard_count": 5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.11420520341296884, + "brier": 0.023378477899824154, + "expected_brier": 0.023378477899824147, + "multiclass_ece": 0.10789869779766832, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 0.2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/HUMAN REVIEW": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.3061394188744529, + "brier": 0.14203544784594604, + "expected_brier": 0.14203544784594602, + "multiclass_ece": 0.2622079729814394, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 0.75, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 0.25, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/NOT A BUG": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.16029784897168978, + "brier": 0.04387313435973226, + "expected_brier": 0.04387313435973228, + "multiclass_ece": 0.14810998339027026, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/PAGE ON-CALL": { + "count": 5, + "hard_count": 5, + "accuracy": 0.4, + "expected_accuracy": 0.4, + "nll": 1.1732490525521546, + "brier": 0.8584357723075986, + "expected_brier": 0.8584357723075986, + "multiclass_ece": 0.43471997975638005, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 1.0, + "accuracy": 0.4, + "expected_accuracy": 0.4, + "expected_risk": 0.6 + }, + { + "threshold": 0.7, + "selected": 5, + "coverage": 1.0, + "accuracy": 0.4, + "expected_accuracy": 0.4, + "expected_risk": 0.6 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 0.6, + "accuracy": 0.3333333333333333, + "expected_accuracy": 0.3333333333333333, + "expected_risk": 0.6666666666666667 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/PRIORITY REVIEW": { + "count": 3, + "hard_count": 3, + "accuracy": 0.6666666666666666, + "expected_accuracy": 0.6666666666666666, + "nll": 0.6007617641001378, + "brier": 0.4174625613881593, + "expected_brier": 0.4174625613881593, + "multiclass_ece": 0.39657408868828004, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 0.6666666666666666, + "expected_accuracy": 0.6666666666666666, + "expected_risk": 0.33333333333333337 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 0.6666666666666666, + "expected_accuracy": 0.6666666666666666, + "expected_risk": 0.33333333333333337 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/CANCEL": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.2461794472240767, + "brier": 0.09556143595878648, + "expected_brier": 0.09556143595878652, + "multiclass_ece": 0.21807474903060187, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/CLOSE": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.42343890642932736, + "brier": 0.24088484318571643, + "expected_brier": 0.24088484318571637, + "multiclass_ece": 0.34330554668263846, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.3333333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/FLAG FOR REVIEW": { + "count": 3, + "hard_count": 3, + "accuracy": 0.3333333333333333, + "expected_accuracy": 0.3333333333333333, + "nll": 0.808550893443008, + "brier": 0.6218561249757514, + "expected_brier": 0.6218561249757514, + "multiclass_ece": 0.5098041778422733, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 0.3333333333333333, + "expected_accuracy": 0.3333333333333333, + "expected_risk": 0.6666666666666667 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.3333333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 0.3333333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/FREEZE CARD": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.1352633608717024, + "brier": 0.032161727670351026, + "expected_brier": 0.03216172767035105, + "multiclass_ece": 0.12646368921852277, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/HAND OFF": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.4533487667659473, + "brier": 0.26572566873978487, + "expected_brier": 0.26572566873978487, + "multiclass_ece": 0.3645035450717763, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/REFUND": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.5278787167824842, + "brier": 0.33651576619753015, + "expected_brier": 0.33651576619753015, + "multiclass_ece": 0.4100382831076648, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/SAY": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.4091952466637975, + "brier": 0.22598379955911987, + "expected_brier": 0.2259837995591199, + "multiclass_ece": 0.33547009687472185, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/SET INTENT": { + "count": 5, + "hard_count": 5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.558032164160687, + "brier": 0.37137006021999686, + "expected_brier": 0.37137006021999686, + "multiclass_ece": 0.4215689892243846, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/DISPUTE LINES": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.3245494216897685, + "brier": 0.1536209390049275, + "expected_brier": 0.15362093900492746, + "multiclass_ece": 0.27714701784876516, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/DUPLICATE": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.1593680857295851, + "brier": 0.043404926403054336, + "expected_brier": 0.04340492640305427, + "multiclass_ece": 0.14731755904007904, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/FRAUD REVIEW": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.18226121391655511, + "brier": 0.055967374861920945, + "expected_brier": 0.05596737486192088, + "multiclass_ece": 0.16645219151675517, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/HOLD": { + "count": 4, + "hard_count": 4, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.6006638293461513, + "brier": 1.2708610841033188, + "expected_brier": 1.2708610841033188, + "multiclass_ece": 0.7967633493435481, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 0.5, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/HOLD FOR DOCUMENTS": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.4199150162646976, + "brier": 0.23515717089867214, + "expected_brier": 0.23515717089867216, + "multiclass_ece": 0.3428973395191862, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/PAY": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.32722747174741146, + "brier": 0.16249127077805606, + "expected_brier": 0.16249127077805614, + "multiclass_ece": 0.2755714909091256, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.3333333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 0.3333333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/REQUEST CORRECTED INVOICE": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.27007595726489, + "brier": 0.11466543721867586, + "expected_brier": 0.11466543721867584, + "multiclass_ece": 0.23543200327648584, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/ROUTE FOR APPROVAL": { + "count": 4, + "hard_count": 4, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "nll": 0.6364141824111614, + "brier": 0.4388310740906164, + "expected_brier": 0.4388310740906164, + "multiclass_ece": 0.4400301973476809, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.25, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/SCHEDULE": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.2499484520940442, + "brier": 0.0978536810533709, + "expected_brier": 0.09785368105337089, + "multiclass_ece": 0.22114516838185116, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/SHORT PAY": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.30183952738549924, + "brier": 0.13596630162092252, + "expected_brier": 0.1359663016209225, + "multiclass_ece": 0.2604385376461942, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/BLOCK EGRESS ASSET": { + "count": 3, + "hard_count": 3, + "accuracy": 0.3333333333333333, + "expected_accuracy": 0.3333333333333333, + "nll": 0.5783778938100491, + "brier": 0.3955546940145814, + "expected_brier": 0.39555469401458154, + "multiclass_ece": 0.42651667109488134, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 0.3333333333333333, + "expected_accuracy": 0.3333333333333333, + "expected_risk": 0.6666666666666667 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.3333333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/BLOCK SENDER": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 0.7536177616583075, + "brier": 0.5604001242001068, + "expected_brier": 0.5604001242001067, + "multiclass_ece": 0.5293392693727279, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/DISABLE ACCOUNT": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.40286686988084186, + "brier": 0.22402640559249803, + "expected_brier": 0.2240264055924981, + "multiclass_ece": 0.32857479256574246, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/ESCALATE TIER2": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.45780910233585936, + "brier": 0.2707057803400803, + "expected_brier": 0.2707057803400803, + "multiclass_ece": 0.3665421573494756, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/ESCALATE URGENT": { + "count": 3, + "hard_count": 3, + "accuracy": 0.6666666666666666, + "expected_accuracy": 0.6666666666666666, + "nll": 0.7043052804280983, + "brier": 0.5106776717673468, + "expected_brier": 0.5106776717673468, + "multiclass_ece": 0.19794519010666745, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 0.6666666666666666, + "expected_accuracy": 0.6666666666666666, + "expected_risk": 0.33333333333333337 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/ISOLATE HOST": { + "count": 3, + "hard_count": 3, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 0.803007969535288, + "brier": 0.6086782990009985, + "expected_brier": 0.6086782990009985, + "multiclass_ece": 0.550287110318156, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/KILL PROCESS": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.5434558405734037, + "brier": 0.35156151233793353, + "expected_brier": 0.35156151233793365, + "multiclass_ece": 0.4192621568529251, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/NOTIFY USER": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.4654943867657212, + "brier": 0.28038702010718586, + "expected_brier": 0.28038702010718586, + "multiclass_ece": 0.36893267194230517, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/QUARANTINE FILE": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.4888853272381369, + "brier": 0.2990588491217942, + "expected_brier": 0.29905884912179426, + "multiclass_ece": 0.38669034712660866, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/QUARANTINE MESSAGE": { + "count": 2, + "hard_count": 2, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 0.6008744991100896, + "brier": 0.4107578235735651, + "expected_brier": 0.41075782357356505, + "multiclass_ece": 0.44501456572088505, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/REMOVE FORWARDING RULES": { + "count": 2, + "hard_count": 2, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.0271714381879997, + "brier": 0.8223455755274527, + "expected_brier": 0.8223455755274527, + "multiclass_ece": 0.6402672874044717, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/REVOKE ACCESS KEY": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.3450675630941394, + "brier": 0.17032760800066302, + "expected_brier": 0.17032760800066304, + "multiclass_ece": 0.29182687303232857, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/REVOKE SESSIONS": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 0.956313230754476, + "brier": 0.7581554225946643, + "expected_brier": 0.7581554225946643, + "multiclass_ece": 0.6156928709164433, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + } + } + }, + "baseline_ood": { + "count": 512, + "hard_count": 484, + "accuracy": 0.5929752066115702, + "expected_accuracy": 0.5720486111111112, + "nll": 0.9614150233345907, + "brier": 0.48374395093374994, + "expected_brier": 0.520956407530972, + "multiclass_ece": 0.061707229004274985, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 337, + "coverage": 0.658203125, + "accuracy": 0.7242424242424242, + "expected_accuracy": 0.7195845697329377, + "expected_risk": 0.2804154302670623 + }, + { + "threshold": 0.7, + "selected": 237, + "coverage": 0.462890625, + "accuracy": 0.7905982905982906, + "expected_accuracy": 0.7869198312236287, + "expected_risk": 0.21308016877637126 + }, + { + "threshold": 0.8, + "selected": 123, + "coverage": 0.240234375, + "accuracy": 0.8211382113821138, + "expected_accuracy": 0.8211382113821138, + "expected_risk": 0.17886178861788615 + }, + { + "threshold": 0.9, + "selected": 51, + "coverage": 0.099609375, + "accuracy": 0.9019607843137255, + "expected_accuracy": 0.9019607843137255, + "expected_risk": 0.0980392156862745 + }, + { + "threshold": 0.95, + "selected": 13, + "coverage": 0.025390625, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ], + "mean_latency_seconds": 0.044071764506952604, + "by_kind": { + "choice": { + "count": 162, + "hard_count": 136, + "accuracy": 0.47058823529411764, + "expected_accuracy": 0.4272976680384088, + "nll": 1.36500787323019, + "brier": 0.5353468031818399, + "expected_brier": 0.6447260898759417, + "multiclass_ece": 0.07059474102818601, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 62, + "coverage": 0.38271604938271603, + "accuracy": 0.7454545454545455, + "expected_accuracy": 0.717741935483871, + "expected_risk": 0.282258064516129 + }, + { + "threshold": 0.7, + "selected": 37, + "coverage": 0.22839506172839505, + "accuracy": 0.8529411764705882, + "expected_accuracy": 0.8243243243243243, + "expected_risk": 0.17567567567567566 + }, + { + "threshold": 0.8, + "selected": 13, + "coverage": 0.08024691358024691, + "accuracy": 0.8461538461538461, + "expected_accuracy": 0.8461538461538461, + "expected_risk": 0.15384615384615385 + }, + { + "threshold": 0.9, + "selected": 4, + "coverage": 0.024691358024691357, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 0.006172839506172839, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "noul": { + "count": 243, + "hard_count": 243, + "accuracy": 0.7654320987654321, + "expected_accuracy": 0.7654320987654321, + "nll": 0.5319400600486455, + "brier": 0.345469812103677, + "expected_brier": 0.34546981210367705, + "multiclass_ece": 0.08793579614763246, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 243, + "coverage": 1.0, + "accuracy": 0.7654320987654321, + "expected_accuracy": 0.7654320987654321, + "expected_risk": 0.23456790123456794 + }, + { + "threshold": 0.7, + "selected": 177, + "coverage": 0.7283950617283951, + "accuracy": 0.8192090395480226, + "expected_accuracy": 0.8192090395480226, + "expected_risk": 0.1807909604519774 + }, + { + "threshold": 0.8, + "selected": 100, + "coverage": 0.411522633744856, + "accuracy": 0.81, + "expected_accuracy": 0.81, + "expected_risk": 0.18999999999999995 + }, + { + "threshold": 0.9, + "selected": 43, + "coverage": 0.17695473251028807, + "accuracy": 0.8837209302325582, + "expected_accuracy": 0.8837209302325582, + "expected_risk": 0.11627906976744184 + }, + { + "threshold": 0.95, + "selected": 11, + "coverage": 0.04526748971193416, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "score": { + "count": 107, + "hard_count": 105, + "accuracy": 0.3523809523809524, + "expected_accuracy": 0.3520249221183801, + "nll": 1.3257175877775578, + "brier": 0.7196407142189585, + "expected_brier": 0.7321017734089895, + "multiclass_ece": 0.1730863778484494, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 32, + "coverage": 0.29906542056074764, + "accuracy": 0.375, + "expected_accuracy": 0.375, + "expected_risk": 0.625 + }, + { + "threshold": 0.7, + "selected": 23, + "coverage": 0.21495327102803738, + "accuracy": 0.4782608695652174, + "expected_accuracy": 0.4782608695652174, + "expected_risk": 0.5217391304347826 + }, + { + "threshold": 0.8, + "selected": 10, + "coverage": 0.09345794392523364, + "accuracy": 0.9, + "expected_accuracy": 0.9, + "expected_risk": 0.09999999999999998 + }, + { + "threshold": 0.9, + "selected": 4, + "coverage": 0.037383177570093455, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 0.009345794392523364, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ], + "ordinal_mae": 0.8628190771141426 + } + }, + "by_source": { + "customer-control-v1": { + "count": 81, + "hard_count": 77, + "accuracy": 0.7272727272727273, + "expected_accuracy": 0.7119341563786008, + "nll": 0.6505367220992968, + "brier": 0.37021600236698843, + "expected_brier": 0.39902258672912827, + "multiclass_ece": 0.11351452318462084, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 73, + "coverage": 0.9012345679012346, + "accuracy": 0.7605633802816901, + "expected_accuracy": 0.7534246575342466, + "expected_risk": 0.2465753424657534 + }, + { + "threshold": 0.7, + "selected": 57, + "coverage": 0.7037037037037037, + "accuracy": 0.7719298245614035, + "expected_accuracy": 0.7719298245614035, + "expected_risk": 0.22807017543859653 + }, + { + "threshold": 0.8, + "selected": 34, + "coverage": 0.41975308641975306, + "accuracy": 0.9117647058823529, + "expected_accuracy": 0.9117647058823529, + "expected_risk": 0.08823529411764708 + }, + { + "threshold": 0.9, + "selected": 18, + "coverage": 0.2222222222222222, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 8, + "coverage": 0.09876543209876543, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1": { + "count": 81, + "hard_count": 81, + "accuracy": 0.5555555555555556, + "expected_accuracy": 0.5555555555555556, + "nll": 1.2632983395515622, + "brier": 0.6399351038611562, + "expected_brier": 0.6399351038611559, + "multiclass_ece": 0.23465058254069918, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 28, + "coverage": 0.345679012345679, + "accuracy": 0.6428571428571429, + "expected_accuracy": 0.6428571428571429, + "expected_risk": 0.3571428571428571 + }, + { + "threshold": 0.7, + "selected": 24, + "coverage": 0.2962962962962963, + "accuracy": 0.6666666666666666, + "expected_accuracy": 0.6666666666666666, + "expected_risk": 0.33333333333333337 + }, + { + "threshold": 0.8, + "selected": 15, + "coverage": 0.18518518518518517, + "accuracy": 0.5333333333333333, + "expected_accuracy": 0.5333333333333333, + "expected_risk": 0.4666666666666667 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 0.012345679012345678, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "reasoning-control-v1": { + "count": 80, + "hard_count": 80, + "accuracy": 0.4125, + "expected_accuracy": 0.4125, + "nll": 1.0713648686645802, + "brier": 0.6175801193688644, + "expected_brier": 0.6175801193688644, + "multiclass_ece": 0.1485804045916807, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 49, + "coverage": 0.6125, + "accuracy": 0.5510204081632653, + "expected_accuracy": 0.5510204081632653, + "expected_risk": 0.44897959183673475 + }, + { + "threshold": 0.7, + "selected": 21, + "coverage": 0.2625, + "accuracy": 0.7142857142857143, + "expected_accuracy": 0.7142857142857143, + "expected_risk": 0.2857142857142857 + }, + { + "threshold": 0.8, + "selected": 13, + "coverage": 0.1625, + "accuracy": 0.8461538461538461, + "expected_accuracy": 0.8461538461538461, + "expected_risk": 0.15384615384615385 + }, + { + "threshold": 0.9, + "selected": 5, + "coverage": 0.0625, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 0.025, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "snake-v1": { + "count": 54, + "hard_count": 48, + "accuracy": 0.7291666666666666, + "expected_accuracy": 0.7037037037037037, + "nll": 0.7977193296776489, + "brier": 0.4069564903973643, + "expected_brier": 0.4625120459529199, + "multiclass_ece": 0.1438453240350075, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 52, + "coverage": 0.9629629629629629, + "accuracy": 0.723404255319149, + "expected_accuracy": 0.7019230769230769, + "expected_risk": 0.29807692307692313 + }, + { + "threshold": 0.7, + "selected": 44, + "coverage": 0.8148148148148148, + "accuracy": 0.7804878048780488, + "expected_accuracy": 0.7613636363636364, + "expected_risk": 0.23863636363636365 + }, + { + "threshold": 0.8, + "selected": 30, + "coverage": 0.5555555555555556, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.9, + "selected": 27, + "coverage": 0.5, + "accuracy": 0.8518518518518519, + "expected_accuracy": 0.8518518518518519, + "expected_risk": 0.14814814814814814 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 0.05555555555555555, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "vizdoom-basic-v1": { + "count": 81, + "hard_count": 81, + "accuracy": 0.37037037037037035, + "expected_accuracy": 0.37037037037037035, + "nll": 1.0737570725528611, + "brier": 0.6180701837775169, + "expected_brier": 0.6180701837775168, + "multiclass_ece": 0.17714415322089644, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 0.3333333333333333, + "accuracy": 0.7037037037037037, + "expected_accuracy": 0.7037037037037037, + "expected_risk": 0.2962962962962963 + }, + { + "threshold": 0.7, + "selected": 23, + "coverage": 0.2839506172839506, + "accuracy": 0.7391304347826086, + "expected_accuracy": 0.7391304347826086, + "expected_risk": 0.26086956521739135 + }, + { + "threshold": 0.8, + "selected": 5, + "coverage": 0.06172839506172839, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "wikispeedia-v1": { + "count": 27, + "hard_count": 9, + "accuracy": 0.1111111111111111, + "expected_accuracy": 0.0823045267489712, + "nll": 2.577358288876876, + "brier": 0.43560974759709586, + "expected_brier": 0.9437373196135566, + "multiclass_ece": 0.12338939883834045, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ], + "optimal_action_hit": 0.2962962962962963, + "optimal_set_probability_mass": 0.3444428310250154 + }, + "workflow-controls-v1/agent_trace_observability": { + "count": 27, + "hard_count": 27, + "accuracy": 0.7777777777777778, + "expected_accuracy": 0.7777777777777778, + "nll": 0.5076084768841964, + "brier": 0.32608026315635136, + "expected_brier": 0.3260802631563513, + "multiclass_ece": 0.17300981658105485, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.7777777777777778, + "expected_accuracy": 0.7777777777777778, + "expected_risk": 0.2222222222222222 + }, + { + "threshold": 0.7, + "selected": 22, + "coverage": 0.8148148148148148, + "accuracy": 0.8636363636363636, + "expected_accuracy": 0.8636363636363636, + "expected_risk": 0.13636363636363635 + }, + { + "threshold": 0.8, + "selected": 17, + "coverage": 0.6296296296296297, + "accuracy": 0.8235294117647058, + "expected_accuracy": 0.8235294117647058, + "expected_risk": 0.17647058823529416 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service": { + "count": 27, + "hard_count": 27, + "accuracy": 0.9259259259259259, + "expected_accuracy": 0.9259259259259259, + "nll": 0.44263673033044626, + "brier": 0.26728882796913134, + "expected_brier": 0.2672888279691314, + "multiclass_ece": 0.25107769908611144, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.9259259259259259, + "expected_accuracy": 0.9259259259259259, + "expected_risk": 0.07407407407407407 + }, + { + "threshold": 0.7, + "selected": 12, + "coverage": 0.4444444444444444, + "accuracy": 0.9166666666666666, + "expected_accuracy": 0.9166666666666666, + "expected_risk": 0.08333333333333337 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 0.07407407407407407, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing": { + "count": 27, + "hard_count": 27, + "accuracy": 0.8888888888888888, + "expected_accuracy": 0.8888888888888888, + "nll": 0.4374955649213505, + "brier": 0.262064120033491, + "expected_brier": 0.262064120033491, + "multiclass_ece": 0.14110822745576662, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.8888888888888888, + "expected_accuracy": 0.8888888888888888, + "expected_risk": 0.11111111111111116 + }, + { + "threshold": 0.7, + "selected": 21, + "coverage": 0.7777777777777778, + "accuracy": 0.8571428571428571, + "expected_accuracy": 0.8571428571428571, + "expected_risk": 0.1428571428571429 + }, + { + "threshold": 0.8, + "selected": 7, + "coverage": 0.25925925925925924, + "accuracy": 0.8571428571428571, + "expected_accuracy": 0.8571428571428571, + "expected_risk": 0.1428571428571429 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents": { + "count": 27, + "hard_count": 27, + "accuracy": 0.6296296296296297, + "expected_accuracy": 0.6296296296296297, + "nll": 0.5335489308778574, + "brier": 0.3537317951941112, + "expected_brier": 0.35373179519411113, + "multiclass_ece": 0.3284688411226127, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.6296296296296297, + "expected_accuracy": 0.6296296296296297, + "expected_risk": 0.37037037037037035 + }, + { + "threshold": 0.7, + "selected": 13, + "coverage": 0.48148148148148145, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + } + }, + "by_question": { + "customer-control-v1/bug_severity": { + "count": 8, + "hard_count": 8, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.866114580940732, + "brier": 1.2360681241195373, + "expected_brier": 1.2360681241195373, + "multiclass_ece": 0.7090090895815236, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 8, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 5, + "coverage": 0.625, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "customer-control-v1/category": { + "count": 27, + "hard_count": 25, + "accuracy": 1.0, + "expected_accuracy": 0.9629629629629629, + "nll": 0.3194373035853267, + "brier": 0.09454076556088625, + "expected_brier": 0.1315778025979233, + "multiclass_ece": 0.19923446714798795, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 0.9629629629629629, + "expected_risk": 0.03703703703703709 + }, + { + "threshold": 0.7, + "selected": 20, + "coverage": 0.7407407407407407, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 10, + "coverage": 0.37037037037037035, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 4, + "coverage": 0.14814814814814814, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 0.037037037037037035, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "customer-control-v1/churn_likelihood_level": { + "count": 9, + "hard_count": 7, + "accuracy": 0.42857142857142855, + "expected_accuracy": 0.4074074074074074, + "nll": 1.0458173184450124, + "brier": 0.4655375283673868, + "expected_brier": 0.6136856765155351, + "multiclass_ece": 0.30494511834948795, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 0.1111111111111111, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.1111111111111111, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 0.1111111111111111, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "customer-control-v1/frustration": { + "count": 10, + "hard_count": 10, + "accuracy": 0.4, + "expected_accuracy": 0.4, + "nll": 1.0637136770996989, + "brier": 0.7731624274380251, + "expected_brier": 0.7731624274380252, + "multiclass_ece": 0.48749886707084, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 10, + "coverage": 1.0, + "accuracy": 0.4, + "expected_accuracy": 0.4, + "expected_risk": 0.6 + }, + { + "threshold": 0.7, + "selected": 10, + "coverage": 1.0, + "accuracy": 0.4, + "expected_accuracy": 0.4, + "expected_risk": 0.6 + }, + { + "threshold": 0.8, + "selected": 5, + "coverage": 0.5, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 0.3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 0.1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "customer-control-v1/has_reproducible_steps": { + "count": 15, + "hard_count": 15, + "accuracy": 0.9333333333333333, + "expected_accuracy": 0.9333333333333333, + "nll": 0.3375759371059293, + "brier": 0.1900974975877034, + "expected_brier": 0.1900974975877034, + "multiclass_ece": 0.21577979708381081, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 15, + "coverage": 1.0, + "accuracy": 0.9333333333333333, + "expected_accuracy": 0.9333333333333333, + "expected_risk": 0.06666666666666665 + }, + { + "threshold": 0.7, + "selected": 9, + "coverage": 0.6, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 6, + "coverage": 0.4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 0.2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "customer-control-v1/refund_requested": { + "count": 12, + "hard_count": 12, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "nll": 0.3355515793435265, + "brier": 0.23111883626029542, + "expected_brier": 0.23111883626029547, + "multiclass_ece": 0.18493655829378228, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 12, + "coverage": 1.0, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.7, + "selected": 12, + "coverage": 1.0, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.8, + "selected": 12, + "coverage": 1.0, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.9, + "selected": 8, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 6, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y11_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.1210475686142187, + "brier": 0.02599573332045748, + "expected_brier": 0.02599573332045746, + "multiclass_ece": 0.11400818681230196, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y11_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.1211791861650335, + "brier": 0.5727056051154836, + "expected_brier": 0.5727056051154835, + "multiclass_ece": 0.6741047232430806, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y3_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.2171774919578426, + "brier": 0.07621614219949233, + "expected_brier": 0.07621614219949224, + "multiclass_ece": 0.19521288661291336, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y8": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.11596765761486866, + "brier": 0.023978737583528725, + "expected_brier": 0.023978737583528753, + "multiclass_ece": 0.10949597614416873, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y8_red": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.5638994298137767, + "brier": 1.703843663989969, + "expected_brier": 1.7038436639899688, + "multiclass_ece": 0.9229961169988661, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x10_y0_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.0427299479395522, + "brier": 0.5269898916881616, + "expected_brier": 0.5269898916881615, + "multiclass_ece": 0.6475089150386963, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x10_y11": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.4131077679228308, + "brier": 0.8927175452553817, + "expected_brier": 0.8927175452553817, + "multiclass_ece": 0.1303008949065322, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x10_y4_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.21578821817490315, + "brier": 0.07534499269849884, + "expected_brier": 0.07534499269849881, + "multiclass_ece": 0.19409403996323382, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x10_y4_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.1732283777405739, + "brier": 0.7209257641004966, + "expected_brier": 0.7209257641004965, + "multiclass_ece": 0.3975851201786825, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x10_y5_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.8358404999332799, + "brier": 0.48148268776072517, + "expected_brier": 0.4814826877607252, + "multiclass_ece": 0.5664900368294585, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x10_y7_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.9165423717764352, + "brier": 0.4598596172403327, + "expected_brier": 0.45985961724033264, + "multiclass_ece": 0.6001006432974462, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x10_y9_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.13190646136460302, + "brier": 0.030542564363926047, + "expected_brier": 0.030542564363926106, + "multiclass_ece": 0.12357702934592263, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x10_y9_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.8633464758303506, + "brier": 0.8485140626827598, + "expected_brier": 0.8485140626827596, + "multiclass_ece": 0.29209914209804316, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x11_y0_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.1214271267354898, + "brier": 0.5620278129852472, + "expected_brier": 0.5620278129852472, + "multiclass_ece": 0.6741855158876077, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x11_y10": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.547989371272067, + "brier": 0.9590232793666599, + "expected_brier": 0.9590232793666598, + "multiclass_ece": 0.2577194725896292, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x11_y1_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.9198614527489248, + "brier": 0.46329117284197735, + "expected_brier": 0.4632911728419773, + "multiclass_ece": 0.6014257413728115, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y0": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.7790169376311558, + "brier": 0.7463327345753376, + "expected_brier": 0.7463327345753377, + "multiclass_ece": 0.8311959894012317, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y10_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.040020265660182, + "brier": 1.5137092576195386, + "expected_brier": 1.5137092576195386, + "multiclass_ece": 0.8699739242125417, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y3_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.8671608106182414, + "brier": 0.5044228218764174, + "expected_brier": 0.5044228218764175, + "multiclass_ece": 0.5798572777796077, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y4_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.3390012807821572, + "brier": 0.834118763568989, + "expected_brier": 0.8341187635689891, + "multiclass_ece": 0.462194212665208, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y6_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.0960927047941555, + "brier": 0.9352012479889672, + "expected_brier": 0.9352012479889673, + "multiclass_ece": 0.3146304268831401, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y7_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.9659079024525905, + "brier": 0.4862967331142099, + "expected_brier": 0.48629673311420984, + "multiclass_ece": 0.6193625388698873, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y7_red": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.7554166029458882, + "brier": 1.3684031345415857, + "expected_brier": 1.3684031345415857, + "multiclass_ece": 0.8271647763721525, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y8_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.1837734865668503, + "brier": 1.02076735333057, + "expected_brier": 1.02076735333057, + "multiclass_ece": 0.38495919162152825, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y9_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.1868200093239136, + "brier": 1.5761177098357373, + "expected_brier": 1.5761177098357373, + "multiclass_ece": 0.8877267906951264, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y9_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.2834394403306066, + "brier": 0.7093984016507052, + "expected_brier": 0.7093984016507053, + "multiclass_ece": 0.4145025826644491, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y10_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.0524267270061147, + "brier": 0.5387443676873233, + "expected_brier": 0.5387443676873231, + "multiclass_ece": 0.6509104247156645, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y11_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.9975784291087288, + "brier": 1.4941555011839394, + "expected_brier": 1.4941555011839394, + "multiclass_ece": 0.8643365956570216, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y4_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.9756647813508317, + "brier": 0.4926552594504331, + "expected_brier": 0.4926552594504332, + "multiclass_ece": 0.6230583135505254, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y6_red": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.4231855729121432, + "brier": 1.1523282439589613, + "expected_brier": 1.1523282439589613, + "multiclass_ece": 0.7590547555871583, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y6_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.3276583162851099, + "brier": 0.8273696128039145, + "expected_brier": 0.8273696128039145, + "multiclass_ece": 0.4603132459956915, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y8": { + "count": 2, + "hard_count": 2, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.289525061104535, + "brier": 0.8784943831455635, + "expected_brier": 0.8784943831455635, + "multiclass_ece": 0.16949821173831903, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x3_y0_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.0643568617028967, + "brier": 0.5428041302252896, + "expected_brier": 0.5428041302252895, + "multiclass_ece": 0.6550503662376626, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x3_y11_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.13600040853561404, + "brier": 0.032338173595824894, + "expected_brier": 0.032338173595824804, + "multiclass_ece": 0.12715772409850867, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x3_y1_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.3451891683081838, + "brier": 0.17042710756181897, + "expected_brier": 0.1704271075618189, + "multiclass_ece": 0.29191360670737754, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x3_y6": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.63875283488097, + "brier": 0.9689504174704332, + "expected_brier": 0.9689504174704333, + "multiclass_ece": 0.26233374168230544, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x3_y7_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.34541653683054285, + "brier": 0.17061512636254206, + "expected_brier": 0.17061512636254195, + "multiclass_ece": 0.2920745849629355, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x3_y7_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.5516390385535432, + "brier": 0.20494022861019515, + "expected_brier": 0.20494022861019512, + "multiclass_ece": 0.42399505805276305, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y10_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.7125151771048408, + "brier": 0.8020780136063839, + "expected_brier": 0.8020780136063839, + "multiclass_ece": 0.2878913620424436, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y11_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.085212405027125, + "brier": 0.5538622702546157, + "expected_brier": 0.5538622702546157, + "multiclass_ece": 0.6621699785214543, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y1_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.582934946980257, + "brier": 0.8711421010695134, + "expected_brier": 0.8711421010695135, + "multiclass_ece": 0.4719898305228452, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y1_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.7740258916612859, + "brier": 0.43556730546629185, + "expected_brier": 0.43556730546629185, + "multiclass_ece": 0.5388472249573687, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y4_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.8287777990816303, + "brier": 0.47649053187478196, + "expected_brier": 0.4764905318747821, + "multiclass_ece": 0.5634174480277654, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y5_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.2454620891687005, + "brier": 0.7707409840758577, + "expected_brier": 0.7707409840758577, + "multiclass_ece": 0.4265040473202603, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y6_red": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.837032950689429, + "brier": 1.4135888108655565, + "expected_brier": 1.4135888108655568, + "multiclass_ece": 0.8407106550013377, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y1_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.0298066834310384, + "brier": 0.5245917133479685, + "expected_brier": 0.5245917133479685, + "multiclass_ece": 0.6429240173988038, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y2": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.2389173058707377, + "brier": 0.561966940310325, + "expected_brier": 0.5619669403103251, + "multiclass_ece": 0.7103022977939206, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y4_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.7415860931807107, + "brier": 0.31900796229597744, + "expected_brier": 0.3190079622959774, + "multiclass_ece": 0.5236422317882308, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y4_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.013255392450606, + "brier": 0.5119562110211175, + "expected_brier": 0.5119562110211175, + "multiclass_ece": 0.6369647683175654, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y7": { + "count": 2, + "hard_count": 2, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 1.9885575934237958, + "brier": 0.7500028687189672, + "expected_brier": 0.7500028687189673, + "multiclass_ece": 0.4375065200972262, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y8": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 0.8019131705097173, + "brier": 0.6083703663436497, + "expected_brier": 0.6083703663436497, + "multiclass_ece": 0.5515298570085075, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y9": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.618598141846503, + "brier": 0.9609783583395789, + "expected_brier": 0.9609783583395788, + "multiclass_ece": 0.24023988372304408, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y9_red": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.1332495688302654, + "brier": 0.03112680636751543, + "expected_brier": 0.031126806367515414, + "multiclass_ece": 0.12475336942847559, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y0_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.131097090272647, + "brier": 0.9911077442062698, + "expected_brier": 0.9911077442062699, + "multiclass_ece": 0.3378663235200809, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y1_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.045266118522619, + "brier": 0.8981592022099203, + "expected_brier": 0.8981592022099203, + "multiclass_ece": 0.2473865103559446, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y3": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.461574570347851, + "brier": 0.9186741337551063, + "expected_brier": 0.9186741337551063, + "multiclass_ece": 0.19911259255226896, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y3_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.3252633204614914, + "brier": 0.15419334713552452, + "expected_brier": 0.15419334713552457, + "multiclass_ece": 0.2776628775471476, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y6_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.7917071972296357, + "brier": 0.3893053771719337, + "expected_brier": 0.3893053771719337, + "multiclass_ece": 0.5469293463006308, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y7_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.8443429666958051, + "brier": 0.4156685885807811, + "expected_brier": 0.41566858858078115, + "multiclass_ece": 0.5701603155601433, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y8_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.9806054673885636, + "brier": 1.486137423378543, + "expected_brier": 1.4861374233785432, + "multiclass_ece": 0.8620143338073164, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y1_lightness": { + "count": 2, + "hard_count": 2, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 1.638730440220779, + "brier": 0.7781160594461305, + "expected_brier": 0.7781160594461304, + "multiclass_ece": 0.520138402175157, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y2": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.3178295280419465, + "brier": 0.14826535291120677, + "expected_brier": 0.1482653529112068, + "multiclass_ece": 0.2722731651404585, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y5_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.25554765819593467, + "brier": 0.10170751337047143, + "expected_brier": 0.10170751337047146, + "multiclass_ece": 0.22550777522124532, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y6_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.2541995607034706, + "brier": 0.7748798581575931, + "expected_brier": 0.7748798581575931, + "multiclass_ece": 0.42328187610499207, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y7": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 0.7068775946292077, + "brier": 0.513729560059192, + "expected_brier": 0.513729560059192, + "multiclass_ece": 0.506818290938277, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y7_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.3745548048857086, + "brier": 0.1951933622485864, + "expected_brier": 0.1951933622485864, + "multiclass_ece": 0.3124046752599794, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y8": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.23743347587370062, + "brier": 0.08933819159822845, + "expected_brier": 0.08933819159822853, + "multiclass_ece": 0.211350646554758, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y9_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.7625348653836377, + "brier": 0.4276192355204247, + "expected_brier": 0.4276192355204247, + "multiclass_ece": 0.5335175431904987, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x8_y0_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.3176577862045752, + "brier": 0.14812925642172856, + "expected_brier": 0.14812925642172847, + "multiclass_ece": 0.2721481732638753, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x8_y11_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.13251271096702122, + "brier": 0.030805690314806896, + "expected_brier": 0.030805690314806844, + "multiclass_ece": 0.1241081993963471, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x8_y3": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.819870773873334, + "brier": 0.7544731543879906, + "expected_brier": 0.7544731543879906, + "multiclass_ece": 0.8379533097529819, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x9_y0": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.047067929434101, + "brier": 0.8166331924889788, + "expected_brier": 0.8166331924889789, + "multiclass_ece": 0.8708870826799296, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x9_y0_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.8081522164263124, + "brier": 0.8421330146521262, + "expected_brier": 0.8421330146521262, + "multiclass_ece": 0.31811870327002234, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x9_y1_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.389687076811005, + "brier": 0.7253106665015256, + "expected_brier": 0.7253106665015256, + "multiclass_ece": 0.37234990817058344, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x9_y2": { + "count": 2, + "hard_count": 2, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 2.035752768532297, + "brier": 0.8091736421568605, + "expected_brier": 0.8091736421568605, + "multiclass_ece": 0.46920054548941303, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x9_y6_red": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.25698754353937997, + "brier": 0.10271520022851548, + "expected_brier": 0.10271520022851544, + "multiclass_ece": 0.2266221527438519, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x9_y9_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.8424183832077738, + "brier": 0.9239664650741735, + "expected_brier": 0.9239664650741735, + "multiclass_ece": 0.43572312550483433, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "reasoning-control-v1/decision": { + "count": 27, + "hard_count": 27, + "accuracy": 0.2962962962962963, + "expected_accuracy": 0.2962962962962963, + "nll": 1.1737532923407277, + "brier": 0.6662803951570733, + "expected_brier": 0.6662803951570733, + "multiclass_ece": 0.12418752923731705, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 9, + "coverage": 0.3333333333333333, + "accuracy": 0.4444444444444444, + "expected_accuracy": 0.4444444444444444, + "expected_risk": 0.5555555555555556 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "reasoning-control-v1/ordinal": { + "count": 26, + "hard_count": 26, + "accuracy": 0.34615384615384615, + "expected_accuracy": 0.34615384615384615, + "nll": 1.3380940522946123, + "brier": 0.673199698616078, + "expected_brier": 0.673199698616078, + "multiclass_ece": 0.25624036419502555, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 13, + "coverage": 0.5, + "accuracy": 0.5384615384615384, + "expected_accuracy": 0.5384615384615384, + "expected_risk": 0.46153846153846156 + }, + { + "threshold": 0.7, + "selected": 7, + "coverage": 0.2692307692307692, + "accuracy": 0.8571428571428571, + "expected_accuracy": 0.8571428571428571, + "expected_risk": 0.1428571428571429 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 0.15384615384615385, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 0.038461538461538464, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "reasoning-control-v1/verification": { + "count": 27, + "hard_count": 27, + "accuracy": 0.5925925925925926, + "expected_accuracy": 0.5925925925925926, + "nll": 0.7121261200113637, + "brier": 0.5153202487500043, + "expected_brier": 0.5153202487500042, + "multiclass_ece": 0.20679300510042456, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.5925925925925926, + "expected_accuracy": 0.5925925925925926, + "expected_risk": 0.40740740740740744 + }, + { + "threshold": 0.7, + "selected": 14, + "coverage": 0.5185185185185185, + "accuracy": 0.6428571428571429, + "expected_accuracy": 0.6428571428571429, + "expected_risk": 0.3571428571428571 + }, + { + "threshold": 0.8, + "selected": 9, + "coverage": 0.3333333333333333, + "accuracy": 0.7777777777777778, + "expected_accuracy": 0.7777777777777778, + "expected_risk": 0.2222222222222222 + }, + { + "threshold": 0.9, + "selected": 4, + "coverage": 0.14814814814814814, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 0.07407407407407407, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "snake-v1/choice": { + "count": 27, + "hard_count": 21, + "accuracy": 0.5714285714285714, + "expected_accuracy": 0.5555555555555556, + "nll": 1.120662633585259, + "brier": 0.5448146502754462, + "expected_brier": 0.6559257613865571, + "multiclass_ece": 0.18987460905858616, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 25, + "coverage": 0.9259259259259259, + "accuracy": 0.55, + "expected_accuracy": 0.54, + "expected_risk": 0.45999999999999996 + }, + { + "threshold": 0.7, + "selected": 17, + "coverage": 0.6296296296296297, + "accuracy": 0.6428571428571429, + "expected_accuracy": 0.6176470588235294, + "expected_risk": 0.38235294117647056 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 0.1111111111111111, + "accuracy": 0.3333333333333333, + "expected_accuracy": 0.3333333333333333, + "expected_risk": 0.6666666666666667 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "snake-v1/noul": { + "count": 27, + "hard_count": 27, + "accuracy": 0.8518518518518519, + "expected_accuracy": 0.8518518518518519, + "nll": 0.47477602577003875, + "brier": 0.2690983305192824, + "expected_brier": 0.2690983305192824, + "multiclass_ece": 0.09781603901142882, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.8518518518518519, + "expected_accuracy": 0.8518518518518519, + "expected_risk": 0.14814814814814814 + }, + { + "threshold": 0.7, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.8518518518518519, + "expected_accuracy": 0.8518518518518519, + "expected_risk": 0.14814814814814814 + }, + { + "threshold": 0.8, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.8518518518518519, + "expected_accuracy": 0.8518518518518519, + "expected_risk": 0.14814814814814814 + }, + { + "threshold": 0.9, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.8518518518518519, + "expected_accuracy": 0.8518518518518519, + "expected_risk": 0.14814814814814814 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 0.1111111111111111, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "vizdoom-basic-v1/choice": { + "count": 27, + "hard_count": 27, + "accuracy": 0.3333333333333333, + "expected_accuracy": 0.3333333333333333, + "nll": 1.138425673671526, + "brier": 0.6894659744516483, + "expected_brier": 0.6894659744516483, + "multiclass_ece": 0.06290138220247099, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "vizdoom-basic-v1/noul": { + "count": 27, + "hard_count": 27, + "accuracy": 0.7037037037037037, + "expected_accuracy": 0.7037037037037037, + "nll": 0.554708046824275, + "brier": 0.37260007631078024, + "expected_brier": 0.3726000763107803, + "multiclass_ece": 0.18704352634953214, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.7037037037037037, + "expected_accuracy": 0.7037037037037037, + "expected_risk": 0.2962962962962963 + }, + { + "threshold": 0.7, + "selected": 23, + "coverage": 0.8518518518518519, + "accuracy": 0.7391304347826086, + "expected_accuracy": 0.7391304347826086, + "expected_risk": 0.26086956521739135 + }, + { + "threshold": 0.8, + "selected": 5, + "coverage": 0.18518518518518517, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "vizdoom-basic-v1/score": { + "count": 27, + "hard_count": 27, + "accuracy": 0.07407407407407407, + "expected_accuracy": 0.07407407407407407, + "nll": 1.5281374971627815, + "brier": 0.7921445005701216, + "expected_brier": 0.7921445005701216, + "multiclass_ece": 0.281487551110686, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "wikispeedia-v1/choice": { + "count": 27, + "hard_count": 9, + "accuracy": 0.1111111111111111, + "expected_accuracy": 0.0823045267489712, + "nll": 2.577358288876876, + "brier": 0.43560974759709586, + "expected_brier": 0.9437373196135566, + "multiclass_ece": 0.12338939883834045, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/AUTO-CLOSE": { + "count": 4, + "hard_count": 4, + "accuracy": 0.25, + "expected_accuracy": 0.25, + "nll": 0.9665141613921123, + "brier": 0.7590607037261342, + "expected_brier": 0.7590607037261342, + "multiclass_ece": 0.3762896589846308, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.25, + "expected_accuracy": 0.25, + "expected_risk": 0.75 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/COUNT ONLY": { + "count": 6, + "hard_count": 6, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.19425876643409581, + "brier": 0.06342486917374719, + "expected_brier": 0.06342486917374716, + "multiclass_ece": 0.17612736099199902, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 6, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 6, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 5, + "coverage": 0.8333333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/FILE ISSUE ยท ROUTE": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.11740445577141177, + "brier": 0.024670461397467557, + "expected_brier": 0.024670461397467536, + "multiclass_ece": 0.11073327670728095, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/HUMAN REVIEW": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.2335321535050816, + "brier": 0.08675100437782884, + "expected_brier": 0.08675100437782879, + "multiclass_ece": 0.20826786163235655, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/NOT A BUG": { + "count": 6, + "hard_count": 6, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "nll": 0.44574259840532354, + "brier": 0.2753969505276038, + "expected_brier": 0.2753969505276038, + "multiclass_ece": 0.062369810465469744, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 6, + "coverage": 1.0, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.7, + "selected": 6, + "coverage": 1.0, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.8, + "selected": 5, + "coverage": 0.8333333333333334, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/PAGE ON-CALL": { + "count": 5, + "hard_count": 5, + "accuracy": 0.6, + "expected_accuracy": 0.6, + "nll": 0.9354946210644824, + "brier": 0.6441778051019124, + "expected_brier": 0.6441778051019122, + "multiclass_ece": 0.3957710246456412, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 1.0, + "accuracy": 0.6, + "expected_accuracy": 0.6, + "expected_risk": 0.4 + }, + { + "threshold": 0.7, + "selected": 5, + "coverage": 1.0, + "accuracy": 0.6, + "expected_accuracy": 0.6, + "expected_risk": 0.4 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 0.8, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/PRIORITY REVIEW": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.2845166236326731, + "brier": 0.12600413980883884, + "expected_brier": 0.12600413980883887, + "multiclass_ece": 0.24590423696863517, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 0.3333333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/CANCEL": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.3172298160185602, + "brier": 0.1543835365398626, + "expected_brier": 0.1543835365398626, + "multiclass_ece": 0.2677464018550541, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 0.75, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/CLOSE": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.3900052221863084, + "brier": 0.22192838944526172, + "expected_brier": 0.22192838944526172, + "multiclass_ece": 0.31217514589543677, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/FLAG FOR REVIEW": { + "count": 5, + "hard_count": 5, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "nll": 0.5580874513607721, + "brier": 0.36717066700167783, + "expected_brier": 0.3671706670016779, + "multiclass_ece": 0.26059129854700913, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 1.0, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/FREEZE CARD": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.3097425406280575, + "brier": 0.17386714819865515, + "expected_brier": 0.17386714819865512, + "multiclass_ece": 0.24686849629582697, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/HAND OFF": { + "count": 4, + "hard_count": 4, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "nll": 0.5813415251069278, + "brier": 0.3909315911594633, + "expected_brier": 0.3909315911594633, + "multiclass_ece": 0.054865783177815985, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 0.5, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/REFUND": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.5142472478033135, + "brier": 0.3232875464976144, + "expected_brier": 0.3232875464976144, + "multiclass_ece": 0.4020494661715237, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/SAY": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.45084426092195845, + "brier": 0.2643019425542209, + "expected_brier": 0.26430194255422085, + "multiclass_ece": 0.36204822151196225, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/SET INTENT": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.4001114446509877, + "brier": 0.22402613056720638, + "expected_brier": 0.22402613056720644, + "multiclass_ece": 0.3242578502472241, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 0.75, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/DISPUTE LINES": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.30727233242438984, + "brier": 0.14202472069901825, + "expected_brier": 0.14202472069901825, + "multiclass_ece": 0.2635010514743082, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/DUPLICATE": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.3352112273974312, + "brier": 0.18233482334869083, + "expected_brier": 0.18233482334869078, + "multiclass_ece": 0.2734389029709454, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/FRAUD REVIEW": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.18272053481111938, + "brier": 0.05590930196843227, + "expected_brier": 0.055909301968432246, + "multiclass_ece": 0.16694925206727426, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/HOLD": { + "count": 3, + "hard_count": 3, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.5283052314921155, + "brier": 1.2241989146907164, + "expected_brier": 1.2241989146907164, + "multiclass_ece": 0.7821020822000605, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 0.3333333333333333, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/HOLD FOR DOCUMENTS": { + "count": 5, + "hard_count": 5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.4220770707796964, + "brier": 0.23747308146731178, + "expected_brier": 0.23747308146731183, + "multiclass_ece": 0.34400618240295583, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/PAY": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.344746139274568, + "brier": 0.17006092681720864, + "expected_brier": 0.17006092681720864, + "multiclass_ece": 0.29159983437684656, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/REQUEST CORRECTED INVOICE": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.29441081551231973, + "brier": 0.1302784787106249, + "expected_brier": 0.1302784787106249, + "multiclass_ece": 0.2549284826227654, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/ROUTE FOR APPROVAL": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.2775215496469541, + "brier": 0.11745809068909388, + "expected_brier": 0.11745809068909385, + "multiclass_ece": 0.24234076286202233, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/SCHEDULE": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.2922371580043377, + "brier": 0.1309402651511874, + "expected_brier": 0.1309402651511874, + "multiclass_ece": 0.25214022814238124, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/SHORT PAY": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.2216759241332722, + "brier": 0.07926559771381608, + "expected_brier": 0.07926559771381608, + "multiclass_ece": 0.19874056448313387, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/AUTO CLOSE": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.32160939440829356, + "brier": 0.1535261342905165, + "expected_brier": 0.15352613429051654, + "multiclass_ece": 0.2738143778203003, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/BLOCK DESTINATION": { + "count": 2, + "hard_count": 2, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 0.5563552121313453, + "brier": 0.38228123085630616, + "expected_brier": 0.38228123085630616, + "multiclass_ece": 0.40048230975793153, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/BLOCK EGRESS ASSET": { + "count": 4, + "hard_count": 4, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "nll": 0.41135670453333223, + "brier": 0.23766525218916001, + "expected_brier": 0.23766525218916001, + "multiclass_ece": 0.3271142173848516, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 0.75, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/BLOCK SENDER": { + "count": 3, + "hard_count": 3, + "accuracy": 0.3333333333333333, + "expected_accuracy": 0.3333333333333333, + "nll": 0.6586219966591308, + "brier": 0.46937741141764544, + "expected_brier": 0.46937741141764544, + "multiclass_ece": 0.4722460943355821, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 0.3333333333333333, + "expected_accuracy": 0.3333333333333333, + "expected_risk": 0.6666666666666667 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/ESCALATE TIER2": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.32456440494186767, + "brier": 0.15363294592161594, + "expected_brier": 0.15363294592161592, + "multiclass_ece": 0.27715784845608826, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/ESCALATE URGENT": { + "count": 2, + "hard_count": 2, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 0.7801197632192878, + "brier": 0.5855072506905011, + "expected_brier": 0.585507250690501, + "multiclass_ece": 0.5376787037253306, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/ISOLATE HOST": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 0.7939101475052832, + "brier": 0.6004465478143917, + "expected_brier": 0.6004465478143917, + "multiclass_ece": 0.5479263398552727, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/NOTIFY USER": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.2680809000429661, + "brier": 0.1105948991105393, + "expected_brier": 0.11059489911053921, + "multiclass_ece": 0.2351540974664691, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/PURGE MAILBOXES": { + "count": 2, + "hard_count": 2, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 0.6831678094412634, + "brier": 0.49606255385967796, + "expected_brier": 0.496062553859678, + "multiclass_ece": 0.4608139734891588, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/QUARANTINE FILE": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.5045364983931437, + "brier": 0.31397207228422186, + "expected_brier": 0.31397207228422186, + "multiclass_ece": 0.3962146339323056, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/QUARANTINE MESSAGE": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.31799841809388796, + "brier": 0.14851180486780757, + "expected_brier": 0.14851180486780757, + "multiclass_ece": 0.272334241610815, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/REQUIRE REAUTH": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.2631453729452586, + "brier": 0.10706401390445662, + "expected_brier": 0.10706401390445652, + "multiclass_ece": 0.23136984883996514, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/REVOKE ACCESS KEY": { + "count": 2, + "hard_count": 2, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 0.5265534590581739, + "brier": 0.3421004473188073, + "expected_brier": 0.3421004473188073, + "multiclass_ece": 0.4002442180760147, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/REVOKE SESSIONS": { + "count": 2, + "hard_count": 2, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 0.9685367473250548, + "brier": 0.768374885069224, + "expected_brier": 0.768374885069224, + "multiclass_ece": 0.6189801636037295, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + } + } + }, + "baseline_calibrated_test": { + "count": 512, + "hard_count": 470, + "accuracy": 0.5468085106382978, + "expected_accuracy": 0.5218850756448413, + "nll": 1.0245739301131427, + "brier": 0.5111158131851092, + "expected_brier": 0.5624078208736013, + "multiclass_ece": 0.06838996065159306, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 283, + "coverage": 0.552734375, + "accuracy": 0.6970802919708029, + "expected_accuracy": 0.690223792697291, + "expected_risk": 0.30977620730270905 + }, + { + "threshold": 0.7, + "selected": 106, + "coverage": 0.20703125, + "accuracy": 0.8269230769230769, + "expected_accuracy": 0.8207547169811321, + "expected_risk": 0.17924528301886788 + }, + { + "threshold": 0.8, + "selected": 52, + "coverage": 0.1015625, + "accuracy": 0.7692307692307693, + "expected_accuracy": 0.7692307692307693, + "expected_risk": 0.23076923076923073 + }, + { + "threshold": 0.9, + "selected": 10, + "coverage": 0.01953125, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 0.001953125, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ], + "mean_latency_seconds": 0.054588349696132354, + "by_kind": { + "choice": { + "count": 197, + "hard_count": 158, + "accuracy": 0.45569620253164556, + "expected_accuracy": 0.40967085649826773, + "nll": 1.381800129848969, + "brier": 0.5503896152677165, + "expected_brier": 0.6760825489555736, + "multiclass_ece": 0.13369573523261513, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 44, + "coverage": 0.2233502538071066, + "accuracy": 0.8157894736842105, + "expected_accuracy": 0.768939393939394, + "expected_risk": 0.23106060606060597 + }, + { + "threshold": 0.7, + "selected": 10, + "coverage": 0.050761421319796954, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 0.005076142131979695, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "noul": { + "count": 219, + "hard_count": 219, + "accuracy": 0.7077625570776256, + "expected_accuracy": 0.7077625570776256, + "nll": 0.6052613380976377, + "brier": 0.404047028276472, + "expected_brier": 0.40404702827647193, + "multiclass_ece": 0.07623049510941937, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 219, + "coverage": 1.0, + "accuracy": 0.7077625570776256, + "expected_accuracy": 0.7077625570776256, + "expected_risk": 0.2922374429223744 + }, + { + "threshold": 0.7, + "selected": 92, + "coverage": 0.4200913242009132, + "accuracy": 0.8043478260869565, + "expected_accuracy": 0.8043478260869565, + "expected_risk": 0.19565217391304346 + }, + { + "threshold": 0.8, + "selected": 51, + "coverage": 0.2328767123287671, + "accuracy": 0.7647058823529411, + "expected_accuracy": 0.7647058823529411, + "expected_risk": 0.23529411764705888 + }, + { + "threshold": 0.9, + "selected": 10, + "coverage": 0.045662100456621, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 0.0045662100456621, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "score": { + "count": 96, + "hard_count": 93, + "accuracy": 0.3225806451612903, + "expected_accuracy": 0.328125, + "nll": 1.2480728499406195, + "brier": 0.6747733640675909, + "expected_brier": 0.6903983640675909, + "multiclass_ece": 0.12916662012627755, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 20, + "coverage": 0.20833333333333334, + "accuracy": 0.29411764705882354, + "expected_accuracy": 0.325, + "expected_risk": 0.675 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 0.041666666666666664, + "accuracy": 1.0, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ], + "ordinal_mae": 0.7985307859859793 + } + }, + "by_source": { + "customer-control-v1": { + "count": 72, + "hard_count": 68, + "accuracy": 0.6323529411764706, + "expected_accuracy": 0.625, + "nll": 0.658852976711753, + "brier": 0.362660855791549, + "expected_brier": 0.39043863356932673, + "multiclass_ece": 0.21377450908805823, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 57, + "coverage": 0.7916666666666666, + "accuracy": 0.7592592592592593, + "expected_accuracy": 0.7456140350877193, + "expected_risk": 0.2543859649122807 + }, + { + "threshold": 0.7, + "selected": 27, + "coverage": 0.375, + "accuracy": 1.0, + "expected_accuracy": 0.9629629629629629, + "expected_risk": 0.03703703703703709 + }, + { + "threshold": 0.8, + "selected": 10, + "coverage": 0.1388888888888889, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1": { + "count": 74, + "hard_count": 74, + "accuracy": 0.4864864864864865, + "expected_accuracy": 0.4864864864864865, + "nll": 1.314144908270539, + "brier": 0.6921768756252061, + "expected_brier": 0.6921768756252061, + "multiclass_ece": 0.2841745953095707, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 25, + "coverage": 0.33783783783783783, + "accuracy": 0.44, + "expected_accuracy": 0.44, + "expected_risk": 0.56 + }, + { + "threshold": 0.7, + "selected": 14, + "coverage": 0.1891891891891892, + "accuracy": 0.5714285714285714, + "expected_accuracy": 0.5714285714285714, + "expected_risk": 0.4285714285714286 + }, + { + "threshold": 0.8, + "selected": 12, + "coverage": 0.16216216216216217, + "accuracy": 0.5833333333333334, + "expected_accuracy": 0.5833333333333334, + "expected_risk": 0.41666666666666663 + }, + { + "threshold": 0.9, + "selected": 9, + "coverage": 0.12162162162162163, + "accuracy": 0.4444444444444444, + "expected_accuracy": 0.4444444444444444, + "expected_risk": 0.5555555555555556 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 0.013513513513513514, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "reasoning-control-v1": { + "count": 72, + "hard_count": 72, + "accuracy": 0.5416666666666666, + "expected_accuracy": 0.5416666666666666, + "nll": 0.9983200131352099, + "brier": 0.5732840547355509, + "expected_brier": 0.5732840547355508, + "multiclass_ece": 0.12056931797881269, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 32, + "coverage": 0.4444444444444444, + "accuracy": 0.625, + "expected_accuracy": 0.625, + "expected_risk": 0.375 + }, + { + "threshold": 0.7, + "selected": 5, + "coverage": 0.06944444444444445, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 0.041666666666666664, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 0.013888888888888888, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "snake-v1": { + "count": 48, + "hard_count": 43, + "accuracy": 0.627906976744186, + "expected_accuracy": 0.6076388888888888, + "nll": 0.8406183306687932, + "brier": 0.466804862109273, + "expected_brier": 0.5258326398870504, + "multiclass_ece": 0.13104156775437445, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 42, + "coverage": 0.875, + "accuracy": 0.6666666666666666, + "expected_accuracy": 0.6507936507936508, + "expected_risk": 0.3492063492063492 + }, + { + "threshold": 0.7, + "selected": 24, + "coverage": 0.5, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.8, + "selected": 24, + "coverage": 0.5, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "tic_tac_toe-v1": { + "count": 24, + "hard_count": 7, + "accuracy": 0.5714285714285714, + "expected_accuracy": 0.2767857142857143, + "nll": 1.2872894456843487, + "brier": 0.2894306760882413, + "expected_brier": 0.7070894062469716, + "multiclass_ece": 0.1330774066417672, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 0.20833333333333334, + "accuracy": 1.0, + "expected_accuracy": 0.7, + "expected_risk": 0.30000000000000004 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "tile_platformer-v1": { + "count": 24, + "hard_count": 24, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.9381376592505102, + "brier": 0.8743126659154127, + "expected_brier": 0.8743126659154128, + "multiclass_ece": 0.23051377658095615, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "trex_runner-v1": { + "count": 4, + "hard_count": 4, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 1.1614418705733276, + "brier": 0.6234538512891123, + "expected_brier": 0.6234538512891123, + "multiclass_ece": 0.24162643478796952, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "vizdoom-basic-v1": { + "count": 73, + "hard_count": 73, + "accuracy": 0.3972602739726027, + "expected_accuracy": 0.3972602739726027, + "nll": 1.0762737754735434, + "brier": 0.6221355906345098, + "expected_brier": 0.6221355906345098, + "multiclass_ece": 0.16065967844803627, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 25, + "coverage": 0.3424657534246575, + "accuracy": 0.64, + "expected_accuracy": 0.64, + "expected_risk": 0.36 + }, + { + "threshold": 0.7, + "selected": 5, + "coverage": 0.0684931506849315, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "wikispeedia-v1": { + "count": 24, + "hard_count": 8, + "accuracy": 0.25, + "expected_accuracy": 0.22481812169312174, + "nll": 2.2382400494374015, + "brier": 0.3973246727101471, + "expected_brier": 0.8725065510170252, + "multiclass_ece": 0.08341916958368423, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ], + "optimal_action_hit": 0.625, + "optimal_set_probability_mass": 0.3996101432926548 + }, + "workflow-controls-v1/agent_trace_observability": { + "count": 24, + "hard_count": 24, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "nll": 0.4750620022052388, + "brier": 0.2964427461665344, + "expected_brier": 0.2964427461665345, + "multiclass_ece": 0.13559377741417267, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.7, + "selected": 17, + "coverage": 0.7083333333333334, + "accuracy": 0.8235294117647058, + "expected_accuracy": 0.8235294117647058, + "expected_risk": 0.17647058823529416 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 0.125, + "accuracy": 0.6666666666666666, + "expected_accuracy": 0.6666666666666666, + "expected_risk": 0.33333333333333337 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service": { + "count": 24, + "hard_count": 24, + "accuracy": 0.9166666666666666, + "expected_accuracy": 0.9166666666666666, + "nll": 0.5177637080060894, + "brier": 0.33337297832454493, + "expected_brier": 0.333372978324545, + "multiclass_ece": 0.29215943761024604, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.9166666666666666, + "expected_accuracy": 0.9166666666666666, + "expected_risk": 0.08333333333333337 + }, + { + "threshold": 0.7, + "selected": 5, + "coverage": 0.20833333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing": { + "count": 24, + "hard_count": 24, + "accuracy": 0.7916666666666666, + "expected_accuracy": 0.7916666666666666, + "nll": 0.5627710464001133, + "brier": 0.3742967795732895, + "expected_brier": 0.3742967795732895, + "multiclass_ece": 0.14008211609107618, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.7916666666666666, + "expected_accuracy": 0.7916666666666666, + "expected_risk": 0.20833333333333337 + }, + { + "threshold": 0.7, + "selected": 9, + "coverage": 0.375, + "accuracy": 0.7777777777777778, + "expected_accuracy": 0.7777777777777778, + "expected_risk": 0.2222222222222222 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents": { + "count": 25, + "hard_count": 25, + "accuracy": 0.56, + "expected_accuracy": 0.56, + "nll": 0.639716276374202, + "brier": 0.4480597857265136, + "expected_brier": 0.4480597857265136, + "multiclass_ece": 0.15943304073250253, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 25, + "coverage": 1.0, + "accuracy": 0.56, + "expected_accuracy": 0.56, + "expected_risk": 0.43999999999999995 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + } + }, + "by_question": { + "customer-control-v1/bug_severity": { + "count": 8, + "hard_count": 8, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.3990500583413161, + "brier": 0.8892123334633173, + "expected_brier": 0.8892123334633173, + "multiclass_ece": 0.5209454051642233, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 6, + "coverage": 0.75, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "customer-control-v1/category": { + "count": 24, + "hard_count": 23, + "accuracy": 0.782608695652174, + "expected_accuracy": 0.7708333333333334, + "nll": 0.5867529960962868, + "brier": 0.2667359219372509, + "expected_brier": 0.2875692552705842, + "multiclass_ece": 0.28724189662596306, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 17, + "coverage": 0.7083333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 10, + "coverage": 0.4166666666666667, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 0.041666666666666664, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "customer-control-v1/churn_likelihood_level": { + "count": 7, + "hard_count": 7, + "accuracy": 0.14285714285714285, + "expected_accuracy": 0.14285714285714285, + "nll": 0.8616005345018606, + "brier": 0.5837747466535647, + "expected_brier": 0.5837747466535647, + "multiclass_ece": 0.3403702517457771, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 0.2857142857142857, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "customer-control-v1/frustration": { + "count": 9, + "hard_count": 6, + "accuracy": 0.16666666666666666, + "expected_accuracy": 0.2777777777777778, + "nll": 0.8759754417576202, + "brier": 0.44492602732499464, + "expected_brier": 0.6115926939916613, + "multiclass_ece": 0.347831331658811, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 8, + "coverage": 0.8888888888888888, + "accuracy": 0.2, + "expected_accuracy": 0.3125, + "expected_risk": 0.6875 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 0.3333333333333333, + "accuracy": 1.0, + "expected_accuracy": 0.6666666666666666, + "expected_risk": 0.33333333333333337 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "customer-control-v1/has_reproducible_steps": { + "count": 14, + "hard_count": 14, + "accuracy": 0.9285714285714286, + "expected_accuracy": 0.9285714285714286, + "nll": 0.46482771686929486, + "brier": 0.2835977458892437, + "expected_brier": 0.2835977458892437, + "multiclass_ece": 0.2880709668251722, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 14, + "coverage": 1.0, + "accuracy": 0.9285714285714286, + "expected_accuracy": 0.9285714285714286, + "expected_risk": 0.0714285714285714 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 0.2857142857142857, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 0.07142857142857142, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "customer-control-v1/refund_requested": { + "count": 10, + "hard_count": 10, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.17403711967030772, + "brier": 0.053509490784164504, + "expected_brier": 0.05350949078416449, + "multiclass_ece": 0.15879932002569425, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 10, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 10, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 8, + "coverage": 0.8, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y0_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.7956013688751105, + "brier": 0.38925257281055337, + "expected_brier": 0.38925257281055337, + "multiclass_ece": 0.5486902503370998, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y1_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.1614701982832114, + "brier": 0.7085507417396752, + "expected_brier": 0.7085507417396752, + "multiclass_ece": 0.36151590637906533, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y2": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.115470491000544, + "brier": 0.8515850432496703, + "expected_brier": 0.8515850432496703, + "multiclass_ece": 0.19737586893702316, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y2_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.0466427201080357, + "brier": 0.5325727056594247, + "expected_brier": 0.5325727056594247, + "multiclass_ece": 0.648885437579839, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y2_red": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.0783092556484475, + "brier": 0.8707507813445119, + "expected_brier": 0.8707507813445119, + "multiclass_ece": 0.6598298194779135, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y3_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.529154435462304, + "brier": 1.2270587523857592, + "expected_brier": 1.2270587523857595, + "multiclass_ece": 0.783281160371472, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y3_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.9871185111210599, + "brier": 0.4987544205034243, + "expected_brier": 0.4987544205034242, + "multiclass_ece": 0.6273510707957997, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y5": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.21871461660716396, + "brier": 0.07718441484323443, + "expected_brier": 0.07718441484323435, + "multiclass_ece": 0.19644899445305697, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y5_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.265275708832231, + "brier": 0.7821015246375038, + "expected_brier": 0.7821015246375038, + "multiclass_ece": 0.42660727830255235, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y6_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.1744612009520488, + "brier": 0.7173284101941813, + "expected_brier": 0.7173284101941813, + "multiclass_ece": 0.3687094150544353, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y7_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.17036139098533667, + "brier": 0.049072192468568435, + "expected_brier": 0.04907219246856842, + "multiclass_ece": 0.1566400211768506, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y7_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.0442814447312792, + "brier": 0.5317736316840989, + "expected_brier": 0.5317736316840989, + "multiclass_ece": 0.648055379797348, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y0": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.7702833593633922, + "brier": 0.7623898197682071, + "expected_brier": 0.7623898197682071, + "multiclass_ece": 0.19568132782784387, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y0_lightness": { + "count": 2, + "hard_count": 2, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 1.7282071956370912, + "brier": 0.8054600779632302, + "expected_brier": 0.8054600779632302, + "multiclass_ece": 0.13428676227015646, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y1_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.25546306458628143, + "brier": 0.10164842100095664, + "expected_brier": 0.10164842100095672, + "multiclass_ece": 0.22544225535706108, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y2": { + "count": 3, + "hard_count": 3, + "accuracy": 0.3333333333333333, + "expected_accuracy": 0.3333333333333333, + "nll": 1.5658647133331876, + "brier": 0.7092892683647718, + "expected_brier": 0.7092892683647718, + "multiclass_ece": 0.3035094249716184, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 0.3333333333333333, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y4_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.8508576820024663, + "brier": 0.8524448783333098, + "expected_brier": 0.8524448783333098, + "multiclass_ece": 0.2698450129598058, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y4_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.046001053287608, + "brier": 0.6450384503175229, + "expected_brier": 0.6450384503175228, + "multiclass_ece": 0.4076925761730064, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y5_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.9742319822059083, + "brier": 0.49931921766351867, + "expected_brier": 0.4993192176635186, + "multiclass_ece": 0.6225178447253207, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y6_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.2220824207714531, + "brier": 0.5942051381048891, + "expected_brier": 0.594205138104889, + "multiclass_ece": 0.7053839868227498, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y6_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.3170716141497705, + "brier": 0.8039604072550668, + "expected_brier": 0.8039604072550668, + "multiclass_ece": 0.3708320182286857, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y0": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 0.7389707844314569, + "brier": 0.5457926094645073, + "expected_brier": 0.5457926094645074, + "multiclass_ece": 0.5223947786226941, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y0_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.7091604784511256, + "brier": 0.9261042151622751, + "expected_brier": 0.9261042151622751, + "multiclass_ece": 0.4854029105831574, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y1": { + "count": 2, + "hard_count": 2, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 1.345097552190964, + "brier": 0.47808633135081, + "expected_brier": 0.4780863313508099, + "multiclass_ece": 0.14272921373447922, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y1_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.4865915278932302, + "brier": 1.0964746461083343, + "expected_brier": 1.0964746461083346, + "multiclass_ece": 0.3968988685442534, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y3_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.4401376305563496, + "brier": 0.6880497574711031, + "expected_brier": 0.6880497574711031, + "multiclass_ece": 0.2500850514373916, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y3_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.9693257223061242, + "brier": 0.5813067643131207, + "expected_brier": 0.5813067643131207, + "multiclass_ece": 0.6206612684653805, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y4_red": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.9707357153855307, + "brier": 1.800194000985308, + "expected_brier": 1.8001940009853081, + "multiclass_ece": 0.9487344204215709, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y5": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.6402028316725553, + "brier": 0.4471071344336208, + "expected_brier": 0.4471071344336208, + "multiclass_ece": 0.47281451671539276, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y5_green": { + "count": 2, + "hard_count": 2, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.063957897129792, + "brier": 0.8575751004000859, + "expected_brier": 0.8575751004000859, + "multiclass_ece": 0.6547139032397429, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y6": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 0.7099682395948106, + "brier": 0.5168194923984318, + "expected_brier": 0.5168194923984317, + "multiclass_ece": 0.5083401874721454, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y6_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.852111048198845, + "brier": 0.8234063512661343, + "expected_brier": 0.8234063512661343, + "multiclass_ece": 0.24245406300972377, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x3_y2_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 0.997697878650375, + "brier": 0.7970103933509209, + "expected_brier": 0.7970103933509209, + "multiclass_ece": 0.631272680127582, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x3_y4": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.3355782438880204, + "brier": 0.8728591710050431, + "expected_brier": 0.8728591710050431, + "multiclass_ece": 0.9032454813614096, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x3_y5_red": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.6661131030872935, + "brier": 1.731578894701808, + "expected_brier": 1.731578894701808, + "multiclass_ece": 0.9304780746212691, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x3_y6_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.8486154015442817, + "brier": 0.8280672376743511, + "expected_brier": 0.8280672376743512, + "multiclass_ece": 0.26265166806264983, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x3_y7_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.2822764049176698, + "brier": 0.5951436251276365, + "expected_brier": 0.5951436251276365, + "multiclass_ece": 0.7225949051738455, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y0_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.45741758597154253, + "brier": 0.2695013436821326, + "expected_brier": 0.2695013436821325, + "multiclass_ece": 0.36708401196601614, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y0_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.00288360456575, + "brier": 0.6044182132383003, + "expected_brier": 0.6044182132383003, + "multiclass_ece": 0.6331798496428354, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y2": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.1667546086329486, + "brier": 0.5210072950702792, + "expected_brier": 0.5210072950702793, + "multiclass_ece": 0.6886241602930917, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y4_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.9099827901473686, + "brier": 0.4593391376885432, + "expected_brier": 0.4593391376885432, + "multiclass_ece": 0.5974688485241779, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y5_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.7187441792521856, + "brier": 0.7822273281611947, + "expected_brier": 0.7822273281611947, + "multiclass_ece": 0.2448269724138885, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y6_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.8706862277084685, + "brier": 0.4394526136931071, + "expected_brier": 0.4394526136931072, + "multiclass_ece": 0.5813358482926047, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y7_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.1011795609447101, + "brier": 0.669244232171483, + "expected_brier": 0.6692442321714829, + "multiclass_ece": 0.3545831279699743, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y2": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.3364729070660808, + "brier": 0.5924681566705371, + "expected_brier": 0.5924681566705372, + "multiclass_ece": 0.737229146765256, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y2_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.0794724590266265, + "brier": 0.8717948315592761, + "expected_brier": 0.8717948315592761, + "multiclass_ece": 0.6602252765379694, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y3_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.4188888273230476, + "brier": 0.23423272805691253, + "expected_brier": 0.23423272805691253, + "multiclass_ece": 0.3422226819315989, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y5_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.236487950559273, + "brier": 0.7552999947403943, + "expected_brier": 0.7552999947403944, + "multiclass_ece": 0.35663301982321965, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y6_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.08831578962103338, + "brier": 0.01429005530321728, + "expected_brier": 0.014290055303217253, + "multiclass_ece": 0.08452826540044833, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y6_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.9515530752516103, + "brier": 0.48386853174814054, + "expected_brier": 0.4838685317481406, + "multiclass_ece": 0.613859148281237, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y7_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.1008695723463564, + "brier": 0.5409653073767899, + "expected_brier": 0.5409653073767899, + "multiclass_ece": 0.6674182459764324, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y7_red": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.9279915849290252, + "brier": 1.7917078820422145, + "expected_brier": 1.7917078820422145, + "multiclass_ece": 0.9464956106718654, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y7_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.1682297299482294, + "brier": 0.7238202846485902, + "expected_brier": 0.7238202846485902, + "multiclass_ece": 0.42059065333845075, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y1_red": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.08048111023514534, + "brier": 0.011959136327381473, + "expected_brier": 0.011959136327381437, + "multiclass_ece": 0.07732766751745934, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y2_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.13682605588273508, + "brier": 0.03270561005700114, + "expected_brier": 0.03270561005700112, + "multiclass_ece": 0.12787808658445188, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y2_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.1948399470075097, + "brier": 0.6257017136618227, + "expected_brier": 0.6257017136618228, + "multiclass_ece": 0.6972475932667672, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y2_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.8421116797574125, + "brier": 0.4215721731015268, + "expected_brier": 0.4215721731015267, + "multiclass_ece": 0.569200149081549, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y4": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.1564370420152223, + "brier": 0.8394294160666869, + "expected_brier": 0.8394294160666869, + "multiclass_ece": 0.8842579113931137, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y5": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.137622477330089, + "brier": 0.83507657280459, + "expected_brier": 0.8350765728045899, + "multiclass_ece": 0.8820650971293144, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y6_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.5839950972278654, + "brier": 0.7536707816540109, + "expected_brier": 0.7536707816540109, + "multiclass_ece": 0.29106506878910826, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y6_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.106902580981901, + "brier": 0.6744107054519748, + "expected_brier": 0.6744107054519748, + "multiclass_ece": 0.3680909442766138, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y0_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.8447571694157553, + "brier": 0.4955030763400158, + "expected_brier": 0.4955030763400159, + "multiclass_ece": 0.5703383194591864, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y1_green": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.6635382636451035, + "brier": 1.730911849157083, + "expected_brier": 1.7309118491570832, + "multiclass_ece": 0.9302988361696157, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y2_green": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.5525735615681886, + "brier": 1.700606987441343, + "expected_brier": 1.700606987441343, + "multiclass_ece": 0.9221190236193327, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y5_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.7041584298545158, + "brier": 0.7930793942711486, + "expected_brier": 0.7930793942711487, + "multiclass_ece": 0.2868314715539248, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y5_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.2233918991967423, + "brier": 0.6264225000681076, + "expected_brier": 0.6264225000681076, + "multiclass_ece": 0.7057695276519595, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y7": { + "count": 2, + "hard_count": 2, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 1.3421317215628286, + "brier": 0.4659774429977618, + "expected_brier": 0.4659774429977618, + "multiclass_ece": 0.08309829115344985, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "reasoning-control-v1/decision": { + "count": 24, + "hard_count": 24, + "accuracy": 0.6666666666666666, + "expected_accuracy": 0.6666666666666666, + "nll": 1.0945755081597421, + "brier": 0.5982135957301157, + "expected_brier": 0.5982135957301157, + "multiclass_ece": 0.2697257480612633, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 0.16666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "reasoning-control-v1/ordinal": { + "count": 24, + "hard_count": 24, + "accuracy": 0.4583333333333333, + "expected_accuracy": 0.4583333333333333, + "nll": 1.2520552471795254, + "brier": 0.654437632217135, + "expected_brier": 0.6544376322171351, + "multiclass_ece": 0.1813734379938634, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 0.16666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.041666666666666664, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "reasoning-control-v1/verification": { + "count": 24, + "hard_count": 24, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 0.6483292840663603, + "brier": 0.46720093625940157, + "expected_brier": 0.4672009362594017, + "multiclass_ece": 0.15421559297032528, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 0.16666666666666666, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 0.125, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 0.041666666666666664, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "snake-v1/choice": { + "count": 24, + "hard_count": 19, + "accuracy": 0.47368421052631576, + "expected_accuracy": 0.46527777777777773, + "nll": 1.0740261200753192, + "brier": 0.5333024117431959, + "expected_brier": 0.6513579672987514, + "multiclass_ece": 0.14946618246852758, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 18, + "coverage": 0.75, + "accuracy": 0.5333333333333333, + "expected_accuracy": 0.5185185185185186, + "expected_risk": 0.4814814814814814 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "snake-v1/noul": { + "count": 24, + "hard_count": 24, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "nll": 0.6072105412622669, + "brier": 0.40030731247534984, + "expected_brier": 0.40030731247534984, + "multiclass_ece": 0.11261695304022135, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.7, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.8, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "tic_tac_toe-v1/choice": { + "count": 24, + "hard_count": 7, + "accuracy": 0.5714285714285714, + "expected_accuracy": 0.2767857142857143, + "nll": 1.2872894456843487, + "brier": 0.2894306760882413, + "expected_brier": 0.7070894062469716, + "multiclass_ece": 0.1330774066417672, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 0.20833333333333334, + "accuracy": 1.0, + "expected_accuracy": 0.7, + "expected_risk": 0.30000000000000004 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "tile_platformer-v1/choice": { + "count": 24, + "hard_count": 24, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.9381376592505102, + "brier": 0.8743126659154127, + "expected_brier": 0.8743126659154128, + "multiclass_ece": 0.23051377658095615, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "trex_runner-v1/choice": { + "count": 4, + "hard_count": 4, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 1.1614418705733276, + "brier": 0.6234538512891123, + "expected_brier": 0.6234538512891123, + "multiclass_ece": 0.24162643478796952, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "vizdoom-basic-v1/choice": { + "count": 24, + "hard_count": 24, + "accuracy": 0.375, + "expected_accuracy": 0.375, + "nll": 1.11140628133159, + "brier": 0.675055224064738, + "expected_brier": 0.675055224064738, + "multiclass_ece": 0.06091378831146649, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "vizdoom-basic-v1/noul": { + "count": 25, + "hard_count": 25, + "accuracy": 0.64, + "expected_accuracy": 0.64, + "nll": 0.6169201089321479, + "brier": 0.42834966562347965, + "expected_brier": 0.42834966562347965, + "multiclass_ece": 0.2884169155318467, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 25, + "coverage": 1.0, + "accuracy": 0.64, + "expected_accuracy": 0.64, + "expected_risk": 0.36 + }, + { + "threshold": 0.7, + "selected": 5, + "coverage": 0.2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "vizdoom-basic-v1/score": { + "count": 24, + "hard_count": 24, + "accuracy": 0.16666666666666666, + "expected_accuracy": 0.16666666666666666, + "nll": 1.519634672262784, + "brier": 0.7710762957574376, + "expected_brier": 0.7710762957574376, + "multiclass_ece": 0.12732511328897025, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "wikispeedia-v1/choice": { + "count": 24, + "hard_count": 8, + "accuracy": 0.25, + "expected_accuracy": 0.22481812169312174, + "nll": 2.2382400494374015, + "brier": 0.3973246727101471, + "expected_brier": 0.8725065510170252, + "multiclass_ece": 0.08341916958368423, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/AUTO-CLOSE": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.2906676115575563, + "brier": 0.13109945718314467, + "expected_brier": 0.1310994571831446, + "multiclass_ece": 0.2502095687845314, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/COUNT ONLY": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.32243615792577746, + "brier": 0.15205141569347974, + "expected_brier": 0.1520514156934797, + "multiclass_ece": 0.27555072488785776, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/FILE ISSUE ยท ROUTE": { + "count": 5, + "hard_count": 5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.2222827647885242, + "brier": 0.07953391288778103, + "expected_brier": 0.07953391288778104, + "multiclass_ece": 0.19927608237971403, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 0.4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/HUMAN REVIEW": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.4092310699904398, + "brier": 0.22695071019034851, + "expected_brier": 0.22695071019034851, + "multiclass_ece": 0.33483205671423366, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.25, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/NOT A BUG": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.27475383290883176, + "brier": 0.11543135325345955, + "expected_brier": 0.11543135325345943, + "multiclass_ece": 0.2402408720986705, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/PAGE ON-CALL": { + "count": 5, + "hard_count": 5, + "accuracy": 0.4, + "expected_accuracy": 0.4, + "nll": 0.9467806680957717, + "brier": 0.7207216016716215, + "expected_brier": 0.7207216016716215, + "multiclass_ece": 0.43327217027369935, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 1.0, + "accuracy": 0.4, + "expected_accuracy": 0.4, + "expected_risk": 0.6 + }, + { + "threshold": 0.7, + "selected": 5, + "coverage": 1.0, + "accuracy": 0.4, + "expected_accuracy": 0.4, + "expected_risk": 0.6 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 0.2, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/PRIORITY REVIEW": { + "count": 3, + "hard_count": 3, + "accuracy": 0.6666666666666666, + "expected_accuracy": 0.6666666666666666, + "nll": 0.6017271557278869, + "brier": 0.4135538408516952, + "expected_brier": 0.4135538408516952, + "multiclass_ece": 0.42681256300931175, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 0.6666666666666666, + "expected_accuracy": 0.6666666666666666, + "expected_risk": 0.33333333333333337 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/CANCEL": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.35868435086047956, + "brier": 0.18185870852618852, + "expected_brier": 0.18185870852618846, + "multiclass_ece": 0.3012991102860978, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/CLOSE": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.5027722034208971, + "brier": 0.31323663453657913, + "expected_brier": 0.3132366345365792, + "multiclass_ece": 0.394100699107596, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/FLAG FOR REVIEW": { + "count": 3, + "hard_count": 3, + "accuracy": 0.3333333333333333, + "expected_accuracy": 0.3333333333333333, + "nll": 0.74355054379444, + "brier": 0.5545054504918373, + "expected_brier": 0.5545054504918373, + "multiclass_ece": 0.5021897300829615, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 0.3333333333333333, + "expected_accuracy": 0.3333333333333333, + "expected_risk": 0.6666666666666667 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.3333333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/FREEZE CARD": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.2469855950031716, + "brier": 0.0959176399289778, + "expected_brier": 0.09591763992897784, + "multiclass_ece": 0.2187904604576317, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/HAND OFF": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.5268719470998514, + "brier": 0.335463977949629, + "expected_brier": 0.3354639779496291, + "multiclass_ece": 0.40955096016834647, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/REFUND": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.580642112943015, + "brier": 0.38803633351609584, + "expected_brier": 0.38803633351609584, + "multiclass_ece": 0.4404102737705619, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/SAY": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.49340177912897965, + "brier": 0.30350528088813217, + "expected_brier": 0.3035052808881322, + "multiclass_ece": 0.3892716494173303, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/SET INTENT": { + "count": 5, + "hard_count": 5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.5973259722649692, + "brier": 0.4065321571628148, + "expected_brier": 0.4065321571628148, + "multiclass_ece": 0.4465219755173801, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/DISPUTE LINES": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.42664790148883913, + "brier": 0.24124385421881928, + "expected_brier": 0.24124385421881933, + "multiclass_ece": 0.34730667587797626, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/DUPLICATE": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.2737602317716062, + "brier": 0.11470670464704737, + "expected_brier": 0.11470670464704735, + "multiclass_ece": 0.23948559940740421, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/FRAUD REVIEW": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.29718793334801796, + "brier": 0.132511029835357, + "expected_brier": 0.132511029835357, + "multiclass_ece": 0.2569371319748768, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/HOLD": { + "count": 4, + "hard_count": 4, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.2447179266120287, + "brier": 1.01259850219788, + "expected_brier": 1.01259850219788, + "multiclass_ece": 0.7112529268788641, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 0.5, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/HOLD FOR DOCUMENTS": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.5018572403407264, + "brier": 0.31141006198910365, + "expected_brier": 0.3114100619891036, + "multiclass_ece": 0.39459476807802685, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/PAY": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.4240731766098745, + "brier": 0.24211318741371102, + "expected_brier": 0.24211318741371102, + "multiclass_ece": 0.34325165489726156, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.3333333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/REQUEST CORRECTED INVOICE": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.3783712133794621, + "brier": 0.1997744766679166, + "expected_brier": 0.1997744766679166, + "multiclass_ece": 0.3141511271674047, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/ROUTE FOR APPROVAL": { + "count": 4, + "hard_count": 4, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "nll": 0.6419774736354035, + "brier": 0.447886974520459, + "expected_brier": 0.44788697452045906, + "multiclass_ece": 0.1361682660824679, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/SCHEDULE": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.36232598892207374, + "brier": 0.18478055462514656, + "expected_brier": 0.18478055462514653, + "multiclass_ece": 0.3039344098490069, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/SHORT PAY": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.4075462355935269, + "brier": 0.2241632189309676, + "expected_brier": 0.22416321893096758, + "multiclass_ece": 0.33465187129086466, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/BLOCK EGRESS ASSET": { + "count": 3, + "hard_count": 3, + "accuracy": 0.3333333333333333, + "expected_accuracy": 0.3333333333333333, + "nll": 0.6071687329206544, + "brier": 0.41777343468929584, + "expected_brier": 0.41777343468929584, + "multiclass_ece": 0.44866720753092776, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 0.3333333333333333, + "expected_accuracy": 0.3333333333333333, + "expected_risk": 0.6666666666666667 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/BLOCK SENDER": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 0.7325554966418191, + "brier": 0.5393885075378817, + "expected_brier": 0.5393885075378817, + "multiclass_ece": 0.519320954486665, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/DISABLE ACCOUNT": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.48603731447493237, + "brier": 0.2978939068753025, + "expected_brier": 0.2978939068753025, + "multiclass_ece": 0.3832859260097785, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/ESCALATE TIER2": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.5295480740953074, + "brier": 0.3383392577936962, + "expected_brier": 0.3383392577936962, + "multiclass_ece": 0.4107287126260686, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/ESCALATE URGENT": { + "count": 3, + "hard_count": 3, + "accuracy": 0.6666666666666666, + "expected_accuracy": 0.6666666666666666, + "nll": 0.6989946515562225, + "brier": 0.5057164386544639, + "expected_brier": 0.5057164386544639, + "multiclass_ece": 0.5014168047279958, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 0.6666666666666666, + "expected_accuracy": 0.6666666666666666, + "expected_risk": 0.33333333333333337 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/ISOLATE HOST": { + "count": 3, + "hard_count": 3, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 0.7636022213713424, + "brier": 0.5701297052384428, + "expected_brier": 0.5701297052384428, + "multiclass_ece": 0.5332851159555024, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/KILL PROCESS": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.5916762325343339, + "brier": 0.39890511785457744, + "expected_brier": 0.3989051178545775, + "multiclass_ece": 0.4466011183677093, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/NOTIFY USER": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.5333053197285921, + "brier": 0.3428458733728703, + "expected_brier": 0.3428458733728703, + "multiclass_ece": 0.41169369765320746, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/QUARANTINE FILE": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.5528465013820195, + "brier": 0.36072343491918607, + "expected_brier": 0.3607234349191861, + "multiclass_ece": 0.42469014288018625, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/QUARANTINE MESSAGE": { + "count": 2, + "hard_count": 2, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 0.6276408445290848, + "brier": 0.4354540918891917, + "expected_brier": 0.4354540918891917, + "multiclass_ece": 0.46310418227632744, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/REMOVE FORWARDING RULES": { + "count": 2, + "hard_count": 2, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 0.9032385611477253, + "brier": 0.7068975744514125, + "expected_brier": 0.7068975744514124, + "multiclass_ece": 0.5940225352251051, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/REVOKE ACCESS KEY": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.44339088641818014, + "brier": 0.25653428658633926, + "expected_brier": 0.25653428658633926, + "multiclass_ece": 0.35814336743565467, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/REVOKE SESSIONS": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 0.8602098604645768, + "brier": 0.6656888651669612, + "expected_brier": 0.6656888651669612, + "multiclass_ece": 0.5769267133557612, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + } + } + }, + "baseline_calibrated_ood": { + "count": 512, + "hard_count": 484, + "accuracy": 0.5929752066115702, + "expected_accuracy": 0.5720486111111112, + "nll": 0.9673530687269922, + "brier": 0.4858965654094257, + "expected_brier": 0.5231090220066483, + "multiclass_ece": 0.05115238903752784, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 321, + "coverage": 0.626953125, + "accuracy": 0.7341772151898734, + "expected_accuracy": 0.7305295950155763, + "expected_risk": 0.26947040498442365 + }, + { + "threshold": 0.7, + "selected": 132, + "coverage": 0.2578125, + "accuracy": 0.8409090909090909, + "expected_accuracy": 0.8409090909090909, + "expected_risk": 0.15909090909090906 + }, + { + "threshold": 0.8, + "selected": 50, + "coverage": 0.09765625, + "accuracy": 0.9, + "expected_accuracy": 0.9, + "expected_risk": 0.09999999999999998 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 0.005859375, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ], + "mean_latency_seconds": 0.044071764506952604, + "by_kind": { + "choice": { + "count": 162, + "hard_count": 136, + "accuracy": 0.47058823529411764, + "expected_accuracy": 0.4272976680384088, + "nll": 1.3932442926232524, + "brier": 0.5461041456294194, + "expected_brier": 0.655483432323521, + "multiclass_ece": 0.07387139827506173, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 51, + "coverage": 0.3148148148148148, + "accuracy": 0.7391304347826086, + "expected_accuracy": 0.7156862745098039, + "expected_risk": 0.28431372549019607 + }, + { + "threshold": 0.7, + "selected": 6, + "coverage": 0.037037037037037035, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 0.018518518518518517, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "noul": { + "count": 243, + "hard_count": 243, + "accuracy": 0.7654320987654321, + "expected_accuracy": 0.7654320987654321, + "nll": 0.5365450218938118, + "brier": 0.35409756337043524, + "expected_brier": 0.35409756337043524, + "multiclass_ece": 0.08345380772386697, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 243, + "coverage": 1.0, + "accuracy": 0.7654320987654321, + "expected_accuracy": 0.7654320987654321, + "expected_risk": 0.23456790123456794 + }, + { + "threshold": 0.7, + "selected": 116, + "coverage": 0.4773662551440329, + "accuracy": 0.8362068965517241, + "expected_accuracy": 0.8362068965517241, + "expected_risk": 0.1637931034482759 + }, + { + "threshold": 0.8, + "selected": 44, + "coverage": 0.18106995884773663, + "accuracy": 0.8863636363636364, + "expected_accuracy": 0.8863636363636364, + "expected_risk": 0.11363636363636365 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 0.012345679012345678, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "score": { + "count": 107, + "hard_count": 105, + "accuracy": 0.3523809523809524, + "expected_accuracy": 0.3520249221183801, + "nll": 1.3009229482528697, + "brier": 0.6940603925106935, + "expected_brier": 0.7065214517007246, + "multiclass_ece": 0.12071699569742401, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 0.2523364485981308, + "accuracy": 0.4444444444444444, + "expected_accuracy": 0.4444444444444444, + "expected_risk": 0.5555555555555556 + }, + { + "threshold": 0.7, + "selected": 10, + "coverage": 0.09345794392523364, + "accuracy": 0.9, + "expected_accuracy": 0.9, + "expected_risk": 0.09999999999999998 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 0.028037383177570093, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ], + "ordinal_mae": 0.8590090044695711 + } + }, + "by_source": { + "customer-control-v1": { + "count": 81, + "hard_count": 77, + "accuracy": 0.7272727272727273, + "expected_accuracy": 0.7119341563786008, + "nll": 0.6727681312997662, + "brier": 0.36955298489873334, + "expected_brier": 0.3983595692608733, + "multiclass_ece": 0.08609680392851264, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 67, + "coverage": 0.8271604938271605, + "accuracy": 0.7424242424242424, + "expected_accuracy": 0.7388059701492538, + "expected_risk": 0.26119402985074625 + }, + { + "threshold": 0.7, + "selected": 30, + "coverage": 0.37037037037037035, + "accuracy": 0.9, + "expected_accuracy": 0.9, + "expected_risk": 0.09999999999999998 + }, + { + "threshold": 0.8, + "selected": 17, + "coverage": 0.20987654320987653, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 0.024691358024691357, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1": { + "count": 81, + "hard_count": 81, + "accuracy": 0.5555555555555556, + "expected_accuracy": 0.5555555555555556, + "nll": 1.3153303392761364, + "brier": 0.6535754906027468, + "expected_brier": 0.6535754906027468, + "multiclass_ece": 0.26906321122042465, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 0.3333333333333333, + "accuracy": 0.6296296296296297, + "expected_accuracy": 0.6296296296296297, + "expected_risk": 0.37037037037037035 + }, + { + "threshold": 0.7, + "selected": 16, + "coverage": 0.19753086419753085, + "accuracy": 0.5625, + "expected_accuracy": 0.5625, + "expected_risk": 0.4375 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 0.012345679012345678, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "reasoning-control-v1": { + "count": 80, + "hard_count": 80, + "accuracy": 0.4125, + "expected_accuracy": 0.4125, + "nll": 1.0503555607912074, + "brier": 0.6002890384668096, + "expected_brier": 0.6002890384668096, + "multiclass_ece": 0.12989847669727683, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 42, + "coverage": 0.525, + "accuracy": 0.6190476190476191, + "expected_accuracy": 0.6190476190476191, + "expected_risk": 0.38095238095238093 + }, + { + "threshold": 0.7, + "selected": 14, + "coverage": 0.175, + "accuracy": 0.8571428571428571, + "expected_accuracy": 0.8571428571428571, + "expected_risk": 0.1428571428571429 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 0.05, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 0.0125, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "snake-v1": { + "count": 54, + "hard_count": 48, + "accuracy": 0.7291666666666666, + "expected_accuracy": 0.7037037037037037, + "nll": 0.7296276207466045, + "brier": 0.37958186323907483, + "expected_brier": 0.4351374187946303, + "multiclass_ece": 0.08996044407164595, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 50, + "coverage": 0.9259259259259259, + "accuracy": 0.7391304347826086, + "expected_accuracy": 0.72, + "expected_risk": 0.28 + }, + { + "threshold": 0.7, + "selected": 28, + "coverage": 0.5185185185185185, + "accuracy": 0.8214285714285714, + "expected_accuracy": 0.8214285714285714, + "expected_risk": 0.1785714285714286 + }, + { + "threshold": 0.8, + "selected": 27, + "coverage": 0.5, + "accuracy": 0.8518518518518519, + "expected_accuracy": 0.8518518518518519, + "expected_risk": 0.14814814814814814 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "vizdoom-basic-v1": { + "count": 81, + "hard_count": 81, + "accuracy": 0.37037037037037035, + "expected_accuracy": 0.37037037037037035, + "nll": 1.0747455587422148, + "brier": 0.6140449763497663, + "expected_brier": 0.6140449763497661, + "multiclass_ece": 0.1257638319061848, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 0.3333333333333333, + "accuracy": 0.7037037037037037, + "expected_accuracy": 0.7037037037037037, + "expected_risk": 0.2962962962962963 + }, + { + "threshold": 0.7, + "selected": 10, + "coverage": 0.12345679012345678, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "wikispeedia-v1": { + "count": 27, + "hard_count": 9, + "accuracy": 0.1111111111111111, + "expected_accuracy": 0.0823045267489712, + "nll": 2.4928829324420256, + "brier": 0.4163735576196348, + "expected_brier": 0.9245011296360957, + "multiclass_ece": 0.10284727189377067, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ], + "optimal_action_hit": 0.2962962962962963, + "optimal_set_probability_mass": 0.34477984576725235 + }, + "workflow-controls-v1/agent_trace_observability": { + "count": 27, + "hard_count": 27, + "accuracy": 0.7777777777777778, + "expected_accuracy": 0.7777777777777778, + "nll": 0.5198509335594539, + "brier": 0.336245164696209, + "expected_brier": 0.336245164696209, + "multiclass_ece": 0.1694919014499214, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.7777777777777778, + "expected_accuracy": 0.7777777777777778, + "expected_risk": 0.2222222222222222 + }, + { + "threshold": 0.7, + "selected": 20, + "coverage": 0.7407407407407407, + "accuracy": 0.85, + "expected_accuracy": 0.85, + "expected_risk": 0.15000000000000002 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 0.037037037037037035, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service": { + "count": 27, + "hard_count": 27, + "accuracy": 0.9259259259259259, + "expected_accuracy": 0.9259259259259259, + "nll": 0.5081781521165888, + "brier": 0.322172104554329, + "expected_brier": 0.322172104554329, + "multiclass_ece": 0.30464249310560043, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.9259259259259259, + "expected_accuracy": 0.9259259259259259, + "expected_risk": 0.07407407407407407 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 0.07407407407407407, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing": { + "count": 27, + "hard_count": 27, + "accuracy": 0.8888888888888888, + "expected_accuracy": 0.8888888888888888, + "nll": 0.4912702574531617, + "brier": 0.30644051601973343, + "expected_brier": 0.3064405160197335, + "multiclass_ece": 0.214788969783477, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.8888888888888888, + "expected_accuracy": 0.8888888888888888, + "expected_risk": 0.11111111111111116 + }, + { + "threshold": 0.7, + "selected": 11, + "coverage": 0.4074074074074074, + "accuracy": 0.9090909090909091, + "expected_accuracy": 0.9090909090909091, + "expected_risk": 0.09090909090909094 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents": { + "count": 27, + "hard_count": 27, + "accuracy": 0.6296296296296297, + "expected_accuracy": 0.6296296296296297, + "nll": 0.5717461847931958, + "brier": 0.383488960718248, + "expected_brier": 0.383488960718248, + "multiclass_ece": 0.2688906933188254, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.6296296296296297, + "expected_accuracy": 0.6296296296296297, + "expected_risk": 0.37037037037037035 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.037037037037037035, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + } + }, + "by_question": { + "customer-control-v1/bug_severity": { + "count": 8, + "hard_count": 8, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.530914488308788, + "brier": 1.0003424613604053, + "expected_brier": 1.0003424613604053, + "multiclass_ece": 0.5894577572844212, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 8, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "customer-control-v1/category": { + "count": 27, + "hard_count": 25, + "accuracy": 1.0, + "expected_accuracy": 0.9629629629629629, + "nll": 0.5100181718711232, + "brier": 0.1956531556655967, + "expected_brier": 0.23269019270263375, + "multiclass_ece": 0.3374681495326716, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 21, + "coverage": 0.7777777777777778, + "accuracy": 1.0, + "expected_accuracy": 0.9761904761904762, + "expected_risk": 0.023809523809523836 + }, + { + "threshold": 0.7, + "selected": 5, + "coverage": 0.18518518518518517, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 0.1111111111111111, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "customer-control-v1/churn_likelihood_level": { + "count": 9, + "hard_count": 7, + "accuracy": 0.42857142857142855, + "expected_accuracy": 0.4074074074074074, + "nll": 1.0191206888494964, + "brier": 0.46163264772733964, + "expected_brier": 0.6097807958754877, + "multiclass_ece": 0.3024574716795273, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 0.1111111111111111, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.1111111111111111, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "customer-control-v1/frustration": { + "count": 10, + "hard_count": 10, + "accuracy": 0.4, + "expected_accuracy": 0.4, + "nll": 0.8994033379190144, + "brier": 0.6294492459879172, + "expected_brier": 0.6294492459879172, + "multiclass_ece": 0.42318492483640135, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 10, + "coverage": 1.0, + "accuracy": 0.4, + "expected_accuracy": 0.4, + "expected_risk": 0.6 + }, + { + "threshold": 0.7, + "selected": 5, + "coverage": 0.5, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 0.3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "customer-control-v1/has_reproducible_steps": { + "count": 15, + "hard_count": 15, + "accuracy": 0.9333333333333333, + "expected_accuracy": 0.9333333333333333, + "nll": 0.41678855423411815, + "brier": 0.24511111824695384, + "expected_brier": 0.24511111824695392, + "multiclass_ece": 0.2560639985701903, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 15, + "coverage": 1.0, + "accuracy": 0.9333333333333333, + "expected_accuracy": 0.9333333333333333, + "expected_risk": 0.06666666666666665 + }, + { + "threshold": 0.7, + "selected": 7, + "coverage": 0.4666666666666667, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 0.2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "customer-control-v1/refund_requested": { + "count": 12, + "hard_count": 12, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "nll": 0.338205349661921, + "brier": 0.21021365165112602, + "expected_brier": 0.21021365165112602, + "multiclass_ece": 0.232460813204974, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 12, + "coverage": 1.0, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.7, + "selected": 12, + "coverage": 1.0, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.8, + "selected": 8, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 0.16666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y11_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.23063070890267895, + "brier": 0.08484510587784777, + "expected_brier": 0.08484510587784788, + "multiclass_ece": 0.2059673589162222, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y11_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.2748771328888977, + "brier": 0.6514948022186009, + "expected_brier": 0.6514948022186009, + "multiclass_ece": 0.7205346967934978, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y3_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.33195255149839054, + "brier": 0.1595883557957513, + "expected_brier": 0.15958835579575137, + "multiclass_ece": 0.28247863263948947, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y8": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.22454112137903015, + "brier": 0.0808963167153417, + "expected_brier": 0.08089631671534159, + "multiclass_ece": 0.20111727513485966, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y8_red": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.8127594055798792, + "brier": 1.4004578655496427, + "expected_brier": 1.400457865549643, + "multiclass_ece": 0.8367968288508396, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x10_y0_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.2204502624333398, + "brier": 0.6227125694493556, + "expected_brier": 0.6227125694493556, + "multiclass_ece": 0.7049027342069016, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x10_y11": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.521098178879286, + "brier": 0.9054994062494798, + "expected_brier": 0.9054994062494797, + "multiclass_ece": 0.1028815180046696, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x10_y4_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.3306279761957566, + "brier": 0.15851556992729732, + "expected_brier": 0.15851556992729732, + "multiclass_ece": 0.28152759183364007, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x10_y4_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.145650311959656, + "brier": 0.6999789196393444, + "expected_brier": 0.6999789196393444, + "multiclass_ece": 0.3751073379660228, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x10_y5_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.920802693179456, + "brier": 0.5433029920579852, + "expected_brier": 0.5433029920579853, + "multiclass_ece": 0.6018007190798274, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x10_y7_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.1221645039240746, + "brier": 0.5741820214087683, + "expected_brier": 0.5741820214087683, + "multiclass_ece": 0.6744256755009107, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x10_y9_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.24332677532238384, + "brier": 0.0932988028119374, + "expected_brier": 0.09329880281193748, + "multiclass_ece": 0.21598472493666931, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x10_y9_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.9367352761750276, + "brier": 0.8416999043445993, + "expected_brier": 0.8416999043445994, + "multiclass_ece": 0.2186328776854894, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x11_y0_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.3922311049853324, + "brier": 0.6611835405026325, + "expected_brier": 0.6611835405026325, + "multiclass_ece": 0.7514797890558085, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x11_y10": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.562247124906817, + "brier": 0.926068373765721, + "expected_brier": 0.9260683737657209, + "multiclass_ece": 0.1690240629310105, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x11_y1_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.123806294944268, + "brier": 0.5756918865046868, + "expected_brier": 0.575691886504687, + "multiclass_ece": 0.6749597619541392, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y0": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.087892566689033, + "brier": 0.8226878952695186, + "expected_brier": 0.8226878952695187, + "multiclass_ece": 0.8760519269350362, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y10_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.5026148859266337, + "brier": 1.2088648964135822, + "expected_brier": 1.2088648964135822, + "multiclass_ece": 0.777452537591068, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y3_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.9426191997994913, + "brier": 0.5589053178007145, + "expected_brier": 0.5589053178007144, + "multiclass_ece": 0.6103939582645563, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y4_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.2488546492112513, + "brier": 0.7701217297005103, + "expected_brier": 0.7701217297005103, + "multiclass_ece": 0.4166303323648858, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y6_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.0618679128118442, + "brier": 0.8884992455897193, + "expected_brier": 0.8884992455897192, + "multiclass_ece": 0.23611857585279794, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y7_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.1614921330705934, + "brier": 0.5937023299059921, + "expected_brier": 0.5937023299059923, + "multiclass_ece": 0.6869812334061813, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y7_red": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.3355578468649334, + "brier": 1.08630435051574, + "expected_brier": 1.0863043505157401, + "multiclass_ece": 0.7369885855682365, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y8_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.9650861162143947, + "brier": 0.9379658695771852, + "expected_brier": 0.9379658695771851, + "multiclass_ece": 0.3146889857202914, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y9_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.5890838561326002, + "brier": 1.2668737535606012, + "expected_brier": 1.266873753560601, + "multiclass_ece": 0.7958874774616702, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y9_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.474504377176733, + "brier": 0.7168876202211282, + "expected_brier": 0.7168876202211281, + "multiclass_ece": 0.2983634038078812, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y10_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.2220249423219094, + "brier": 0.6271064760724068, + "expected_brier": 0.6271064760724068, + "multiclass_ece": 0.7053670522644202, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y11_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.4776637408585305, + "brier": 1.1914426941832092, + "expected_brier": 1.191442694183209, + "multiclass_ece": 0.7718298692662812, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y4_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.1675868490290697, + "brier": 0.5975824085327801, + "expected_brier": 0.5975824085327801, + "multiclass_ece": 0.6888831920419783, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y6_red": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.1403923076090292, + "brier": 0.9256336529396876, + "expected_brier": 0.9256336529396876, + "multiclass_ece": 0.6803064210117701, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y6_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.2416367331227591, + "brier": 0.7656394488672806, + "expected_brier": 0.7656394488672806, + "multiclass_ece": 0.4154111044994898, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y8": { + "count": 2, + "hard_count": 2, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.4293675128757455, + "brier": 0.8929019302785516, + "expected_brier": 0.8929019302785516, + "multiclass_ece": 0.12304449620321489, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x3_y0_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.232940931440015, + "brier": 0.6311795921183583, + "expected_brier": 0.6311795921183583, + "multiclass_ece": 0.7085657719581444, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x3_y11_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.24800801435152092, + "brier": 0.09648900367484109, + "expected_brier": 0.09648900367484115, + "multiclass_ece": 0.21964631077580288, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x3_y1_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.44348979014592793, + "brier": 0.25662481587693026, + "expected_brier": 0.2566248158769302, + "multiclass_ece": 0.3582072136884811, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x3_y6": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.6293096244346645, + "brier": 0.934006983802412, + "expected_brier": 0.9340069838024121, + "multiclass_ece": 0.16976061557464905, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x3_y7_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.4436733841137994, + "brier": 0.25679365734057336, + "expected_brier": 0.25679365734057336, + "multiclass_ece": 0.3583250321569603, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x3_y7_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.9716339841946517, + "brier": 0.43710350657307506, + "expected_brier": 0.43710350657307506, + "multiclass_ece": 0.6215358718071143, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y10_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.831871848514696, + "brier": 0.8123466072600078, + "expected_brier": 0.8123466072600078, + "multiclass_ece": 0.21777122944663305, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y11_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.248667951481408, + "brier": 0.6387250322399264, + "expected_brier": 0.6387250322399263, + "multiclass_ece": 0.7131133105563254, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y1_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.670282669299773, + "brier": 0.8005169000148206, + "expected_brier": 0.8005169000148205, + "multiclass_ece": 0.32541709012046166, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y1_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.8771927558850522, + "brier": 0.5116901477920095, + "expected_brier": 0.5116901477920095, + "multiclass_ece": 0.5840510555270289, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y4_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.9157674332797907, + "brier": 0.5397861297493286, + "expected_brier": 0.5397861297493285, + "multiclass_ece": 0.5997906257844159, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y5_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.1910666034578326, + "brier": 0.7310032762686283, + "expected_brier": 0.7310032762686283, + "multiclass_ece": 0.39368100631536396, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y6_red": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.3834194928641712, + "brier": 1.1228417825941897, + "expected_brier": 1.1228417825941897, + "multiclass_ece": 0.7492802488369054, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y1_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.2062219781245296, + "brier": 0.6184633267698082, + "expected_brier": 0.6184633267698083, + "multiclass_ece": 0.700673993890746, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y2": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.6704667228567713, + "brier": 0.7136640503416908, + "expected_brier": 0.7136640503416909, + "multiclass_ece": 0.811840773069746, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y4_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.131904544624681, + "brier": 0.52367487303233, + "expected_brier": 0.52367487303233, + "multiclass_ece": 0.6775813893339322, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y4_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.1973918315525252, + "brier": 0.6118209944946463, + "expected_brier": 0.6118209944946464, + "multiclass_ece": 0.698019197513269, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y7": { + "count": 2, + "hard_count": 2, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 2.203638558425087, + "brier": 0.8251008329578768, + "expected_brier": 0.8251008329578768, + "multiclass_ece": 0.46795211898920214, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y8": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 0.7635278521408722, + "brier": 0.5702704004232231, + "expected_brier": 0.5702704004232231, + "multiclass_ece": 0.5339805241875508, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y9": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.61801441394592, + "brier": 0.9315566188927892, + "expected_brier": 0.9315566188927891, + "multiclass_ece": 0.15989552150537809, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y9_red": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.24486863549773644, + "brier": 0.09434528175470507, + "expected_brier": 0.09434528175470502, + "multiclass_ece": 0.21719263541232825, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y0_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.9379248195268448, + "brier": 0.9238105280186124, + "expected_brier": 0.9238105280186124, + "multiclass_ece": 0.28663367753321467, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y1_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.0507135205544693, + "brier": 0.8746865269836542, + "expected_brier": 0.8746865269836541, + "multiclass_ece": 0.19711740783120935, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y3": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.533045525717225, + "brier": 0.9130543124531431, + "expected_brier": 0.9130543124531432, + "multiclass_ece": 0.13873701403808053, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y3_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.4272364361494068, + "brier": 0.24177763948348704, + "expected_brier": 0.24177763948348696, + "multiclass_ece": 0.34769069550642784, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y6_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.021054423597579, + "brier": 0.5211081367473445, + "expected_brier": 0.5211081367473445, + "multiclass_ece": 0.6397850792550723, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y7_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.0675952731908749, + "brier": 0.5438549000612977, + "expected_brier": 0.5438549000612976, + "multiclass_ece": 0.656165648248682, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y8_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.467690599243395, + "brier": 1.184392584101169, + "expected_brier": 1.184392584101169, + "multiclass_ece": 0.7695429111171024, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y1_lightness": { + "count": 2, + "hard_count": 2, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 1.5908527115677087, + "brier": 0.7779080328649663, + "expected_brier": 0.7779080328649663, + "multiclass_ece": 0.5129832817553724, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y2": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.42108603618649904, + "brier": 0.23621314699935383, + "expected_brier": 0.2362131469993538, + "multiclass_ece": 0.3436663694627057, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y5_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.36739356705623455, + "brier": 0.18906695382400052, + "expected_brier": 0.18906695382400063, + "multiclass_ece": 0.30746296835879317, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y6_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.1970898801380017, + "brier": 0.734293314266187, + "expected_brier": 0.7342933142661869, + "multiclass_ece": 0.39161417033103735, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y7": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 0.7021623381344039, + "brier": 0.509014914989029, + "expected_brier": 0.5090149149890291, + "multiclass_ece": 0.5044873214408013, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y7_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.4668784690541495, + "brier": 0.27832323350730304, + "expected_brier": 0.27832323350730315, + "multiclass_ece": 0.37304371962767513, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y8": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.35092415732465065, + "brier": 0.17518801965620218, + "expected_brier": 0.17518801965620212, + "multiclass_ece": 0.2959628521083365, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y9_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.8686919316349115, + "brier": 0.5058083693563138, + "expected_brier": 0.5058083693563138, + "multiclass_ece": 0.5805000749059281, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x8_y0_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.42094336206277966, + "brier": 0.23608442917111325, + "expected_brier": 0.23608442917111327, + "multiclass_ece": 0.3435727209566508, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x8_y11_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.24402348175138755, + "brier": 0.09377114280920529, + "expected_brier": 0.09377114280920518, + "multiclass_ece": 0.21653076318297748, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x8_y3": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.122948084948761, + "brier": 0.8290543010228879, + "expected_brier": 0.8290543010228879, + "multiclass_ece": 0.880321713828402, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x9_y0": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.275603058278874, + "brier": 0.8619786577501026, + "expected_brier": 0.8619786577501025, + "multiclass_ece": 0.897265065402592, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x9_y0_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.8940786251195485, + "brier": 0.832611882936688, + "expected_brier": 0.832611882936688, + "multiclass_ece": 0.23273117279355549, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x9_y1_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.575326558633497, + "brier": 0.7426857924945629, + "expected_brier": 0.7426857924945629, + "multiclass_ece": 0.26957146943150156, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x9_y2": { + "count": 2, + "hard_count": 2, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 2.253073844784778, + "brier": 0.8548384985147142, + "expected_brier": 0.8548384985147143, + "multiclass_ece": 0.4822427506685555, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x9_y6_red": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.3686842355880288, + "brier": 0.19016712612278192, + "expected_brier": 0.19016712612278197, + "multiclass_ece": 0.30835622753787695, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x9_y9_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.874297839969473, + "brier": 0.8506331338525093, + "expected_brier": 0.8506331338525093, + "multiclass_ece": 0.29864167326643376, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "reasoning-control-v1/decision": { + "count": 27, + "hard_count": 27, + "accuracy": 0.2962962962962963, + "expected_accuracy": 0.2962962962962963, + "nll": 1.1703672571730277, + "brier": 0.6637695365969065, + "expected_brier": 0.6637695365969065, + "multiclass_ece": 0.1685835910772117, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 7, + "coverage": 0.25925925925925924, + "accuracy": 0.42857142857142855, + "expected_accuracy": 0.42857142857142855, + "expected_risk": 0.5714285714285714 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "reasoning-control-v1/ordinal": { + "count": 26, + "hard_count": 26, + "accuracy": 0.34615384615384615, + "expected_accuracy": 0.34615384615384615, + "nll": 1.3272683777713112, + "brier": 0.6614395053507791, + "expected_brier": 0.6614395053507791, + "multiclass_ece": 0.23518731753764055, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 8, + "coverage": 0.3076923076923077, + "accuracy": 0.875, + "expected_accuracy": 0.875, + "expected_risk": 0.125 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 0.15384615384615385, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "reasoning-control-v1/verification": { + "count": 27, + "hard_count": 27, + "accuracy": 0.5925925925925926, + "expected_accuracy": 0.5925925925925926, + "nll": 0.6636870776878055, + "brier": 0.47792290555955586, + "expected_brier": 0.477922905559556, + "multiclass_ece": 0.23172037099888046, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.5925925925925926, + "expected_accuracy": 0.5925925925925926, + "expected_risk": 0.40740740740740744 + }, + { + "threshold": 0.7, + "selected": 10, + "coverage": 0.37037037037037035, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 0.14814814814814814, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 0.037037037037037035, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "snake-v1/choice": { + "count": 27, + "hard_count": 21, + "accuracy": 0.5714285714285714, + "expected_accuracy": 0.5555555555555556, + "nll": 1.0402399973724923, + "brier": 0.5068943529801929, + "expected_brier": 0.6180054640913042, + "multiclass_ece": 0.16204289048600556, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 23, + "coverage": 0.8518518518518519, + "accuracy": 0.5789473684210527, + "expected_accuracy": 0.5652173913043478, + "expected_risk": 0.4347826086956522 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.037037037037037035, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "snake-v1/noul": { + "count": 27, + "hard_count": 27, + "accuracy": 0.8518518518518519, + "expected_accuracy": 0.8518518518518519, + "nll": 0.4190152441207165, + "brier": 0.25226937349795664, + "expected_brier": 0.2522693734979566, + "multiclass_ece": 0.017877997657286333, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.8518518518518519, + "expected_accuracy": 0.8518518518518519, + "expected_risk": 0.14814814814814814 + }, + { + "threshold": 0.7, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.8518518518518519, + "expected_accuracy": 0.8518518518518519, + "expected_risk": 0.14814814814814814 + }, + { + "threshold": 0.8, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.8518518518518519, + "expected_accuracy": 0.8518518518518519, + "expected_risk": 0.14814814814814814 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "vizdoom-basic-v1/choice": { + "count": 27, + "hard_count": 27, + "accuracy": 0.3333333333333333, + "expected_accuracy": 0.3333333333333333, + "nll": 1.1215783822553531, + "brier": 0.6803867651047123, + "expected_brier": 0.6803867651047123, + "multiclass_ece": 0.04108271365193536, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "vizdoom-basic-v1/noul": { + "count": 27, + "hard_count": 27, + "accuracy": 0.7037037037037037, + "expected_accuracy": 0.7037037037037037, + "nll": 0.5702146237063598, + "brier": 0.3828828938013468, + "expected_brier": 0.3828828938013467, + "multiclass_ece": 0.11151535079704884, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.7037037037037037, + "expected_accuracy": 0.7037037037037037, + "expected_risk": 0.2962962962962963 + }, + { + "threshold": 0.7, + "selected": 10, + "coverage": 0.37037037037037035, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "vizdoom-basic-v1/score": { + "count": 27, + "hard_count": 27, + "accuracy": 0.07407407407407407, + "expected_accuracy": 0.07407407407407407, + "nll": 1.5324436702649293, + "brier": 0.7788652701432405, + "expected_brier": 0.7788652701432405, + "multiclass_ece": 0.22469343126957006, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "wikispeedia-v1/choice": { + "count": 27, + "hard_count": 9, + "accuracy": 0.1111111111111111, + "expected_accuracy": 0.0823045267489712, + "nll": 2.4928829324420256, + "brier": 0.4163735576196348, + "expected_brier": 0.9245011296360957, + "multiclass_ece": 0.10284727189377067, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/AUTO-CLOSE": { + "count": 4, + "hard_count": 4, + "accuracy": 0.25, + "expected_accuracy": 0.25, + "nll": 0.8632333748907437, + "brier": 0.6661704330527739, + "expected_brier": 0.6661704330527739, + "multiclass_ece": 0.5735530645162839, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.25, + "expected_accuracy": 0.25, + "expected_risk": 0.75 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/COUNT ONLY": { + "count": 6, + "hard_count": 6, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.30872824266323445, + "brier": 0.14176575165408284, + "expected_brier": 0.14176575165408287, + "multiclass_ece": 0.2652595433780158, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 6, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 5, + "coverage": 0.8333333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/FILE ISSUE ยท ROUTE": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.22608982396587085, + "brier": 0.08202162217915736, + "expected_brier": 0.08202162217915732, + "multiclass_ece": 0.20229992326818735, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/HUMAN REVIEW": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.34731791611368507, + "brier": 0.17218981914679635, + "expected_brier": 0.17218981914679632, + "multiclass_ece": 0.29341934083048815, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/NOT A BUG": { + "count": 6, + "hard_count": 6, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "nll": 0.47317165464389555, + "brier": 0.2936817279940039, + "expected_brier": 0.2936817279940039, + "multiclass_ece": 0.09568238640439164, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 6, + "coverage": 1.0, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.7, + "selected": 6, + "coverage": 1.0, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/PAGE ON-CALL": { + "count": 5, + "hard_count": 5, + "accuracy": 0.6, + "expected_accuracy": 0.6, + "nll": 0.7841319807637331, + "brier": 0.5666672765742101, + "expected_brier": 0.5666672765742102, + "multiclass_ece": 0.27746709169001266, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 1.0, + "accuracy": 0.6, + "expected_accuracy": 0.6, + "expected_risk": 0.4 + }, + { + "threshold": 0.7, + "selected": 5, + "coverage": 1.0, + "accuracy": 0.6, + "expected_accuracy": 0.6, + "expected_risk": 0.4 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/PRIORITY REVIEW": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.3904949516118019, + "brier": 0.21056113010728897, + "expected_brier": 0.21056113010728894, + "multiclass_ece": 0.3221537748354108, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.3333333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/CANCEL": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.4160608640021789, + "brier": 0.23430587537047914, + "expected_brier": 0.2343058753704792, + "multiclass_ece": 0.33802494044777237, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/CLOSE": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.46929727483610256, + "brier": 0.2855313374489989, + "expected_brier": 0.28553133744899895, + "multiclass_ece": 0.3688823471987357, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/FLAG FOR REVIEW": { + "count": 5, + "hard_count": 5, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "nll": 0.599865322124221, + "brier": 0.40740530486101934, + "expected_brier": 0.40740530486101945, + "multiclass_ece": 0.44945794587504134, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 1.0, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/FREEZE CARD": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.39046456219181325, + "brier": 0.22355382747031968, + "expected_brier": 0.2235538274703197, + "multiclass_ece": 0.3114156897652715, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/HAND OFF": { + "count": 4, + "hard_count": 4, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "nll": 0.6014562893571106, + "brier": 0.4097433760212271, + "expected_brier": 0.40974337602122707, + "multiclass_ece": 0.11757701818450572, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/REFUND": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.5710406776344851, + "brier": 0.37855925351872843, + "expected_brier": 0.3785592535187283, + "multiclass_ece": 0.4350627848476174, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/SAY": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.5243544015901896, + "brier": 0.33332972221501317, + "expected_brier": 0.3333297222150131, + "multiclass_ece": 0.40763155386369476, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/SET INTENT": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.4820063724002088, + "brier": 0.2949048445433088, + "expected_brier": 0.2949048445433088, + "multiclass_ece": 0.3796056721657405, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/DISPUTE LINES": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.4108227404714535, + "brier": 0.22794932051842576, + "expected_brier": 0.22794932051842576, + "multiclass_ece": 0.33619612486733547, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.3333333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/DUPLICATE": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.42085463165613485, + "brier": 0.2454252651262599, + "expected_brier": 0.24542526512625995, + "multiclass_ece": 0.33616184920403536, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/FRAUD REVIEW": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.2979684908695264, + "brier": 0.13287992961199668, + "expected_brier": 0.13287992961199668, + "multiclass_ece": 0.2576301264785245, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/HOLD": { + "count": 3, + "hard_count": 3, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.2022217379067621, + "brier": 0.9777504820109636, + "expected_brier": 0.9777504820109638, + "multiclass_ece": 0.6989945752092214, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.3333333333333333, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/HOLD FOR DOCUMENTS": { + "count": 5, + "hard_count": 5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.5032350382115035, + "brier": 0.312853178467717, + "expected_brier": 0.312853178467717, + "multiclass_ece": 0.39526868293008766, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/PAY": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.443131938694592, + "brier": 0.25629578976189804, + "expected_brier": 0.25629578976189804, + "multiclass_ece": 0.35797750611029877, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/REQUEST CORRECTED INVOICE": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.40123677518938633, + "brier": 0.21856154581284584, + "expected_brier": 0.2185615458128459, + "multiclass_ece": 0.3304422603472584, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/ROUTE FOR APPROVAL": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.3868169042200822, + "brier": 0.2058054614223787, + "expected_brier": 0.20580546142237877, + "multiclass_ece": 0.32078455497606073, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/SCHEDULE": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.3977465422159223, + "brier": 0.2165472313931614, + "expected_brier": 0.21654723139316134, + "multiclass_ece": 0.3273265541108304, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/SHORT PAY": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.33605533405191845, + "brier": 0.16303949052710015, + "expected_brier": 0.16303949052710015, + "multiclass_ece": 0.28534952308391426, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/AUTO CLOSE": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.4226574691909153, + "brier": 0.23864367531311825, + "expected_brier": 0.23864367531311825, + "multiclass_ece": 0.3439178725312782, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.3333333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/BLOCK DESTINATION": { + "count": 2, + "hard_count": 2, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 0.5828750304431148, + "brier": 0.3969903413123217, + "expected_brier": 0.3969903413123218, + "multiclass_ece": 0.4286418910667888, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/BLOCK EGRESS ASSET": { + "count": 4, + "hard_count": 4, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "nll": 0.48742004069786676, + "brier": 0.30118380774562314, + "expected_brier": 0.30118380774562314, + "multiclass_ece": 0.3807953259285537, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/BLOCK SENDER": { + "count": 3, + "hard_count": 3, + "accuracy": 0.3333333333333333, + "expected_accuracy": 0.3333333333333333, + "nll": 0.6646074197101562, + "brier": 0.47276082409721987, + "expected_brier": 0.47276082409721987, + "multiclass_ece": 0.4807401853887603, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 0.3333333333333333, + "expected_accuracy": 0.3333333333333333, + "expected_risk": 0.6666666666666667 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/ESCALATE TIER2": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.42666025817325104, + "brier": 0.24125505856729265, + "expected_brier": 0.24125505856729257, + "multiclass_ece": 0.3473147409535713, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/ESCALATE URGENT": { + "count": 2, + "hard_count": 2, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 0.7480735297865202, + "brier": 0.5545234922286498, + "expected_brier": 0.5545234922286498, + "multiclass_ece": 0.5250326966858994, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/ISOLATE HOST": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 0.7584233102308179, + "brier": 0.5651878202614115, + "expected_brier": 0.5651878202614116, + "multiclass_ece": 0.531595626515781, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/NOTIFY USER": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.3785422048461696, + "brier": 0.1986276164720277, + "expected_brier": 0.19862761647202776, + "multiclass_ece": 0.3151409339264163, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/PURGE MAILBOXES": { + "count": 2, + "hard_count": 2, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 0.6688127779592022, + "brier": 0.4782377058310284, + "expected_brier": 0.4782377058310284, + "multiclass_ece": 0.12843079540188773, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/QUARANTINE FILE": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.5641061037242163, + "brier": 0.3717488631673522, + "expected_brier": 0.37174886316735223, + "multiclass_ece": 0.43113157108204925, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/QUARANTINE MESSAGE": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.42114966890329464, + "brier": 0.2363186703599959, + "expected_brier": 0.23631867035999587, + "multiclass_ece": 0.34366965965656116, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/REQUIRE REAUTH": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.3741747823061255, + "brier": 0.19486690925373318, + "expected_brier": 0.19486690925373318, + "multiclass_ece": 0.31214332385439003, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/REVOKE ACCESS KEY": { + "count": 2, + "hard_count": 2, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 0.5736062811682758, + "brier": 0.38341413143558606, + "expected_brier": 0.3834141314355861, + "multiclass_ece": 0.43207477787759635, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/security_incidents/REVOKE SESSIONS": { + "count": 2, + "hard_count": 2, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 0.8673654620601048, + "brier": 0.6722991300611049, + "expected_brier": 0.672299130061105, + "multiclass_ece": 0.5793638354113543, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + } + } + }, + "trained_test": { + "count": 512, + "hard_count": 470, + "accuracy": 0.9170212765957447, + "expected_accuracy": 0.8705829923115078, + "nll": 0.3445566359449767, + "brier": 0.11845151732024654, + "expected_brier": 0.16974352500873854, + "multiclass_ece": 0.03822707632976721, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 460, + "coverage": 0.8984375, + "accuracy": 0.9486607142857143, + "expected_accuracy": 0.936231884057971, + "expected_risk": 0.06376811594202902 + }, + { + "threshold": 0.7, + "selected": 434, + "coverage": 0.84765625, + "accuracy": 0.9629629629629629, + "expected_accuracy": 0.9600614439324118, + "expected_risk": 0.03993855606758823 + }, + { + "threshold": 0.8, + "selected": 427, + "coverage": 0.833984375, + "accuracy": 0.9670588235294117, + "expected_accuracy": 0.9640905542544888, + "expected_risk": 0.03590944574551125 + }, + { + "threshold": 0.9, + "selected": 422, + "coverage": 0.82421875, + "accuracy": 0.9691211401425178, + "expected_accuracy": 0.9676145339652448, + "expected_risk": 0.03238546603475523 + }, + { + "threshold": 0.95, + "selected": 408, + "coverage": 0.796875, + "accuracy": 0.9779411764705882, + "expected_accuracy": 0.9779411764705882, + "expected_risk": 0.022058823529411797 + }, + { + "threshold": 0.99, + "selected": 367, + "coverage": 0.716796875, + "accuracy": 0.9918256130790191, + "expected_accuracy": 0.9918256130790191, + "expected_risk": 0.008174386920980936 + } + ], + "mean_latency_seconds": 0.040071311304927804, + "by_kind": { + "choice": { + "count": 197, + "hard_count": 158, + "accuracy": 0.8481012658227848, + "expected_accuracy": 0.7474035130126501, + "nll": 0.6352537617928283, + "brier": 0.17937014041883728, + "expected_brier": 0.30506307410669475, + "multiclass_ece": 0.05413382209596949, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 151, + "coverage": 0.766497461928934, + "accuracy": 0.9295774647887324, + "expected_accuracy": 0.901766004415011, + "expected_risk": 0.09823399558498902 + }, + { + "threshold": 0.7, + "selected": 134, + "coverage": 0.6802030456852792, + "accuracy": 0.9545454545454546, + "expected_accuracy": 0.9452736318407959, + "expected_risk": 0.054726368159204064 + }, + { + "threshold": 0.8, + "selected": 129, + "coverage": 0.6548223350253807, + "accuracy": 0.9606299212598425, + "expected_accuracy": 0.9509043927648578, + "expected_risk": 0.04909560723514217 + }, + { + "threshold": 0.9, + "selected": 125, + "coverage": 0.6345177664974619, + "accuracy": 0.9596774193548387, + "expected_accuracy": 0.9546666666666667, + "expected_risk": 0.04533333333333334 + }, + { + "threshold": 0.95, + "selected": 118, + "coverage": 0.5989847715736041, + "accuracy": 0.9830508474576272, + "expected_accuracy": 0.9830508474576272, + "expected_risk": 0.016949152542372836 + }, + { + "threshold": 0.99, + "selected": 100, + "coverage": 0.5076142131979695, + "accuracy": 0.99, + "expected_accuracy": 0.99, + "expected_risk": 0.010000000000000009 + } + ] + }, + "noul": { + "count": 219, + "hard_count": 219, + "accuracy": 0.958904109589041, + "expected_accuracy": 0.958904109589041, + "nll": 0.13259250568722236, + "brier": 0.06836412768662707, + "expected_brier": 0.06836412768662709, + "multiclass_ece": 0.02992741567289525, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 219, + "coverage": 1.0, + "accuracy": 0.958904109589041, + "expected_accuracy": 0.958904109589041, + "expected_risk": 0.04109589041095896 + }, + { + "threshold": 0.7, + "selected": 215, + "coverage": 0.9817351598173516, + "accuracy": 0.9674418604651163, + "expected_accuracy": 0.9674418604651163, + "expected_risk": 0.032558139534883734 + }, + { + "threshold": 0.8, + "selected": 213, + "coverage": 0.9726027397260274, + "accuracy": 0.971830985915493, + "expected_accuracy": 0.971830985915493, + "expected_risk": 0.028169014084507005 + }, + { + "threshold": 0.9, + "selected": 213, + "coverage": 0.9726027397260274, + "accuracy": 0.971830985915493, + "expected_accuracy": 0.971830985915493, + "expected_risk": 0.028169014084507005 + }, + { + "threshold": 0.95, + "selected": 207, + "coverage": 0.9452054794520548, + "accuracy": 0.9758454106280193, + "expected_accuracy": 0.9758454106280193, + "expected_risk": 0.024154589371980673 + }, + { + "threshold": 0.99, + "selected": 190, + "coverage": 0.867579908675799, + "accuracy": 0.9894736842105263, + "expected_accuracy": 0.9894736842105263, + "expected_risk": 0.010526315789473717 + } + ] + }, + "score": { + "count": 96, + "hard_count": 93, + "accuracy": 0.9354838709677419, + "expected_accuracy": 0.921875, + "nll": 0.23156508109520055, + "brier": 0.10770328377170828, + "expected_brier": 0.12332828377170828, + "multiclass_ece": 0.045470337364496044, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 90, + "coverage": 0.9375, + "accuracy": 0.9540229885057471, + "expected_accuracy": 0.9388888888888889, + "expected_risk": 0.061111111111111116 + }, + { + "threshold": 0.7, + "selected": 85, + "coverage": 0.8854166666666666, + "accuracy": 0.9647058823529412, + "expected_accuracy": 0.9647058823529412, + "expected_risk": 0.03529411764705881 + }, + { + "threshold": 0.8, + "selected": 85, + "coverage": 0.8854166666666666, + "accuracy": 0.9647058823529412, + "expected_accuracy": 0.9647058823529412, + "expected_risk": 0.03529411764705881 + }, + { + "threshold": 0.9, + "selected": 84, + "coverage": 0.875, + "accuracy": 0.9761904761904762, + "expected_accuracy": 0.9761904761904762, + "expected_risk": 0.023809523809523836 + }, + { + "threshold": 0.95, + "selected": 83, + "coverage": 0.8645833333333334, + "accuracy": 0.9759036144578314, + "expected_accuracy": 0.9759036144578314, + "expected_risk": 0.02409638554216864 + }, + { + "threshold": 0.99, + "selected": 77, + "coverage": 0.8020833333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ], + "ordinal_mae": 0.08757058469206182 + } + }, + "by_source": { + "customer-control-v1": { + "count": 72, + "hard_count": 68, + "accuracy": 1.0, + "expected_accuracy": 0.9722222222222222, + "nll": 0.039063016526917856, + "brier": 0.0001948080159315778, + "expected_brier": 0.027972585793709363, + "multiclass_ece": 0.002490174341177086, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 72, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 0.9722222222222222, + "expected_risk": 0.02777777777777779 + }, + { + "threshold": 0.7, + "selected": 68, + "coverage": 0.9444444444444444, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 68, + "coverage": 0.9444444444444444, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 68, + "coverage": 0.9444444444444444, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 68, + "coverage": 0.9444444444444444, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 68, + "coverage": 0.9444444444444444, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1": { + "count": 74, + "hard_count": 74, + "accuracy": 0.9864864864864865, + "expected_accuracy": 0.9864864864864865, + "nll": 0.014949831586112952, + "brier": 0.01187261707578913, + "expected_brier": 0.011872617075789122, + "multiclass_ece": 0.00921761613785085, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 74, + "coverage": 1.0, + "accuracy": 0.9864864864864865, + "expected_accuracy": 0.9864864864864865, + "expected_risk": 0.013513513513513487 + }, + { + "threshold": 0.7, + "selected": 73, + "coverage": 0.9864864864864865, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 73, + "coverage": 0.9864864864864865, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 73, + "coverage": 0.9864864864864865, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 73, + "coverage": 0.9864864864864865, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 73, + "coverage": 0.9864864864864865, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "reasoning-control-v1": { + "count": 72, + "hard_count": 72, + "accuracy": 0.8055555555555556, + "expected_accuracy": 0.8055555555555556, + "nll": 0.5863997398479727, + "brier": 0.30197363711126324, + "expected_brier": 0.3019736371112633, + "multiclass_ece": 0.1009585700755068, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 63, + "coverage": 0.875, + "accuracy": 0.8412698412698413, + "expected_accuracy": 0.8412698412698413, + "expected_risk": 0.15873015873015872 + }, + { + "threshold": 0.7, + "selected": 56, + "coverage": 0.7777777777777778, + "accuracy": 0.8928571428571429, + "expected_accuracy": 0.8928571428571429, + "expected_risk": 0.1071428571428571 + }, + { + "threshold": 0.8, + "selected": 52, + "coverage": 0.7222222222222222, + "accuracy": 0.9038461538461539, + "expected_accuracy": 0.9038461538461539, + "expected_risk": 0.09615384615384615 + }, + { + "threshold": 0.9, + "selected": 50, + "coverage": 0.6944444444444444, + "accuracy": 0.92, + "expected_accuracy": 0.92, + "expected_risk": 0.07999999999999996 + }, + { + "threshold": 0.95, + "selected": 44, + "coverage": 0.6111111111111112, + "accuracy": 0.9090909090909091, + "expected_accuracy": 0.9090909090909091, + "expected_risk": 0.09090909090909094 + }, + { + "threshold": 0.99, + "selected": 28, + "coverage": 0.3888888888888889, + "accuracy": 0.9642857142857143, + "expected_accuracy": 0.9642857142857143, + "expected_risk": 0.0357142857142857 + } + ] + }, + "snake-v1": { + "count": 48, + "hard_count": 43, + "accuracy": 0.8604651162790697, + "expected_accuracy": 0.8159722222222222, + "nll": 0.5913114926507481, + "brier": 0.26357674066932263, + "expected_brier": 0.3226045184471003, + "multiclass_ece": 0.14233832901848453, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 48, + "coverage": 1.0, + "accuracy": 0.8604651162790697, + "expected_accuracy": 0.8159722222222222, + "expected_risk": 0.1840277777777778 + }, + { + "threshold": 0.7, + "selected": 42, + "coverage": 0.875, + "accuracy": 0.85, + "expected_accuracy": 0.8253968253968254, + "expected_risk": 0.17460317460317465 + }, + { + "threshold": 0.8, + "selected": 40, + "coverage": 0.8333333333333334, + "accuracy": 0.868421052631579, + "expected_accuracy": 0.8416666666666666, + "expected_risk": 0.15833333333333344 + }, + { + "threshold": 0.9, + "selected": 38, + "coverage": 0.7916666666666666, + "accuracy": 0.8648648648648649, + "expected_accuracy": 0.850877192982456, + "expected_risk": 0.149122807017544 + }, + { + "threshold": 0.95, + "selected": 33, + "coverage": 0.6875, + "accuracy": 0.8787878787878788, + "expected_accuracy": 0.8787878787878788, + "expected_risk": 0.12121212121212122 + }, + { + "threshold": 0.99, + "selected": 18, + "coverage": 0.375, + "accuracy": 0.9444444444444444, + "expected_accuracy": 0.9444444444444444, + "expected_risk": 0.05555555555555558 + } + ] + }, + "tic_tac_toe-v1": { + "count": 24, + "hard_count": 7, + "accuracy": 0.2857142857142857, + "expected_accuracy": 0.3323412698412699, + "nll": 1.2596593300800374, + "brier": 0.2679342073845446, + "expected_brier": 0.6855929375432749, + "multiclass_ece": 0.1396299186022388, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 0.20833333333333334, + "accuracy": 1.0, + "expected_accuracy": 0.7, + "expected_risk": 0.30000000000000004 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.041666666666666664, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 0.041666666666666664, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "tile_platformer-v1": { + "count": 24, + "hard_count": 24, + "accuracy": 0.9583333333333334, + "expected_accuracy": 0.9583333333333334, + "nll": 0.3296336081760603, + "brier": 0.08316409006817983, + "expected_brier": 0.08316409006817983, + "multiclass_ece": 0.03909390729886505, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.9583333333333334, + "expected_accuracy": 0.9583333333333334, + "expected_risk": 0.04166666666666663 + }, + { + "threshold": 0.7, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.9583333333333334, + "expected_accuracy": 0.9583333333333334, + "expected_risk": 0.04166666666666663 + }, + { + "threshold": 0.8, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.9583333333333334, + "expected_accuracy": 0.9583333333333334, + "expected_risk": 0.04166666666666663 + }, + { + "threshold": 0.9, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.9583333333333334, + "expected_accuracy": 0.9583333333333334, + "expected_risk": 0.04166666666666663 + }, + { + "threshold": 0.95, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.9583333333333334, + "expected_accuracy": 0.9583333333333334, + "expected_risk": 0.04166666666666663 + }, + { + "threshold": 0.99, + "selected": 23, + "coverage": 0.9583333333333334, + "accuracy": 0.9565217391304348, + "expected_accuracy": 0.9565217391304348, + "expected_risk": 0.04347826086956519 + } + ] + }, + "trex_runner-v1": { + "count": 4, + "hard_count": 4, + "accuracy": 0.25, + "expected_accuracy": 0.25, + "nll": 1.7164753177424987, + "brier": 0.9991415297201487, + "expected_brier": 0.9991415297201485, + "multiclass_ece": 0.5572777459998814, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.25, + "expected_accuracy": 0.25, + "expected_risk": 0.75 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 0.5, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 0.5, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 0.5, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 0.25, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 0.25, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "vizdoom-basic-v1": { + "count": 73, + "hard_count": 73, + "accuracy": 0.9726027397260274, + "expected_accuracy": 0.9726027397260274, + "nll": 0.09228000248495685, + "brier": 0.05373730032476735, + "expected_brier": 0.053737300324767354, + "multiclass_ece": 0.03817257735589474, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 73, + "coverage": 1.0, + "accuracy": 0.9726027397260274, + "expected_accuracy": 0.9726027397260274, + "expected_risk": 0.0273972602739726 + }, + { + "threshold": 0.7, + "selected": 71, + "coverage": 0.9726027397260274, + "accuracy": 0.971830985915493, + "expected_accuracy": 0.971830985915493, + "expected_risk": 0.028169014084507005 + }, + { + "threshold": 0.8, + "selected": 71, + "coverage": 0.9726027397260274, + "accuracy": 0.971830985915493, + "expected_accuracy": 0.971830985915493, + "expected_risk": 0.028169014084507005 + }, + { + "threshold": 0.9, + "selected": 71, + "coverage": 0.9726027397260274, + "accuracy": 0.971830985915493, + "expected_accuracy": 0.971830985915493, + "expected_risk": 0.028169014084507005 + }, + { + "threshold": 0.95, + "selected": 69, + "coverage": 0.9452054794520548, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 62, + "coverage": 0.8493150684931506, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "wikispeedia-v1": { + "count": 24, + "hard_count": 8, + "accuracy": 0.125, + "expected_accuracy": 0.23315145502645504, + "nll": 2.0710662644850317, + "brier": 0.36865062334177034, + "expected_brier": 0.8438325016486488, + "multiclass_ece": 0.05961736540510668, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ], + "optimal_action_hit": 0.7083333333333334, + "optimal_set_probability_mass": 0.4594821286799471 + }, + "workflow-controls-v1/agent_trace_observability": { + "count": 24, + "hard_count": 24, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.014364802694363044, + "brier": 0.006834675556245828, + "expected_brier": 0.0068346755562457935, + "multiclass_ece": 0.012238795904922157, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 23, + "coverage": 0.9583333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 23, + "coverage": 0.9583333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 23, + "coverage": 0.9583333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 23, + "coverage": 0.9583333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service": { + "count": 24, + "hard_count": 24, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0010430486947276827, + "brier": 8.844365007948566e-06, + "expected_brier": 8.844365007957636e-06, + "multiclass_ece": 0.0010408288172536473, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing": { + "count": 24, + "hard_count": 24, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00259726316558385, + "brier": 0.00012995654445749162, + "expected_brier": 0.00012995654445747498, + "multiclass_ece": 0.002564078079094756, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 22, + "coverage": 0.9166666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents": { + "count": 25, + "hard_count": 25, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00029419571026418733, + "brier": 2.6010495516973165e-06, + "expected_brier": 2.6010495516715083e-06, + "multiclass_ece": 0.0002935430427758945, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 25, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 25, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 25, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 25, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 25, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 25, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "by_question": { + "customer-control-v1/bug_severity": { + "count": 8, + "hard_count": 8, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 6.199653210544953e-05, + "brier": 2.9605578278342718e-08, + "expected_brier": 2.9605578266700405e-08, + "multiclass_ece": 6.198897087472677e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 8, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 8, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 8, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 8, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 8, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 8, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "customer-control-v1/category": { + "count": 24, + "hard_count": 23, + "accuracy": 1.0, + "expected_accuracy": 0.9791666666666666, + "nll": 0.0291172734407852, + "brier": 7.045966303887269e-05, + "expected_brier": 0.02090379299637221, + "multiclass_ece": 0.0011622807497648213, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 0.9791666666666666, + "expected_risk": 0.02083333333333337 + }, + { + "threshold": 0.7, + "selected": 23, + "coverage": 0.9583333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 23, + "coverage": 0.9583333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 23, + "coverage": 0.9583333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 23, + "coverage": 0.9583333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 23, + "coverage": 0.9583333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "customer-control-v1/churn_likelihood_level": { + "count": 7, + "hard_count": 7, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00014877869038393545, + "brier": 1.8542726250463026e-07, + "expected_brier": 1.8542726252869711e-07, + "multiclass_ece": 0.0001487317391912768, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 7, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 7, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 7, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 7, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 7, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 7, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "customer-control-v1/frustration": { + "count": 9, + "hard_count": 6, + "accuracy": 1.0, + "expected_accuracy": 0.8333333333333334, + "nll": 0.23367856541572285, + "brier": 0.0013571205909060885, + "expected_brier": 0.16802378725757275, + "multiclass_ece": 0.015645855648804583, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 9, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.7, + "selected": 6, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 6, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 6, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 6, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 6, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "customer-control-v1/has_reproducible_steps": { + "count": 14, + "hard_count": 14, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0006301844506309475, + "brier": 8.536297123656253e-06, + "expected_brier": 8.53629712365451e-06, + "multiclass_ece": 0.0006280393429103677, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 14, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 14, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 14, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 14, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 14, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 14, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "customer-control-v1/refund_requested": { + "count": 10, + "hard_count": 10, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.555332193708772e-05, + "brier": 1.6920790916120438e-09, + "expected_brier": 1.692079099058219e-09, + "multiclass_ece": 2.555289890509016e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 10, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 10, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 10, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 10, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 10, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 10, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x0_y0_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.172645969326485e-06, + "brier": 8.899991485011589e-12, + "expected_brier": 8.899991854605105e-12, + "multiclass_ece": 2.1726436091329404e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x0_y1_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 5.997391723118475e-07, + "brier": 7.007190543651158e-13, + "expected_brier": 7.007727731433988e-13, + "multiclass_ece": 5.997389924683461e-07, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x0_y2": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 4.122413714119248e-07, + "brier": 2.6356348134345867e-13, + "expected_brier": 2.6356694604601216e-13, + "multiclass_ece": 4.122412864404623e-07, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x0_y2_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.7447120758604634e-05, + "brier": 1.4874863595861668e-09, + "expected_brier": 1.4874863563107965e-09, + "multiclass_ece": 2.744674408983183e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x0_y2_red": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 1.0868778103970906, + "brier": 0.8784277634420016, + "expected_brier": 0.8784277634420015, + "multiclass_ece": 0.6627321342148732, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y3_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00020358119004006486, + "brier": 8.287372891272755e-08, + "expected_brier": 8.28737289904069e-08, + "multiclass_ece": 0.0002035604687957715, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x0_y3_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 3.495297036462284e-05, + "brier": 2.4194290473114375e-09, + "expected_brier": 2.4194291015078306e-09, + "multiclass_ece": 3.4952359516671194e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x0_y5": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.1537510556153884e-05, + "brier": 9.277087411987397e-10, + "expected_brier": 9.277087986703236e-10, + "multiclass_ece": 2.153727862563848e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x0_y5_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.8757608621617665e-07, + "brier": 1.6020237260998597e-13, + "expected_brier": 1.6009416015094757e-13, + "multiclass_ece": 2.8757604486617794e-07, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x0_y6_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 5.438757681243895e-09, + "brier": 4.439987659392788e-17, + "expected_brier": 0.0, + "multiclass_ece": 5.438757666453853e-09, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x0_y7_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.2923930680384593e-06, + "brier": 3.340555367565526e-12, + "expected_brier": 3.340661081097096e-12, + "multiclass_ece": 1.292392232898898e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x0_y7_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0002013357480518198, + "brier": 8.048293224757489e-08, + "expected_brier": 8.048293231865955e-08, + "multiclass_ece": 0.00020131548137025668, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y0": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 3.250921560123206e-05, + "brier": 2.0554698940411572e-09, + "expected_brier": 2.055469794370879e-09, + "multiclass_ece": 3.2508687182408735e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y0_lightness": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.003913540774552931, + "brier": 5.6571255055068765e-05, + "expected_brier": 5.657125505509342e-05, + "multiclass_ece": 0.0038985191257032703, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y1_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.7294819204254149e-06, + "brier": 5.982205079906878e-12, + "expected_brier": 5.9821037012852685e-12, + "multiclass_ece": 1.7294804248724205e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y2": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.5267893711774167e-05, + "brier": 5.867869138810663e-10, + "expected_brier": 5.867869153538171e-10, + "multiclass_ece": 1.5267710929152873e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y4_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 4.0487748751711986e-08, + "brier": 2.1267037620957803e-15, + "expected_brier": 2.220446049250313e-15, + "multiclass_ece": 4.0487747932083096e-08, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y4_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 3.619463090891394e-05, + "brier": 2.587250760421574e-09, + "expected_brier": 2.587250857999379e-09, + "multiclass_ece": 3.6193975891163355e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y5_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.578665949053186e-05, + "brier": 4.909302152492981e-10, + "expected_brier": 4.909301853928127e-10, + "multiclass_ece": 1.5786534881878644e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y6_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 8.796354288027967e-05, + "brier": 1.1506936751367964e-08, + "expected_brier": 1.1506936736793705e-08, + "multiclass_ece": 8.79596742012767e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y6_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 3.0798956765743017e-09, + "brier": 1.426747003677936e-17, + "expected_brier": 0.0, + "multiclass_ece": 3.079895671831423e-09, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y0": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.4283115010640545e-05, + "brier": 1.1793107116160716e-09, + "expected_brier": 1.1793106491353456e-09, + "multiclass_ece": 2.4282820178189723e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y0_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.567332602950668e-06, + "brier": 3.8088100077987556e-12, + "expected_brier": 3.808731108279062e-12, + "multiclass_ece": 1.5673313746855655e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y1": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.0663641848862547e-05, + "brier": 1.3270886433217273e-09, + "expected_brier": 1.3270886611849164e-09, + "multiclass_ece": 2.0663308431756455e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y1_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0010945772134051395, + "brier": 2.39350907528627e-06, + "expected_brier": 2.393509075382383e-06, + "multiclass_ece": 0.001093978382275984, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y3_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 7.461803129184872e-05, + "brier": 8.875082733397143e-09, + "expected_brier": 8.87508266878001e-09, + "multiclass_ece": 7.461524743579417e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y3_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.6849366775330644e-06, + "brier": 1.354402553043203e-11, + "expected_brier": 1.3544054766612135e-11, + "multiclass_ece": 2.6849330730938092e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y4_red": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 8.713842387566774e-05, + "brier": 1.5184886595945922e-08, + "expected_brier": 1.5184886681396392e-08, + "multiclass_ece": 8.713462743348277e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y5": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.2406545240997248e-05, + "brier": 1.0040840410073274e-09, + "expected_brier": 1.004084149158757e-09, + "multiclass_ece": 2.24062942162373e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y5_green": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00026866345709406473, + "brier": 1.4432159167550265e-07, + "expected_brier": 1.4432159167743208e-07, + "multiclass_ece": 0.00026862737023336525, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y6": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0033490329048770228, + "brier": 2.2357063701699375e-05, + "expected_brier": 2.2357063701683444e-05, + "multiclass_ece": 0.003343431149410714, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y6_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.4961356655033905e-07, + "brier": 9.775921293820471e-14, + "expected_brier": 9.769962616701378e-14, + "multiclass_ece": 2.4961353539687536e-07, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x3_y2_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00040319855387696573, + "brier": 3.250070832942238e-07, + "expected_brier": 3.250070832461205e-07, + "multiclass_ece": 0.000403117280263543, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x3_y4": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 6.289732766458931e-07, + "brier": 6.159757306288801e-13, + "expected_brier": 6.159517340620368e-13, + "multiclass_ece": 6.289730788422432e-07, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x3_y5_red": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 6.468663426137355e-05, + "brier": 8.368179980139693e-09, + "expected_brier": 8.36817992677652e-09, + "multiclass_ece": 6.468454212615882e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x3_y6_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 6.230862231555607e-08, + "brier": 5.5586618859120896e-15, + "expected_brier": 5.551115123125783e-15, + "multiclass_ece": 6.23086203743739e-08, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x3_y7_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 4.418175298433424e-06, + "brier": 2.9673674115297003e-11, + "expected_brier": 2.9673596912971334e-11, + "multiclass_ece": 4.418165538311314e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x4_y0_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 4.04183824640024e-06, + "brier": 3.2672780761803214e-11, + "expected_brier": 3.267275339169373e-11, + "multiclass_ece": 4.04183007818304e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x4_y0_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 3.584367043471579e-05, + "brier": 2.278347532106479e-09, + "expected_brier": 2.278347510653589e-09, + "multiclass_ece": 3.584302805803574e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x4_y2": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00036720039099975426, + "brier": 2.6491482774318946e-07, + "expected_brier": 2.649148278521807e-07, + "multiclass_ece": 0.00036713298118740134, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x4_y4_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0002245133541200558, + "brier": 1.0020142229469897e-07, + "expected_brier": 1.0020142227062934e-07, + "multiclass_ece": 0.00022448815288300672, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x4_y5_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 3.433247400913301e-08, + "brier": 1.5891354534873812e-15, + "expected_brier": 1.5543122344752192e-15, + "multiclass_ece": 3.4332473419773635e-08, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x4_y6_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00078302076887181, + "brier": 1.209378266957864e-06, + "expected_brier": 1.2093782668642916e-06, + "multiclass_ece": 0.0007827142881083882, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x4_y7_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.5751542465552473e-05, + "brier": 1.2638790715063345e-09, + "expected_brier": 1.2638790014563028e-09, + "multiclass_ece": 2.5751210897428933e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x5_y2": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0004489617053626598, + "brier": 3.6594206415957484e-07, + "expected_brier": 3.659420642065925e-07, + "multiclass_ece": 0.00044886093713714104, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x5_y2_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0014666196569893047, + "brier": 4.295642511736678e-06, + "expected_brier": 4.295642511698006e-06, + "multiclass_ece": 0.0014655446959640495, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x5_y3_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.4491098803368072e-05, + "brier": 4.1997780307983486e-10, + "expected_brier": 4.199778302904633e-10, + "multiclass_ece": 1.4490993807902974e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x5_y5_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 4.496982221131167e-08, + "brier": 3.854519300942247e-15, + "expected_brier": 3.774758283725532e-15, + "multiclass_ece": 4.4969821200169235e-08, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x5_y6_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 5.40061028147118e-06, + "brier": 5.833286779089424e-11, + "expected_brier": 5.833289407064512e-11, + "multiclass_ece": 5.400595698201727e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x5_y6_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00035732088503695036, + "brier": 2.5276840494192646e-07, + "expected_brier": 2.5276840487364893e-07, + "multiclass_ece": 0.00035725705353251147, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x5_y7_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00012340252205576488, + "brier": 2.3072985053658352e-08, + "expected_brier": 2.3072985078087527e-08, + "multiclass_ece": 0.0001233949082777297, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x5_y7_red": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 5.3885414776999595e-05, + "brier": 5.8069629337524085e-09, + "expected_brier": 5.806962910526181e-09, + "multiclass_ece": 5.388396298411369e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x5_y7_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.1045288450441843e-07, + "brier": 7.959613003627739e-14, + "expected_brier": 7.949196856316121e-14, + "multiclass_ece": 2.1045286235921168e-07, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x6_y1_red": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.0962215721908778e-05, + "brier": 8.788105539680209e-10, + "expected_brier": 8.788105798629431e-10, + "multiclass_ece": 2.096199601619997e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x6_y2_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.7684827710087534e-06, + "brier": 1.5328951268733674e-11, + "expected_brier": 1.5328849300999536e-11, + "multiclass_ece": 2.768478938763863e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x6_y2_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00017605525498589196, + "brier": 5.935117141158661e-08, + "expected_brier": 5.935117131095069e-08, + "multiclass_ece": 0.00017603975816893325, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x6_y2_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00030791544519671665, + "brier": 1.8820875854152686e-07, + "expected_brier": 1.8820875857805675e-07, + "multiclass_ece": 0.0003078680441013226, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x6_y4": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.4002662132052775e-06, + "brier": 5.197773219366958e-12, + "expected_brier": 5.197842156690058e-12, + "multiclass_ece": 1.4002644950217302e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x6_y5": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.6864144531310774e-07, + "brier": 1.0218314805523048e-13, + "expected_brier": 1.021405182655144e-13, + "multiclass_ece": 2.6864140922899793e-07, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x6_y6_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 4.252264286722616e-08, + "brier": 2.5021199916037035e-15, + "expected_brier": 2.4424906541753444e-15, + "multiclass_ece": 4.2522641963138597e-08, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x6_y6_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.334945010946983e-06, + "brier": 1.0022942608083749e-11, + "expected_brier": 1.0022871421710988e-11, + "multiclass_ece": 2.3349422849650026e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x7_y0_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 6.429791104109464e-05, + "brier": 8.232936576657277e-09, + "expected_brier": 8.232936554719572e-09, + "multiclass_ece": 6.429584397471544e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x7_y1_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00031717941852263404, + "brier": 2.0114176061005075e-07, + "expected_brier": 2.011417605807253e-07, + "multiclass_ece": 0.0003171291224486339, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x7_y2_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00030647871494350967, + "brier": 1.8780084111598052e-07, + "expected_brier": 1.8780084110048278e-07, + "multiclass_ece": 0.0003064317551396689, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x7_y5_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 3.3467651543929e-08, + "brier": 1.6166807677896766e-15, + "expected_brier": 1.5543122344752192e-15, + "multiclass_ece": 3.3467650983887154e-08, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x7_y5_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.396993459391031e-05, + "brier": 1.0305267699048919e-09, + "expected_brier": 1.0305267750254643e-09, + "multiclass_ece": 2.3969647317323428e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x7_y7": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.2466206017769536e-05, + "brier": 5.233948989790757e-10, + "expected_brier": 5.233948829896917e-10, + "multiclass_ece": 1.2466074781669612e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "reasoning-control-v1/decision": { + "count": 24, + "hard_count": 24, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "nll": 0.4117335718066246, + "brier": 0.216022779312096, + "expected_brier": 0.21602277931209599, + "multiclass_ece": 0.09340459767884833, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 21, + "coverage": 0.875, + "accuracy": 0.9047619047619048, + "expected_accuracy": 0.9047619047619048, + "expected_risk": 0.09523809523809523 + }, + { + "threshold": 0.7, + "selected": 18, + "coverage": 0.75, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 15, + "coverage": 0.625, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 14, + "coverage": 0.5833333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 13, + "coverage": 0.5416666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 11, + "coverage": 0.4583333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "reasoning-control-v1/ordinal": { + "count": 24, + "hard_count": 24, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "nll": 0.8380394442343312, + "brier": 0.4302992411240769, + "expected_brier": 0.43029924112407686, + "multiclass_ece": 0.18833864411527465, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 18, + "coverage": 0.75, + "accuracy": 0.7777777777777778, + "expected_accuracy": 0.7777777777777778, + "expected_risk": 0.2222222222222222 + }, + { + "threshold": 0.7, + "selected": 16, + "coverage": 0.6666666666666666, + "accuracy": 0.8125, + "expected_accuracy": 0.8125, + "expected_risk": 0.1875 + }, + { + "threshold": 0.8, + "selected": 16, + "coverage": 0.6666666666666666, + "accuracy": 0.8125, + "expected_accuracy": 0.8125, + "expected_risk": 0.1875 + }, + { + "threshold": 0.9, + "selected": 15, + "coverage": 0.625, + "accuracy": 0.8666666666666667, + "expected_accuracy": 0.8666666666666667, + "expected_risk": 0.1333333333333333 + }, + { + "threshold": 0.95, + "selected": 14, + "coverage": 0.5833333333333334, + "accuracy": 0.8571428571428571, + "expected_accuracy": 0.8571428571428571, + "expected_risk": 0.1428571428571429 + }, + { + "threshold": 0.99, + "selected": 8, + "coverage": 0.3333333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "reasoning-control-v1/verification": { + "count": 24, + "hard_count": 24, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "nll": 0.5094262035029631, + "brier": 0.259598890897617, + "expected_brier": 0.2595988908976169, + "multiclass_ece": 0.11341880374280262, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.7, + "selected": 22, + "coverage": 0.9166666666666666, + "accuracy": 0.8636363636363636, + "expected_accuracy": 0.8636363636363636, + "expected_risk": 0.13636363636363635 + }, + { + "threshold": 0.8, + "selected": 21, + "coverage": 0.875, + "accuracy": 0.9047619047619048, + "expected_accuracy": 0.9047619047619048, + "expected_risk": 0.09523809523809523 + }, + { + "threshold": 0.9, + "selected": 21, + "coverage": 0.875, + "accuracy": 0.9047619047619048, + "expected_accuracy": 0.9047619047619048, + "expected_risk": 0.09523809523809523 + }, + { + "threshold": 0.95, + "selected": 17, + "coverage": 0.7083333333333334, + "accuracy": 0.8823529411764706, + "expected_accuracy": 0.8823529411764706, + "expected_risk": 0.11764705882352944 + }, + { + "threshold": 0.99, + "selected": 9, + "coverage": 0.375, + "accuracy": 0.8888888888888888, + "expected_accuracy": 0.8888888888888888, + "expected_risk": 0.11111111111111116 + } + ] + }, + "snake-v1/choice": { + "count": 24, + "hard_count": 19, + "accuracy": 0.8947368421052632, + "expected_accuracy": 0.7986111111111112, + "nll": 0.5493327328174721, + "brier": 0.2065791483561824, + "expected_brier": 0.32463470391173793, + "multiclass_ece": 0.11748854113039371, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.8947368421052632, + "expected_accuracy": 0.7986111111111112, + "expected_risk": 0.20138888888888884 + }, + { + "threshold": 0.7, + "selected": 19, + "coverage": 0.7916666666666666, + "accuracy": 0.8823529411764706, + "expected_accuracy": 0.8245614035087719, + "expected_risk": 0.17543859649122806 + }, + { + "threshold": 0.8, + "selected": 17, + "coverage": 0.7083333333333334, + "accuracy": 0.9333333333333333, + "expected_accuracy": 0.8627450980392157, + "expected_risk": 0.13725490196078427 + }, + { + "threshold": 0.9, + "selected": 15, + "coverage": 0.625, + "accuracy": 0.9285714285714286, + "expected_accuracy": 0.888888888888889, + "expected_risk": 0.11111111111111105 + }, + { + "threshold": 0.95, + "selected": 12, + "coverage": 0.5, + "accuracy": 0.9166666666666666, + "expected_accuracy": 0.9166666666666666, + "expected_risk": 0.08333333333333337 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 0.125, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "snake-v1/noul": { + "count": 24, + "hard_count": 24, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "nll": 0.6332902524840243, + "brier": 0.3205743329824628, + "expected_brier": 0.3205743329824628, + "multiclass_ece": 0.16718811690657506, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.7, + "selected": 23, + "coverage": 0.9583333333333334, + "accuracy": 0.8260869565217391, + "expected_accuracy": 0.8260869565217391, + "expected_risk": 0.17391304347826086 + }, + { + "threshold": 0.8, + "selected": 23, + "coverage": 0.9583333333333334, + "accuracy": 0.8260869565217391, + "expected_accuracy": 0.8260869565217391, + "expected_risk": 0.17391304347826086 + }, + { + "threshold": 0.9, + "selected": 23, + "coverage": 0.9583333333333334, + "accuracy": 0.8260869565217391, + "expected_accuracy": 0.8260869565217391, + "expected_risk": 0.17391304347826086 + }, + { + "threshold": 0.95, + "selected": 21, + "coverage": 0.875, + "accuracy": 0.8571428571428571, + "expected_accuracy": 0.8571428571428571, + "expected_risk": 0.1428571428571429 + }, + { + "threshold": 0.99, + "selected": 15, + "coverage": 0.625, + "accuracy": 0.9333333333333333, + "expected_accuracy": 0.9333333333333333, + "expected_risk": 0.06666666666666665 + } + ] + }, + "tic_tac_toe-v1/choice": { + "count": 24, + "hard_count": 7, + "accuracy": 0.2857142857142857, + "expected_accuracy": 0.3323412698412699, + "nll": 1.2596593300800374, + "brier": 0.2679342073845446, + "expected_brier": 0.6855929375432749, + "multiclass_ece": 0.1396299186022388, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 0.20833333333333334, + "accuracy": 1.0, + "expected_accuracy": 0.7, + "expected_risk": 0.30000000000000004 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.041666666666666664, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 0.041666666666666664, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "tile_platformer-v1/choice": { + "count": 24, + "hard_count": 24, + "accuracy": 0.9583333333333334, + "expected_accuracy": 0.9583333333333334, + "nll": 0.3296336081760603, + "brier": 0.08316409006817983, + "expected_brier": 0.08316409006817983, + "multiclass_ece": 0.03909390729886505, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.9583333333333334, + "expected_accuracy": 0.9583333333333334, + "expected_risk": 0.04166666666666663 + }, + { + "threshold": 0.7, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.9583333333333334, + "expected_accuracy": 0.9583333333333334, + "expected_risk": 0.04166666666666663 + }, + { + "threshold": 0.8, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.9583333333333334, + "expected_accuracy": 0.9583333333333334, + "expected_risk": 0.04166666666666663 + }, + { + "threshold": 0.9, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.9583333333333334, + "expected_accuracy": 0.9583333333333334, + "expected_risk": 0.04166666666666663 + }, + { + "threshold": 0.95, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.9583333333333334, + "expected_accuracy": 0.9583333333333334, + "expected_risk": 0.04166666666666663 + }, + { + "threshold": 0.99, + "selected": 23, + "coverage": 0.9583333333333334, + "accuracy": 0.9565217391304348, + "expected_accuracy": 0.9565217391304348, + "expected_risk": 0.04347826086956519 + } + ] + }, + "trex_runner-v1/choice": { + "count": 4, + "hard_count": 4, + "accuracy": 0.25, + "expected_accuracy": 0.25, + "nll": 1.7164753177424987, + "brier": 0.9991415297201487, + "expected_brier": 0.9991415297201485, + "multiclass_ece": 0.5572777459998814, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.25, + "expected_accuracy": 0.25, + "expected_risk": 0.75 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 0.5, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 0.5, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 0.5, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 0.25, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 0.25, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "vizdoom-basic-v1/choice": { + "count": 24, + "hard_count": 24, + "accuracy": 0.9166666666666666, + "expected_accuracy": 0.9166666666666666, + "nll": 0.27769600149101703, + "brier": 0.16338497548521436, + "expected_brier": 0.16338497548521438, + "multiclass_ece": 0.1131358966780407, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.9166666666666666, + "expected_accuracy": 0.9166666666666666, + "expected_risk": 0.08333333333333337 + }, + { + "threshold": 0.7, + "selected": 22, + "coverage": 0.9166666666666666, + "accuracy": 0.9090909090909091, + "expected_accuracy": 0.9090909090909091, + "expected_risk": 0.09090909090909094 + }, + { + "threshold": 0.8, + "selected": 22, + "coverage": 0.9166666666666666, + "accuracy": 0.9090909090909091, + "expected_accuracy": 0.9090909090909091, + "expected_risk": 0.09090909090909094 + }, + { + "threshold": 0.9, + "selected": 22, + "coverage": 0.9166666666666666, + "accuracy": 0.9090909090909091, + "expected_accuracy": 0.9090909090909091, + "expected_risk": 0.09090909090909094 + }, + { + "threshold": 0.95, + "selected": 20, + "coverage": 0.8333333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 14, + "coverage": 0.5833333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "vizdoom-basic-v1/noul": { + "count": 25, + "hard_count": 25, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0028058285790131376, + "brier": 6.332255938444656e-05, + "expected_brier": 6.332255938445109e-05, + "multiclass_ece": 0.0027898543937202192, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 25, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 25, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 25, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 25, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 25, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 24, + "coverage": 0.96, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "vizdoom-basic-v1/score": { + "count": 24, + "hard_count": 24, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 6.626796425470535e-05, + "brier": 1.866992753124178e-08, + "expected_brier": 1.8669927502138844e-08, + "multiclass_ece": 6.62611193478213e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "wikispeedia-v1/choice": { + "count": 24, + "hard_count": 8, + "accuracy": 0.125, + "expected_accuracy": 0.23315145502645504, + "nll": 2.0710662644850317, + "brier": 0.36865062334177034, + "expected_brier": 0.8438325016486488, + "multiclass_ece": 0.05961736540510668, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/AUTO-CLOSE": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 3.3061158495888886e-05, + "brier": 2.5649928240516783e-09, + "expected_brier": 2.5649928107422206e-09, + "multiclass_ece": 3.306051722917758e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/agent_trace_observability/COUNT ONLY": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.612635862505493e-05, + "brier": 6.000466690123967e-10, + "expected_brier": 6.000466790112569e-10, + "multiclass_ece": 1.6126208611281594e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/agent_trace_observability/FILE ISSUE ยท ROUTE": { + "count": 5, + "hard_count": 5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 3.0099669276621256e-05, + "brier": 2.0189609701582924e-09, + "expected_brier": 2.0189609095666582e-09, + "multiclass_ece": 3.0099164523988974e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/agent_trace_observability/HUMAN REVIEW": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 3.30949954112954e-05, + "brier": 2.542965841753953e-09, + "expected_brier": 2.54296578239277e-09, + "multiclass_ece": 3.309435965159757e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/agent_trace_observability/NOT A BUG": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 7.137922235118342e-06, + "brier": 1.0189914031891241e-10, + "expected_brier": 1.0189915577996089e-10, + "multiclass_ece": 7.137896760212037e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/agent_trace_observability/PAGE ON-CALL": { + "count": 5, + "hard_count": 5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0010028512098091205, + "brier": 3.210327284943542e-06, + "expected_brier": 3.2103272848971187e-06, + "multiclass_ece": 0.0010020476920717058, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/agent_trace_observability/PRIORITY REVIEW": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.11310114327134711, + "brier": 0.05467204394992977, + "expected_brier": 0.05467204394992975, + "multiclass_ece": 0.09609443064075485, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service/CANCEL": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.416957613974782e-05, + "brier": 1.1683658243451508e-09, + "expected_brier": 1.1683658485139858e-09, + "multiclass_ece": 2.416928404358476e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service/CLOSE": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0003714054742858682, + "brier": 3.699679809711367e-07, + "expected_brier": 3.69967980917707e-07, + "multiclass_ece": 0.0003713129501251207, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service/FLAG FOR REVIEW": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0016512031231536298, + "brier": 8.465722777314143e-06, + "expected_brier": 8.465722777388033e-06, + "multiclass_ece": 0.00164908291303556, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service/FREEZE CARD": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.209544867124111e-05, + "brier": 2.9271209530292997e-10, + "expected_brier": 2.927120767992619e-10, + "multiclass_ece": 1.209537549258973e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service/HAND OFF": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00035433206484843886, + "brier": 2.5101346910643695e-07, + "expected_brier": 2.51013469121375e-07, + "multiclass_ece": 0.00035426929665616225, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service/REFUND": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.4701714060385343e-05, + "brier": 4.326585988760313e-10, + "expected_brier": 4.326586496479952e-10, + "multiclass_ece": 1.4701605894747635e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service/SAY": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0006365462341670471, + "brier": 8.233487487517735e-07, + "expected_brier": 8.23348748735242e-07, + "multiclass_ece": 0.0006363403069015172, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service/SET INTENT": { + "count": 5, + "hard_count": 5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0031872190478309777, + "brier": 3.6441753321612995e-05, + "expected_brier": 3.644175332162547e-05, + "multiclass_ece": 0.0031780687976182163, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/DISPUTE LINES": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 7.12131832384304e-06, + "brier": 1.0142562705380522e-10, + "expected_brier": 1.014255346376558e-10, + "multiclass_ece": 7.121292967315895e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/DUPLICATE": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 9.52803871027539e-06, + "brier": 1.8156531336012347e-10, + "expected_brier": 1.8156520731338333e-10, + "multiclass_ece": 9.527993318658723e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/FRAUD REVIEW": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.2227328759287392e-05, + "brier": 3.2307575340009005e-10, + "expected_brier": 3.23075732833189e-10, + "multiclass_ece": 1.2227247989549106e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/HOLD": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.002182734896011967, + "brier": 1.1747670688790077e-05, + "expected_brier": 1.174767068878868e-05, + "multiclass_ece": 0.0021797920008571614, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/HOLD FOR DOCUMENTS": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.0225755917156143e-05, + "brier": 2.091300296309827e-10, + "expected_brier": 2.0913004661338164e-10, + "multiclass_ece": 1.0225703634292316e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/PAY": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.011972573392541966, + "brier": 0.0008272924425458614, + "expected_brier": 0.0008272924425457943, + "multiclass_ece": 0.01176076107336869, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/REQUEST CORRECTED INVOICE": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.9564490052285583e-05, + "brier": 1.215418969447865e-09, + "expected_brier": 1.2154188766544394e-09, + "multiclass_ece": 1.9564186190645927e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/ROUTE FOR APPROVAL": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.004351510209608228, + "brier": 0.00014751736756324663, + "expected_brier": 0.00014751736756329903, + "multiclass_ece": 0.004314203049391785, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 0.75, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/SCHEDULE": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 4.72262385464682e-05, + "brier": 4.477377194651059e-09, + "expected_brier": 4.47737724673658e-09, + "multiclass_ece": 4.722511916666061e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/SHORT PAY": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 3.512474435988826e-05, + "brier": 3.2081588781524873e-09, + "expected_brier": 3.2081588496879476e-09, + "multiclass_ece": 3.512394229265858e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/BLOCK EGRESS ASSET": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 3.073346233461575e-05, + "brier": 2.15800599934032e-09, + "expected_brier": 2.158005922107501e-09, + "multiclass_ece": 3.073292281966881e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/BLOCK SENDER": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 5.346894175932663e-05, + "brier": 5.717549747585692e-09, + "expected_brier": 5.7175497669703645e-09, + "multiclass_ece": 5.346751232093716e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/DISABLE ACCOUNT": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.4706689360782918e-05, + "brier": 1.5172053838374415e-09, + "expected_brier": 1.517205361345475e-09, + "multiclass_ece": 2.4706310050692792e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/ESCALATE TIER2": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 3.8705570998190654e-05, + "brier": 3.0350599142119585e-09, + "expected_brier": 3.0350598700934484e-09, + "multiclass_ece": 3.87048122131306e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/ESCALATE URGENT": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.001911102011844048, + "brier": 2.118151457650642e-05, + "expected_brier": 2.118151457652928e-05, + "multiclass_ece": 0.0019057866539748207, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/ISOLATE HOST": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 3.423173629149177e-05, + "brier": 2.93964677075743e-09, + "expected_brier": 2.93964674824565e-09, + "multiclass_ece": 3.4231001357021285e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/KILL PROCESS": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 4.4778050062942524e-05, + "brier": 4.009967972977979e-09, + "expected_brier": 4.009967913276569e-09, + "multiclass_ece": 4.477704754102252e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/NOTIFY USER": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.6474352949827146e-05, + "brier": 6.087855566618374e-10, + "expected_brier": 6.087855775049889e-10, + "multiclass_ece": 1.6474200751348533e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/QUARANTINE FILE": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.5313867169494252e-05, + "brier": 1.2815513007905828e-09, + "expected_brier": 1.2815513095176811e-09, + "multiclass_ece": 2.5313546776262186e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/QUARANTINE MESSAGE": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00042206810060792726, + "brier": 6.80308277800062e-07, + "expected_brier": 6.803082777118163e-07, + "multiclass_ece": 0.00042189793003499787, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/REMOVE FORWARDING RULES": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0001244280209527456, + "brier": 4.124218351123477e-08, + "expected_brier": 4.1242183446144054e-08, + "multiclass_ece": 0.00012441770912507977, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/REVOKE ACCESS KEY": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.208019775326709e-05, + "brier": 2.9835298898328143e-10, + "expected_brier": 2.983530089650799e-10, + "multiclass_ece": 1.2080123164448509e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/REVOKE SESSIONS": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.620440095697275e-05, + "brier": 1.3733052719917098e-09, + "expected_brier": 1.3733052472986174e-09, + "multiclass_ece": 2.620405762465694e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + } + }, + "trained_ood": { + "count": 512, + "hard_count": 484, + "accuracy": 0.878099173553719, + "expected_accuracy": 0.8438422309027778, + "nll": 0.6851368983769346, + "brier": 0.20787976349730586, + "expected_brier": 0.245092220094528, + "multiclass_ece": 0.09432452243203898, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 472, + "coverage": 0.921875, + "accuracy": 0.896328293736501, + "expected_accuracy": 0.8884180790960453, + "expected_risk": 0.11158192090395469 + }, + { + "threshold": 0.7, + "selected": 455, + "coverage": 0.888671875, + "accuracy": 0.9008810572687225, + "expected_accuracy": 0.9, + "expected_risk": 0.09999999999999998 + }, + { + "threshold": 0.8, + "selected": 452, + "coverage": 0.8828125, + "accuracy": 0.9048672566371682, + "expected_accuracy": 0.9048672566371682, + "expected_risk": 0.09513274336283184 + }, + { + "threshold": 0.9, + "selected": 437, + "coverage": 0.853515625, + "accuracy": 0.9221967963386728, + "expected_accuracy": 0.9221967963386728, + "expected_risk": 0.0778032036613272 + }, + { + "threshold": 0.95, + "selected": 433, + "coverage": 0.845703125, + "accuracy": 0.9260969976905312, + "expected_accuracy": 0.9260969976905312, + "expected_risk": 0.07390300230946878 + }, + { + "threshold": 0.99, + "selected": 385, + "coverage": 0.751953125, + "accuracy": 0.9428571428571428, + "expected_accuracy": 0.9428571428571428, + "expected_risk": 0.05714285714285716 + } + ], + "mean_latency_seconds": 0.038995993836579146, + "by_kind": { + "choice": { + "count": 162, + "hard_count": 136, + "accuracy": 0.8235294117647058, + "expected_accuracy": 0.7307441700960221, + "nll": 0.9849819306657265, + "brier": 0.2614185858768291, + "expected_brier": 0.37079787257093055, + "multiclass_ece": 0.11288227925869491, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 129, + "coverage": 0.7962962962962963, + "accuracy": 0.8760330578512396, + "expected_accuracy": 0.8527131782945736, + "expected_risk": 0.1472868217054264 + }, + { + "threshold": 0.7, + "selected": 118, + "coverage": 0.7283950617283951, + "accuracy": 0.8803418803418803, + "expected_accuracy": 0.8771186440677966, + "expected_risk": 0.1228813559322034 + }, + { + "threshold": 0.8, + "selected": 116, + "coverage": 0.7160493827160493, + "accuracy": 0.8879310344827587, + "expected_accuracy": 0.8879310344827587, + "expected_risk": 0.11206896551724133 + }, + { + "threshold": 0.9, + "selected": 108, + "coverage": 0.6666666666666666, + "accuracy": 0.9259259259259259, + "expected_accuracy": 0.9259259259259259, + "expected_risk": 0.07407407407407407 + }, + { + "threshold": 0.95, + "selected": 107, + "coverage": 0.6604938271604939, + "accuracy": 0.9252336448598131, + "expected_accuracy": 0.9252336448598131, + "expected_risk": 0.07476635514018692 + }, + { + "threshold": 0.99, + "selected": 92, + "coverage": 0.5679012345679012, + "accuracy": 0.9347826086956522, + "expected_accuracy": 0.9347826086956522, + "expected_risk": 0.06521739130434778 + } + ] + }, + "noul": { + "count": 243, + "hard_count": 243, + "accuracy": 0.8930041152263375, + "expected_accuracy": 0.8930041152263375, + "nll": 0.5224303328418759, + "brier": 0.19139402461237634, + "expected_brier": 0.1913940246123764, + "multiclass_ece": 0.09528500549398275, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 243, + "coverage": 1.0, + "accuracy": 0.8930041152263375, + "expected_accuracy": 0.8930041152263375, + "expected_risk": 0.10699588477366251 + }, + { + "threshold": 0.7, + "selected": 240, + "coverage": 0.9876543209876543, + "accuracy": 0.9, + "expected_accuracy": 0.9, + "expected_risk": 0.09999999999999998 + }, + { + "threshold": 0.8, + "selected": 239, + "coverage": 0.9835390946502057, + "accuracy": 0.9037656903765691, + "expected_accuracy": 0.9037656903765691, + "expected_risk": 0.09623430962343094 + }, + { + "threshold": 0.9, + "selected": 233, + "coverage": 0.9588477366255144, + "accuracy": 0.9141630901287554, + "expected_accuracy": 0.9141630901287554, + "expected_risk": 0.0858369098712446 + }, + { + "threshold": 0.95, + "selected": 231, + "coverage": 0.9506172839506173, + "accuracy": 0.922077922077922, + "expected_accuracy": 0.922077922077922, + "expected_risk": 0.07792207792207795 + }, + { + "threshold": 0.99, + "selected": 216, + "coverage": 0.8888888888888888, + "accuracy": 0.9398148148148148, + "expected_accuracy": 0.9398148148148148, + "expected_risk": 0.06018518518518523 + } + ] + }, + "score": { + "count": 107, + "hard_count": 105, + "accuracy": 0.9142857142857143, + "expected_accuracy": 0.9034267912772586, + "nll": 0.6006770871081025, + "brier": 0.16426056091370858, + "expected_brier": 0.17672162010373968, + "multiclass_ece": 0.09040818476919783, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 100, + "coverage": 0.9345794392523364, + "accuracy": 0.9292929292929293, + "expected_accuracy": 0.9233333333333335, + "expected_risk": 0.07666666666666655 + }, + { + "threshold": 0.7, + "selected": 97, + "coverage": 0.9065420560747663, + "accuracy": 0.9278350515463918, + "expected_accuracy": 0.9278350515463918, + "expected_risk": 0.07216494845360821 + }, + { + "threshold": 0.8, + "selected": 97, + "coverage": 0.9065420560747663, + "accuracy": 0.9278350515463918, + "expected_accuracy": 0.9278350515463918, + "expected_risk": 0.07216494845360821 + }, + { + "threshold": 0.9, + "selected": 96, + "coverage": 0.897196261682243, + "accuracy": 0.9375, + "expected_accuracy": 0.9375, + "expected_risk": 0.0625 + }, + { + "threshold": 0.95, + "selected": 95, + "coverage": 0.8878504672897196, + "accuracy": 0.9368421052631579, + "expected_accuracy": 0.9368421052631579, + "expected_risk": 0.06315789473684208 + }, + { + "threshold": 0.99, + "selected": 77, + "coverage": 0.719626168224299, + "accuracy": 0.961038961038961, + "expected_accuracy": 0.961038961038961, + "expected_risk": 0.038961038961038974 + } + ], + "ordinal_mae": 0.15554934363735728 + } + }, + "by_source": { + "customer-control-v1": { + "count": 81, + "hard_count": 77, + "accuracy": 0.922077922077922, + "expected_accuracy": 0.8971193415637859, + "nll": 0.5110543030088744, + "brier": 0.15218995336316685, + "expected_brier": 0.18099653772530677, + "multiclass_ece": 0.08477253301543, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 80, + "coverage": 0.9876543209876543, + "accuracy": 0.922077922077922, + "expected_accuracy": 0.9041666666666668, + "expected_risk": 0.09583333333333321 + }, + { + "threshold": 0.7, + "selected": 76, + "coverage": 0.9382716049382716, + "accuracy": 0.9210526315789473, + "expected_accuracy": 0.9210526315789473, + "expected_risk": 0.07894736842105265 + }, + { + "threshold": 0.8, + "selected": 76, + "coverage": 0.9382716049382716, + "accuracy": 0.9210526315789473, + "expected_accuracy": 0.9210526315789473, + "expected_risk": 0.07894736842105265 + }, + { + "threshold": 0.9, + "selected": 76, + "coverage": 0.9382716049382716, + "accuracy": 0.9210526315789473, + "expected_accuracy": 0.9210526315789473, + "expected_risk": 0.07894736842105265 + }, + { + "threshold": 0.95, + "selected": 76, + "coverage": 0.9382716049382716, + "accuracy": 0.9210526315789473, + "expected_accuracy": 0.9210526315789473, + "expected_risk": 0.07894736842105265 + }, + { + "threshold": 0.99, + "selected": 73, + "coverage": 0.9012345679012346, + "accuracy": 0.9452054794520548, + "expected_accuracy": 0.9452054794520548, + "expected_risk": 0.0547945205479452 + } + ] + }, + "painting-geometry-v1": { + "count": 81, + "hard_count": 81, + "accuracy": 0.8765432098765432, + "expected_accuracy": 0.8765432098765432, + "nll": 1.2951006539174603, + "brier": 0.24579289801169704, + "expected_brier": 0.24579289801169704, + "multiclass_ece": 0.12086731715031661, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 81, + "coverage": 1.0, + "accuracy": 0.8765432098765432, + "expected_accuracy": 0.8765432098765432, + "expected_risk": 0.12345679012345678 + }, + { + "threshold": 0.7, + "selected": 81, + "coverage": 1.0, + "accuracy": 0.8765432098765432, + "expected_accuracy": 0.8765432098765432, + "expected_risk": 0.12345679012345678 + }, + { + "threshold": 0.8, + "selected": 81, + "coverage": 1.0, + "accuracy": 0.8765432098765432, + "expected_accuracy": 0.8765432098765432, + "expected_risk": 0.12345679012345678 + }, + { + "threshold": 0.9, + "selected": 81, + "coverage": 1.0, + "accuracy": 0.8765432098765432, + "expected_accuracy": 0.8765432098765432, + "expected_risk": 0.12345679012345678 + }, + { + "threshold": 0.95, + "selected": 81, + "coverage": 1.0, + "accuracy": 0.8765432098765432, + "expected_accuracy": 0.8765432098765432, + "expected_risk": 0.12345679012345678 + }, + { + "threshold": 0.99, + "selected": 73, + "coverage": 0.9012345679012346, + "accuracy": 0.8767123287671232, + "expected_accuracy": 0.8767123287671232, + "expected_risk": 0.12328767123287676 + } + ] + }, + "reasoning-control-v1": { + "count": 80, + "hard_count": 80, + "accuracy": 0.675, + "expected_accuracy": 0.675, + "nll": 1.0863388639046936, + "brier": 0.5188036932640236, + "expected_brier": 0.5188036932640236, + "multiclass_ece": 0.2362943150702972, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 68, + "coverage": 0.85, + "accuracy": 0.7205882352941176, + "expected_accuracy": 0.7205882352941176, + "expected_risk": 0.27941176470588236 + }, + { + "threshold": 0.7, + "selected": 62, + "coverage": 0.775, + "accuracy": 0.7419354838709677, + "expected_accuracy": 0.7419354838709677, + "expected_risk": 0.25806451612903225 + }, + { + "threshold": 0.8, + "selected": 62, + "coverage": 0.775, + "accuracy": 0.7419354838709677, + "expected_accuracy": 0.7419354838709677, + "expected_risk": 0.25806451612903225 + }, + { + "threshold": 0.9, + "selected": 52, + "coverage": 0.65, + "accuracy": 0.7884615384615384, + "expected_accuracy": 0.7884615384615384, + "expected_risk": 0.21153846153846156 + }, + { + "threshold": 0.95, + "selected": 50, + "coverage": 0.625, + "accuracy": 0.82, + "expected_accuracy": 0.82, + "expected_risk": 0.18000000000000005 + }, + { + "threshold": 0.99, + "selected": 36, + "coverage": 0.45, + "accuracy": 0.8888888888888888, + "expected_accuracy": 0.8888888888888888, + "expected_risk": 0.11111111111111116 + } + ] + }, + "snake-v1": { + "count": 54, + "hard_count": 48, + "accuracy": 0.9166666666666666, + "expected_accuracy": 0.8703703703703703, + "nll": 0.401903655230354, + "brier": 0.14144024274194222, + "expected_brier": 0.19699579829749775, + "multiclass_ece": 0.07012314918009387, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 54, + "coverage": 1.0, + "accuracy": 0.9166666666666666, + "expected_accuracy": 0.8703703703703703, + "expected_risk": 0.12962962962962965 + }, + { + "threshold": 0.7, + "selected": 48, + "coverage": 0.8888888888888888, + "accuracy": 0.9148936170212766, + "expected_accuracy": 0.90625, + "expected_risk": 0.09375 + }, + { + "threshold": 0.8, + "selected": 46, + "coverage": 0.8518518518518519, + "accuracy": 0.9347826086956522, + "expected_accuracy": 0.9347826086956522, + "expected_risk": 0.06521739130434778 + }, + { + "threshold": 0.9, + "selected": 46, + "coverage": 0.8518518518518519, + "accuracy": 0.9347826086956522, + "expected_accuracy": 0.9347826086956522, + "expected_risk": 0.06521739130434778 + }, + { + "threshold": 0.95, + "selected": 45, + "coverage": 0.8333333333333334, + "accuracy": 0.9333333333333333, + "expected_accuracy": 0.9333333333333333, + "expected_risk": 0.06666666666666665 + }, + { + "threshold": 0.99, + "selected": 31, + "coverage": 0.5740740740740741, + "accuracy": 0.9354838709677419, + "expected_accuracy": 0.9354838709677419, + "expected_risk": 0.06451612903225812 + } + ] + }, + "vizdoom-basic-v1": { + "count": 81, + "hard_count": 81, + "accuracy": 0.9135802469135802, + "expected_accuracy": 0.9135802469135802, + "nll": 0.339365796908814, + "brier": 0.1471597888107676, + "expected_brier": 0.14715978881076763, + "multiclass_ece": 0.07599148444066224, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 81, + "coverage": 1.0, + "accuracy": 0.9135802469135802, + "expected_accuracy": 0.9135802469135802, + "expected_risk": 0.0864197530864198 + }, + { + "threshold": 0.7, + "selected": 80, + "coverage": 0.9876543209876543, + "accuracy": 0.9125, + "expected_accuracy": 0.9125, + "expected_risk": 0.08750000000000002 + }, + { + "threshold": 0.8, + "selected": 79, + "coverage": 0.9753086419753086, + "accuracy": 0.9240506329113924, + "expected_accuracy": 0.9240506329113924, + "expected_risk": 0.07594936708860756 + }, + { + "threshold": 0.9, + "selected": 74, + "coverage": 0.9135802469135802, + "accuracy": 0.972972972972973, + "expected_accuracy": 0.972972972972973, + "expected_risk": 0.027027027027026973 + }, + { + "threshold": 0.95, + "selected": 73, + "coverage": 0.9012345679012346, + "accuracy": 0.9726027397260274, + "expected_accuracy": 0.9726027397260274, + "expected_risk": 0.0273972602739726 + }, + { + "threshold": 0.99, + "selected": 67, + "coverage": 0.8271604938271605, + "accuracy": 0.9850746268656716, + "expected_accuracy": 0.9850746268656716, + "expected_risk": 0.014925373134328401 + } + ] + }, + "wikispeedia-v1": { + "count": 27, + "hard_count": 9, + "accuracy": 0.5555555555555556, + "expected_accuracy": 0.27335390946502053, + "nll": 2.151915626932359, + "brier": 0.3401768692105326, + "expected_brier": 0.8483044412269933, + "multiclass_ece": 0.09307988515415551, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ], + "optimal_action_hit": 0.6666666666666666, + "optimal_set_probability_mass": 0.4304439939980443 + }, + "workflow-controls-v1/agent_trace_observability": { + "count": 27, + "hard_count": 27, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00030922194919647184, + "brier": 1.1262537342270023e-06, + "expected_brier": 1.1262537342162073e-06, + "multiclass_ece": 0.00030893993143739823, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service": { + "count": 27, + "hard_count": 27, + "accuracy": 0.9259259259259259, + "expected_accuracy": 0.9259259259259259, + "nll": 0.3801569512936742, + "brier": 0.14632888858035506, + "expected_brier": 0.14632888858035506, + "multiclass_ece": 0.06949383442001665, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.9259259259259259, + "expected_accuracy": 0.9259259259259259, + "expected_risk": 0.07407407407407407 + }, + { + "threshold": 0.7, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.9259259259259259, + "expected_accuracy": 0.9259259259259259, + "expected_risk": 0.07407407407407407 + }, + { + "threshold": 0.8, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.9259259259259259, + "expected_accuracy": 0.9259259259259259, + "expected_risk": 0.07407407407407407 + }, + { + "threshold": 0.9, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.9259259259259259, + "expected_accuracy": 0.9259259259259259, + "expected_risk": 0.07407407407407407 + }, + { + "threshold": 0.95, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.9259259259259259, + "expected_accuracy": 0.9259259259259259, + "expected_risk": 0.07407407407407407 + }, + { + "threshold": 0.99, + "selected": 24, + "coverage": 0.8888888888888888, + "accuracy": 0.9166666666666666, + "expected_accuracy": 0.9166666666666666, + "expected_risk": 0.08333333333333337 + } + ] + }, + "workflow-controls-v1/invoice_processing": { + "count": 27, + "hard_count": 27, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0005562845044390747, + "brier": 4.254491828483914e-06, + "expected_brier": 4.254491828499569e-06, + "multiclass_ece": 0.000555217441721033, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents": { + "count": 27, + "hard_count": 27, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00013615285844097386, + "brier": 5.831819810536594e-07, + "expected_brier": 5.83181981043905e-07, + "multiclass_ece": 0.00013600679227307764, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "by_question": { + "customer-control-v1/bug_severity": { + "count": 8, + "hard_count": 8, + "accuracy": 0.875, + "expected_accuracy": 0.875, + "nll": 0.7521520762251711, + "brier": 0.246438295667848, + "expected_brier": 0.24643829566784803, + "multiclass_ece": 0.1234803887049406, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 8, + "coverage": 1.0, + "accuracy": 0.875, + "expected_accuracy": 0.875, + "expected_risk": 0.125 + }, + { + "threshold": 0.7, + "selected": 8, + "coverage": 1.0, + "accuracy": 0.875, + "expected_accuracy": 0.875, + "expected_risk": 0.125 + }, + { + "threshold": 0.8, + "selected": 8, + "coverage": 1.0, + "accuracy": 0.875, + "expected_accuracy": 0.875, + "expected_risk": 0.125 + }, + { + "threshold": 0.9, + "selected": 8, + "coverage": 1.0, + "accuracy": 0.875, + "expected_accuracy": 0.875, + "expected_risk": 0.125 + }, + { + "threshold": 0.95, + "selected": 8, + "coverage": 1.0, + "accuracy": 0.875, + "expected_accuracy": 0.875, + "expected_risk": 0.125 + }, + { + "threshold": 0.99, + "selected": 7, + "coverage": 0.875, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "customer-control-v1/category": { + "count": 27, + "hard_count": 25, + "accuracy": 1.0, + "expected_accuracy": 0.9629629629629629, + "nll": 0.0543057087595555, + "brier": 0.0021126652468425556, + "expected_brier": 0.0391497022838796, + "multiclass_ece": 0.006885799378080685, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 0.9629629629629629, + "expected_risk": 0.03703703703703709 + }, + { + "threshold": 0.7, + "selected": 25, + "coverage": 0.9259259259259259, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 25, + "coverage": 0.9259259259259259, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 25, + "coverage": 0.9259259259259259, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 25, + "coverage": 0.9259259259259259, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 25, + "coverage": 0.9259259259259259, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "customer-control-v1/churn_likelihood_level": { + "count": 9, + "hard_count": 7, + "accuracy": 1.0, + "expected_accuracy": 0.8518518518518517, + "nll": 0.3278608219887238, + "brier": 0.04505423941532698, + "expected_brier": 0.19320238756347508, + "multiclass_ece": 0.08821772312843887, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 8, + "coverage": 0.8888888888888888, + "accuracy": 1.0, + "expected_accuracy": 0.9166666666666666, + "expected_risk": 0.08333333333333337 + }, + { + "threshold": 0.7, + "selected": 6, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 6, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 6, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 6, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 5, + "coverage": 0.5555555555555556, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "customer-control-v1/frustration": { + "count": 10, + "hard_count": 10, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.001040033729025758, + "brier": 4.296087644061276e-06, + "expected_brier": 4.296087644029001e-06, + "multiclass_ece": 0.0010389489134793096, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 10, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 10, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 10, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 10, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 10, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 10, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "customer-control-v1/has_reproducible_steps": { + "count": 15, + "hard_count": 15, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "nll": 0.9832731200211111, + "brier": 0.3930528258416484, + "expected_brier": 0.39305282584164847, + "multiclass_ece": 0.1982120841882067, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 15, + "coverage": 1.0, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.7, + "selected": 15, + "coverage": 1.0, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.8, + "selected": 15, + "coverage": 1.0, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.9, + "selected": 15, + "coverage": 1.0, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.95, + "selected": 15, + "coverage": 1.0, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.99, + "selected": 14, + "coverage": 0.9333333333333333, + "accuracy": 0.8571428571428571, + "expected_accuracy": 0.8571428571428571, + "expected_risk": 0.1428571428571429 + } + ] + }, + "customer-control-v1/refund_requested": { + "count": 12, + "hard_count": 12, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "nll": 1.3501402718253355, + "brier": 0.3331261993474894, + "expected_brier": 0.3331261993474895, + "multiclass_ece": 0.16659556877981085, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 12, + "coverage": 1.0, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.7, + "selected": 12, + "coverage": 1.0, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.8, + "selected": 12, + "coverage": 1.0, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.9, + "selected": 12, + "coverage": 1.0, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.95, + "selected": 12, + "coverage": 1.0, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.99, + "selected": 12, + "coverage": 1.0, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + } + ] + }, + "painting-geometry-v1/x0_y11_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.9522811393075507e-06, + "brier": 7.622788412243256e-12, + "expected_brier": 7.622791287076325e-12, + "multiclass_ece": 1.9522792336079675e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x0_y11_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.01467130302602269, + "brier": 0.00031391990692427305, + "expected_brier": 0.00031391990692419824, + "multiclass_ece": 0.014564203860945035, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y3_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.444421939178697e-06, + "brier": 4.172703449723489e-12, + "expected_brier": 4.172662215751188e-12, + "multiclass_ece": 1.44442089600183e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x0_y8": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00014670506177149334, + "brier": 4.3038435956358015e-08, + "expected_brier": 4.30384359262348e-08, + "multiclass_ece": 0.0001466943011101396, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x0_y8_red": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0002654663901933341, + "brier": 1.409073983284278e-07, + "expected_brier": 1.4090739841599031e-07, + "multiclass_ece": 0.00026543115710897514, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x10_y0_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0305295642815158, + "brier": 0.0012731712464618503, + "expected_brier": 0.0012731712464617573, + "multiclass_ece": 0.030068243692171093, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x10_y11": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 7.789887556374314e-06, + "brier": 8.571823592582906e-11, + "expected_brier": 8.571832132986401e-11, + "multiclass_ece": 7.789857215279028e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x10_y4_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 4.895666946846396e-06, + "brier": 4.793487503631005e-11, + "expected_brier": 4.7934767266610834e-11, + "multiclass_ece": 4.895654963088525e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x10_y4_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.4409963517500872e-05, + "brier": 3.968211812375814e-10, + "expected_brier": 3.9682124253204165e-10, + "multiclass_ece": 1.440985969447528e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x10_y5_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 15.024692586828042, + "brier": 1.999994531907044, + "expected_brier": 1.9999945319070438, + "multiclass_ece": 0.9999975643895237, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + } + ] + }, + "painting-geometry-v1/x10_y7_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.001537006043315243, + "brier": 3.5279133078137338e-06, + "expected_brier": 3.5279133077104063e-06, + "multiclass_ece": 0.0015358254544615146, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x10_y9_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.990209588224567e-05, + "brier": 7.921710750181948e-10, + "expected_brier": 7.921709954672451e-10, + "multiclass_ece": 1.9901897836849258e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x10_y9_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 7.49891631098218e-06, + "brier": 9.975127583065473e-11, + "expected_brier": 9.975131831652106e-11, + "multiclass_ece": 7.498888194179543e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x11_y0_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.3785853318066851e-07, + "brier": 8.231841299420405e-14, + "expected_brier": 8.237854842718662e-14, + "multiclass_ece": 2.3785850489232985e-07, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x11_y10": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.816243893855926e-07, + "brier": 1.0623211861126997e-13, + "expected_brier": 1.0613732115416497e-13, + "multiclass_ece": 2.8162434972944794e-07, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x11_y1_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.012962229470247723, + "brier": 0.00032269158371086887, + "expected_brier": 0.000322691583710899, + "multiclass_ece": 0.012878581584920279, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y0": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.000128305450046792, + "brier": 3.03169017779815e-08, + "expected_brier": 3.0316901877824876e-08, + "multiclass_ece": 0.0001282972192545584, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y10_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00035292481784936794, + "brier": 2.49023954449386e-07, + "expected_brier": 2.4902395434622804e-07, + "multiclass_ece": 0.0003528625472116742, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y3_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.004584275653200041, + "brier": 4.1837552467338946e-05, + "expected_brier": 4.183755246733867e-05, + "multiclass_ece": 0.004573783900053474, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y4_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.0195543141355496e-06, + "brier": 1.70489947269906e-12, + "expected_brier": 1.7048584766143904e-12, + "multiclass_ece": 1.0195537943902266e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y6_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0029212007225944436, + "brier": 1.691979542712627e-05, + "expected_brier": 1.69197954271727e-05, + "multiclass_ece": 0.0029169381673669337, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y7_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.000879963215622871, + "brier": 1.4421045559368693e-06, + "expected_brier": 1.44210455599314e-06, + "multiclass_ece": 0.000879576161531892, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y7_red": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00041096724432219527, + "brier": 3.376493652192716e-07, + "expected_brier": 3.3764936513414057e-07, + "multiclass_ece": 0.00041088280885137607, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y8_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.5852451523952406e-07, + "brier": 1.0006688726314381e-13, + "expected_brier": 1.0014211682118912e-13, + "multiclass_ece": 2.5852448182206444e-07, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y9_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00023231880206167244, + "brier": 1.0791897754815708e-07, + "expected_brier": 1.0791897753215096e-07, + "multiclass_ece": 0.00023229181813844146, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y9_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 4.1455140638629345e-06, + "brier": 3.0079231843866135e-11, + "expected_brier": 3.0079272406169366e-11, + "multiclass_ece": 4.145505471231381e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y10_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0034888795035870677, + "brier": 1.7951735909453338e-05, + "expected_brier": 1.795173590934951e-05, + "multiclass_ece": 0.003482800435259059, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y11_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00015186896094090328, + "brier": 4.612135774861029e-08, + "expected_brier": 4.6121357843986743e-08, + "multiclass_ece": 0.00015185742943402136, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y4_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.003518812043101213, + "brier": 1.8160427082434226e-05, + "expected_brier": 1.8160427082358765e-05, + "multiclass_ece": 0.0035126282792977692, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y6_red": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 9.978437336721664e-05, + "brier": 1.9911855361964998e-08, + "expected_brier": 1.99118552757227e-08, + "multiclass_ece": 9.977939507221922e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y6_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 4.9714481868543814e-06, + "brier": 4.834889242422484e-11, + "expected_brier": 4.834888045479602e-11, + "multiclass_ece": 4.971435829226323e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y8": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 4.924777391384053e-05, + "brier": 8.71554072124494e-09, + "expected_brier": 8.715540733561511e-09, + "multiclass_ece": 4.924535167427635e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x3_y0_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0039472923751762675, + "brier": 2.3844239673094157e-05, + "expected_brier": 2.384423967316529e-05, + "multiclass_ece": 0.0039395120570586695, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x3_y11_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 4.730634477307738e-06, + "brier": 4.4757593383729735e-11, + "expected_brier": 4.475753101473856e-11, + "multiclass_ece": 4.730623287874103e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x3_y1_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.8398603351827058e-06, + "brier": 1.6129567640768465e-11, + "expected_brier": 1.61295421463592e-11, + "multiclass_ece": 2.8398563027831614e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x3_y6": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 4.329475853571527e-07, + "brier": 3.2926292461598587e-13, + "expected_brier": 3.2929214910382143e-13, + "multiclass_ece": 4.329474916353604e-07, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x3_y7_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 5.024632832461804e-06, + "brier": 5.0493616489765516e-11, + "expected_brier": 5.0493609293766895e-11, + "multiclass_ece": 5.024620209015396e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x3_y7_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.533430581352499e-06, + "brier": 7.272272902732672e-12, + "expected_brier": 7.2721828558997e-12, + "multiclass_ece": 2.533427372219954e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x4_y10_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 4.92626191403477e-05, + "brier": 4.653850107992144e-09, + "expected_brier": 4.653850194102915e-09, + "multiclass_ece": 4.9261405757450305e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x4_y11_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0007957512711804574, + "brier": 1.1544620231606477e-06, + "expected_brier": 1.154462023089664e-06, + "multiclass_ece": 0.0007954347451019084, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x4_y1_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 3.455139297289403e-05, + "brier": 2.151770282497106e-09, + "expected_brier": 2.15177031748226e-09, + "multiclass_ece": 3.4550796080390356e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x4_y1_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 8.73294799897814e-05, + "brier": 1.5238732247179992e-08, + "expected_brier": 1.5238732276046107e-08, + "multiclass_ece": 8.732566688174348e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x4_y4_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.023168507104316154, + "brier": 0.0010489754668801326, + "expected_brier": 0.0010489754668800977, + "multiclass_ece": 0.0229021780242894, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y5_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 5.2921038846937935e-06, + "brier": 5.419098611275967e-11, + "expected_brier": 5.4191096054978516e-11, + "multiclass_ece": 5.292089881536732e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x4_y6_red": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00016308895371474277, + "brier": 5.3187338790633557e-08, + "expected_brier": 5.318733875547821e-08, + "multiclass_ece": 0.00016307565543427494, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x5_y1_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.017767688991294446, + "brier": 0.00045161479261122267, + "expected_brier": 0.0004516147926112435, + "multiclass_ece": 0.017610774316676348, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y2": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 15.117011968525413, + "brier": 1.9999986128810203, + "expected_brier": 1.9999986128810203, + "multiclass_ece": 0.9999995785632991, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + } + ] + }, + "painting-geometry-v1/x5_y4_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.7456516336085176e-05, + "brier": 4.059194241311343e-10, + "expected_brier": 4.059195202188448e-10, + "multiclass_ece": 1.7456363971990463e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x5_y4_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0028742118463159255, + "brier": 1.198615829488669e-05, + "expected_brier": 1.1986158294785554e-05, + "multiclass_ece": 0.002870085253960819, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x5_y7": { + "count": 2, + "hard_count": 2, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 6.20307185284183, + "brier": 0.9999909164704477, + "expected_brier": 0.9999909164704476, + "multiclass_ece": 0.4999897278828367, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + } + ] + }, + "painting-geometry-v1/x5_y8": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 9.127577697103215, + "brier": 1.9995655100100689, + "expected_brier": 1.9995655100100689, + "multiclass_ece": 0.9998913716024529, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + } + ] + }, + "painting-geometry-v1/x5_y9": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 6.025483110346419e-07, + "brier": 4.998409776325495e-13, + "expected_brier": 4.998224056862455e-13, + "multiclass_ece": 6.025481295024449e-07, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x5_y9_red": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.0807857234144935e-05, + "brier": 8.659158273435021e-10, + "expected_brier": 8.659157835211317e-10, + "multiclass_ece": 2.0807640752185108e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x6_y0_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 5.441047142052022e-07, + "brier": 4.4988109172339823e-13, + "expected_brier": 4.4986236957811343e-13, + "multiclass_ece": 5.44104566180259e-07, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x6_y1_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0038534978725364853, + "brier": 2.9507394059558046e-05, + "expected_brier": 2.9507394059491432e-05, + "multiclass_ece": 0.0038460826774803225, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x6_y3": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 9.36127776925564e-08, + "brier": 1.254076646863648e-14, + "expected_brier": 1.2434497875801753e-14, + "multiclass_ece": 9.361277331088047e-08, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x6_y3_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.7081319966842794e-06, + "brier": 1.4667918100049598e-11, + "expected_brier": 1.4667822512137718e-11, + "multiclass_ece": 2.708128329698134e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x6_y6_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 15.649195921319723, + "brier": 1.999999179876847, + "expected_brier": 1.999999179876847, + "multiclass_ece": 0.9999997497618509, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + } + ] + }, + "painting-geometry-v1/x6_y7_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.003551802740607822, + "brier": 1.903739681308588e-05, + "expected_brier": 1.903739681319294e-05, + "multiclass_ece": 0.003545502550471813, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x6_y8_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 10.079158766856478, + "brier": 1.999832224767863, + "expected_brier": 1.999832224767863, + "multiclass_ece": 0.9999580553122873, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + } + ] + }, + "painting-geometry-v1/x7_y1_lightness": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0067038468854915935, + "brier": 0.00016665287252551992, + "expected_brier": 0.00016665287252559668, + "multiclass_ece": 0.006659108940699654, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x7_y2": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0003446306049209988, + "brier": 2.3745866042260927e-07, + "expected_brier": 2.3745866051250175e-07, + "multiclass_ece": 0.00034457122661546435, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x7_y5_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 7.31385025341395, + "brier": 1.9973359002178304, + "expected_brier": 1.9973359002178306, + "multiclass_ece": 0.9993337531119997, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + } + ] + }, + "painting-geometry-v1/x7_y6_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00022325361311932755, + "brier": 8.641578650182098e-08, + "expected_brier": 8.641578652657245e-08, + "multiclass_ece": 0.00022322869388591293, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x7_y7": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 3.7852526651568454, + "brier": 1.9102183607862024, + "expected_brier": 1.9102183607862022, + "multiclass_ece": 0.9772968742368418, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y7_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 5.828140108460244e-06, + "brier": 6.793403831794919e-11, + "expected_brier": 6.793410278760348e-11, + "multiclass_ece": 5.8281231248846765e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x7_y8": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00010362938704403858, + "brier": 2.1475874090492268e-08, + "expected_brier": 2.1475874190102218e-08, + "multiclass_ece": 0.0001036240177045844, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x7_y9_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.03934928570467057, + "brier": 0.0029775920228175797, + "expected_brier": 0.0029775920228176656, + "multiclass_ece": 0.038585157967589434, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x8_y0_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.4037204325126063e-05, + "brier": 1.1555466071483484e-09, + "expected_brier": 1.155546547337849e-09, + "multiclass_ece": 2.4036915433844896e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x8_y11_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 4.76426922443524e-06, + "brier": 4.5396306204899856e-11, + "expected_brier": 4.5396353343107876e-11, + "multiclass_ece": 4.764257875322642e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x8_y3": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 9.370810730992986, + "brier": 1.999658282676168, + "expected_brier": 1.999658282676168, + "multiclass_ece": 0.9999143083461733, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + } + ] + }, + "painting-geometry-v1/x9_y0": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.3569136908587665e-05, + "brier": 9.147839124472341e-10, + "expected_brier": 9.147838042622425e-10, + "multiclass_ece": 2.356885915866247e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x9_y0_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 8.28504232392622e-05, + "brier": 1.3090167542698382e-08, + "expected_brier": 1.309016761652515e-08, + "multiclass_ece": 8.284699123772832e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x9_y1_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 4.141238516106648e-06, + "brier": 2.991970968338504e-11, + "expected_brier": 2.991962233522827e-11, + "multiclass_ece": 4.141229941190261e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x9_y2": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.7444554835738955e-05, + "brier": 4.5072323197345854e-10, + "expected_brier": 4.507232365114078e-10, + "multiclass_ece": 1.7444401789079578e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x9_y6_red": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 6.842424359888946, + "brier": 1.995732230260233, + "expected_brier": 1.995732230260233, + "multiclass_ece": 0.9989324877738818, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + } + ] + }, + "painting-geometry-v1/x9_y9_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.0809549052951223e-05, + "brier": 5.974409493078514e-10, + "expected_brier": 5.974409855724616e-10, + "multiclass_ece": 2.0809332535787206e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "reasoning-control-v1/decision": { + "count": 27, + "hard_count": 27, + "accuracy": 0.5925925925925926, + "expected_accuracy": 0.5925925925925926, + "nll": 1.502417812530906, + "brier": 0.6168485239676222, + "expected_brier": 0.6168485239676221, + "multiclass_ece": 0.26481365019916775, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 21, + "coverage": 0.7777777777777778, + "accuracy": 0.7142857142857143, + "expected_accuracy": 0.7142857142857143, + "expected_risk": 0.2857142857142857 + }, + { + "threshold": 0.7, + "selected": 19, + "coverage": 0.7037037037037037, + "accuracy": 0.7368421052631579, + "expected_accuracy": 0.7368421052631579, + "expected_risk": 0.26315789473684215 + }, + { + "threshold": 0.8, + "selected": 19, + "coverage": 0.7037037037037037, + "accuracy": 0.7368421052631579, + "expected_accuracy": 0.7368421052631579, + "expected_risk": 0.26315789473684215 + }, + { + "threshold": 0.9, + "selected": 16, + "coverage": 0.5925925925925926, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.95, + "selected": 16, + "coverage": 0.5925925925925926, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.99, + "selected": 14, + "coverage": 0.5185185185185185, + "accuracy": 0.7857142857142857, + "expected_accuracy": 0.7857142857142857, + "expected_risk": 0.2142857142857143 + } + ] + }, + "reasoning-control-v1/ordinal": { + "count": 26, + "hard_count": 26, + "accuracy": 0.7692307692307693, + "expected_accuracy": 0.7692307692307693, + "nll": 0.937447938803616, + "brier": 0.43023101205625175, + "expected_brier": 0.4302310120562518, + "multiclass_ece": 0.2478364988169227, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 20, + "coverage": 0.7692307692307693, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.7, + "selected": 19, + "coverage": 0.7307692307692307, + "accuracy": 0.7894736842105263, + "expected_accuracy": 0.7894736842105263, + "expected_risk": 0.21052631578947367 + }, + { + "threshold": 0.8, + "selected": 19, + "coverage": 0.7307692307692307, + "accuracy": 0.7894736842105263, + "expected_accuracy": 0.7894736842105263, + "expected_risk": 0.21052631578947367 + }, + { + "threshold": 0.9, + "selected": 18, + "coverage": 0.6923076923076923, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.95, + "selected": 18, + "coverage": 0.6923076923076923, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.99, + "selected": 10, + "coverage": 0.38461538461538464, + "accuracy": 0.9, + "expected_accuracy": 0.9, + "expected_risk": 0.09999999999999998 + } + ] + }, + "reasoning-control-v1/verification": { + "count": 27, + "hard_count": 27, + "accuracy": 0.6666666666666666, + "expected_accuracy": 0.6666666666666666, + "nll": 0.813636361672112, + "brier": 0.5060510740938344, + "expected_brier": 0.5060510740938345, + "multiclass_ece": 0.2522035931276929, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.6666666666666666, + "expected_accuracy": 0.6666666666666666, + "expected_risk": 0.33333333333333337 + }, + { + "threshold": 0.7, + "selected": 24, + "coverage": 0.8888888888888888, + "accuracy": 0.7083333333333334, + "expected_accuracy": 0.7083333333333334, + "expected_risk": 0.29166666666666663 + }, + { + "threshold": 0.8, + "selected": 24, + "coverage": 0.8888888888888888, + "accuracy": 0.7083333333333334, + "expected_accuracy": 0.7083333333333334, + "expected_risk": 0.29166666666666663 + }, + { + "threshold": 0.9, + "selected": 18, + "coverage": 0.6666666666666666, + "accuracy": 0.7777777777777778, + "expected_accuracy": 0.7777777777777778, + "expected_risk": 0.2222222222222222 + }, + { + "threshold": 0.95, + "selected": 16, + "coverage": 0.5925925925925926, + "accuracy": 0.875, + "expected_accuracy": 0.875, + "expected_risk": 0.125 + }, + { + "threshold": 0.99, + "selected": 12, + "coverage": 0.4444444444444444, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "snake-v1/choice": { + "count": 27, + "hard_count": 21, + "accuracy": 1.0, + "expected_accuracy": 0.8888888888888888, + "nll": 0.19440325657563673, + "brier": 0.020131978776713792, + "expected_brier": 0.1312430898878249, + "multiclass_ece": 0.029420512352787535, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 0.8888888888888888, + "expected_risk": 0.11111111111111116 + }, + { + "threshold": 0.7, + "selected": 21, + "coverage": 0.7777777777777778, + "accuracy": 1.0, + "expected_accuracy": 0.9761904761904762, + "expected_risk": 0.023809523809523836 + }, + { + "threshold": 0.8, + "selected": 20, + "coverage": 0.7407407407407407, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 20, + "coverage": 0.7407407407407407, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 19, + "coverage": 0.7037037037037037, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 10, + "coverage": 0.37037037037037035, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "snake-v1/noul": { + "count": 27, + "hard_count": 27, + "accuracy": 0.8518518518518519, + "expected_accuracy": 0.8518518518518519, + "nll": 0.6094040538850714, + "brier": 0.2627485067071707, + "expected_brier": 0.26274850670717065, + "multiclass_ece": 0.13460139998797044, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.8518518518518519, + "expected_accuracy": 0.8518518518518519, + "expected_risk": 0.14814814814814814 + }, + { + "threshold": 0.7, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.8518518518518519, + "expected_accuracy": 0.8518518518518519, + "expected_risk": 0.14814814814814814 + }, + { + "threshold": 0.8, + "selected": 26, + "coverage": 0.9629629629629629, + "accuracy": 0.8846153846153846, + "expected_accuracy": 0.8846153846153846, + "expected_risk": 0.11538461538461542 + }, + { + "threshold": 0.9, + "selected": 26, + "coverage": 0.9629629629629629, + "accuracy": 0.8846153846153846, + "expected_accuracy": 0.8846153846153846, + "expected_risk": 0.11538461538461542 + }, + { + "threshold": 0.95, + "selected": 26, + "coverage": 0.9629629629629629, + "accuracy": 0.8846153846153846, + "expected_accuracy": 0.8846153846153846, + "expected_risk": 0.11538461538461542 + }, + { + "threshold": 0.99, + "selected": 21, + "coverage": 0.7777777777777778, + "accuracy": 0.9047619047619048, + "expected_accuracy": 0.9047619047619048, + "expected_risk": 0.09523809523809523 + } + ] + }, + "vizdoom-basic-v1/choice": { + "count": 27, + "hard_count": 27, + "accuracy": 0.7777777777777778, + "expected_accuracy": 0.7777777777777778, + "nll": 0.6401358810062345, + "brier": 0.3670309141183484, + "expected_brier": 0.3670309141183485, + "multiclass_ece": 0.1965554420387896, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.7777777777777778, + "expected_accuracy": 0.7777777777777778, + "expected_risk": 0.2222222222222222 + }, + { + "threshold": 0.7, + "selected": 26, + "coverage": 0.9629629629629629, + "accuracy": 0.7692307692307693, + "expected_accuracy": 0.7692307692307693, + "expected_risk": 0.23076923076923073 + }, + { + "threshold": 0.8, + "selected": 25, + "coverage": 0.9259259259259259, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.9, + "selected": 20, + "coverage": 0.7407407407407407, + "accuracy": 0.95, + "expected_accuracy": 0.95, + "expected_risk": 0.050000000000000044 + }, + { + "threshold": 0.95, + "selected": 20, + "coverage": 0.7407407407407407, + "accuracy": 0.95, + "expected_accuracy": 0.95, + "expected_risk": 0.050000000000000044 + }, + { + "threshold": 0.99, + "selected": 16, + "coverage": 0.5925925925925926, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "vizdoom-basic-v1/noul": { + "count": 27, + "hard_count": 27, + "accuracy": 0.9629629629629629, + "expected_accuracy": 0.9629629629629629, + "nll": 0.37539937221273895, + "brier": 0.07422688947497312, + "expected_brier": 0.07422688947497315, + "multiclass_ece": 0.03391932587399235, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.9629629629629629, + "expected_accuracy": 0.9629629629629629, + "expected_risk": 0.03703703703703709 + }, + { + "threshold": 0.7, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.9629629629629629, + "expected_accuracy": 0.9629629629629629, + "expected_risk": 0.03703703703703709 + }, + { + "threshold": 0.8, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.9629629629629629, + "expected_accuracy": 0.9629629629629629, + "expected_risk": 0.03703703703703709 + }, + { + "threshold": 0.9, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.9629629629629629, + "expected_accuracy": 0.9629629629629629, + "expected_risk": 0.03703703703703709 + }, + { + "threshold": 0.95, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.9629629629629629, + "expected_accuracy": 0.9629629629629629, + "expected_risk": 0.03703703703703709 + }, + { + "threshold": 0.99, + "selected": 26, + "coverage": 0.9629629629629629, + "accuracy": 0.9615384615384616, + "expected_accuracy": 0.9615384615384616, + "expected_risk": 0.038461538461538436 + } + ] + }, + "vizdoom-basic-v1/score": { + "count": 27, + "hard_count": 27, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.002562137507468964, + "brier": 0.00022156283898120443, + "expected_brier": 0.0002215628389812181, + "multiclass_ece": 0.0025003145907955167, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 26, + "coverage": 0.9629629629629629, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 25, + "coverage": 0.9259259259259259, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "wikispeedia-v1/choice": { + "count": 27, + "hard_count": 9, + "accuracy": 0.5555555555555556, + "expected_accuracy": 0.27335390946502053, + "nll": 2.151915626932359, + "brier": 0.3401768692105326, + "expected_brier": 0.8483044412269933, + "multiclass_ece": 0.09307988515415551, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/AUTO-CLOSE": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 4.778943656201916e-05, + "brier": 5.521982003692882e-09, + "expected_brier": 5.52198203829235e-09, + "multiclass_ece": 4.778805600458291e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/agent_trace_observability/COUNT ONLY": { + "count": 6, + "hard_count": 6, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.4737388303659237e-05, + "brier": 4.935604484434741e-10, + "expected_brier": 4.935604517678863e-10, + "multiclass_ece": 1.473726491196814e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 6, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 6, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 6, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 6, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 6, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 6, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/agent_trace_observability/FILE ISSUE ยท ROUTE": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 3.7203522345526786e-05, + "brier": 3.2087445746446093e-09, + "expected_brier": 3.20874460335574e-09, + "multiclass_ece": 3.720272013407744e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/agent_trace_observability/HUMAN REVIEW": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.2010814652667876e-05, + "brier": 9.689305962049293e-10, + "expected_brier": 9.68930491396236e-10, + "multiclass_ece": 2.2010572416464314e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/agent_trace_observability/NOT A BUG": { + "count": 6, + "hard_count": 6, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0007307348129932742, + "brier": 3.589629002360736e-06, + "expected_brier": 3.5896290023392416e-06, + "multiclass_ece": 0.0007298357614965075, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 6, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 6, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 6, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 6, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 6, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 6, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/agent_trace_observability/PAGE ON-CALL": { + "count": 5, + "hard_count": 5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0006749572793394489, + "brier": 1.7585863344137055e-06, + "expected_brier": 1.758586334377199e-06, + "multiclass_ece": 0.000674517152735632, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/agent_trace_observability/PRIORITY REVIEW": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 7.126580607803153e-05, + "brier": 1.523647581506189e-08, + "expected_brier": 1.5236475784755992e-08, + "multiclass_ece": 7.126199665918875e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service/CANCEL": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00250434697641378, + "brier": 4.9120135579473054e-05, + "expected_brier": 4.9120135579505675e-05, + "multiclass_ece": 0.0024919851924700698, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service/CLOSE": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0001432868376515998, + "brier": 1.1053721694104322e-07, + "expected_brier": 1.1053721700922854e-07, + "multiclass_ece": 0.00014325919586092675, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service/FLAG FOR REVIEW": { + "count": 5, + "hard_count": 5, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "nll": 0.9314891816494775, + "brier": 0.3924430250823311, + "expected_brier": 0.3924430250823311, + "multiclass_ece": 0.19808931941203695, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 1.0, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.7, + "selected": 5, + "coverage": 1.0, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.8, + "selected": 5, + "coverage": 1.0, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.9, + "selected": 5, + "coverage": 1.0, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.95, + "selected": 5, + "coverage": 1.0, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.99, + "selected": 5, + "coverage": 1.0, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + } + ] + }, + "workflow-controls-v1/customer_service/FREEZE CARD": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0017248414468705889, + "brier": 1.749754170181558e-05, + "expected_brier": 1.749754170177198e-05, + "multiclass_ece": 0.0017204520641126801, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service/HAND OFF": { + "count": 4, + "hard_count": 4, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "nll": 1.3945689819641403, + "brier": 0.497085957716629, + "expected_brier": 0.49708595771662895, + "multiclass_ece": 0.2285805277349755, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.9, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.95, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 0.25, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + } + ] + }, + "workflow-controls-v1/customer_service/REFUND": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.5363204178649037e-05, + "brier": 4.720488330428916e-10, + "expected_brier": 4.720488444576176e-10, + "multiclass_ece": 1.5363086165232076e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service/SAY": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.000657763803596837, + "brier": 1.4824175333774262e-06, + "expected_brier": 1.4824175333938427e-06, + "multiclass_ece": 0.0006573928901505433, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service/SET INTENT": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0027263553627651324, + "brier": 1.6820721922970587e-05, + "expected_brier": 1.682072192299433e-05, + "multiclass_ece": 0.002722140588306221, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/DISPUTE LINES": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.2133738577881176e-05, + "brier": 9.874866856141383e-10, + "expected_brier": 9.874866850149526e-10, + "multiclass_ece": 2.2133491702547303e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/DUPLICATE": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.000503096794424492, + "brier": 9.900239062991796e-07, + "expected_brier": 9.90023906299875e-07, + "multiclass_ece": 0.0005028491241749755, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/FRAUD REVIEW": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.3012898787256536e-05, + "brier": 3.674297298018996e-10, + "expected_brier": 3.674297162821934e-10, + "multiclass_ece": 1.3012806928891507e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/HOLD": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0022840253061071907, + "brier": 1.7808844243052765e-05, + "expected_brier": 1.7808844243072386e-05, + "multiclass_ece": 0.0022795589598016974, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/HOLD FOR DOCUMENTS": { + "count": 5, + "hard_count": 5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 8.713627795069097e-06, + "brier": 1.6031455477252777e-10, + "expected_brier": 1.6031456162579615e-10, + "multiclass_ece": 8.713587716080439e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/PAY": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.719754101329739e-05, + "brier": 5.915006613776244e-10, + "expected_brier": 5.915006262569023e-10, + "multiclass_ece": 1.7197393136436645e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/REQUEST CORRECTED INVOICE": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.2309465869623787e-05, + "brier": 3.3256035191541327e-10, + "expected_brier": 3.3256042364371297e-10, + "multiclass_ece": 1.2309382728670215e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/ROUTE FOR APPROVAL": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0013340581627552925, + "brier": 3.5546775799253404e-06, + "expected_brier": 3.5546775798422914e-06, + "multiclass_ece": 0.0013331687027389183, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/SCHEDULE": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0027290700657674107, + "brier": 2.7936563016637914e-05, + "expected_brier": 2.7936563016695004e-05, + "multiclass_ece": 0.002722061250995056, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/SHORT PAY": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 4.1519957175355124e-05, + "brier": 7.609927592660607e-09, + "expected_brier": 7.609927632490354e-09, + "multiclass_ece": 4.1518054546885175e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/AUTO CLOSE": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.186097620677739e-05, + "brier": 1.2964098549248138e-09, + "expected_brier": 1.2964097943305812e-09, + "multiclass_ece": 2.186065209667613e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/BLOCK DESTINATION": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.6620258657322584e-05, + "brier": 5.801599115722715e-10, + "expected_brier": 5.801599201049612e-10, + "multiclass_ece": 1.662011361558413e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/BLOCK EGRESS ASSET": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.8682109500481296e-05, + "brier": 7.411163367115808e-10, + "expected_brier": 7.411163349679839e-10, + "multiclass_ece": 1.8681924218832968e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/BLOCK SENDER": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 4.744025270368777e-05, + "brier": 5.126112835911588e-09, + "expected_brier": 5.126112793500208e-09, + "multiclass_ece": 4.7438971126171715e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/ESCALATE TIER2": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.099206339974016e-05, + "brier": 8.813149507798197e-10, + "expected_brier": 8.813150209618925e-10, + "multiclass_ece": 2.0991843067919014e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/ESCALATE URGENT": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00144888903913702, + "brier": 7.830920920538481e-06, + "expected_brier": 7.830920920604001e-06, + "multiclass_ece": 0.0014469276553795707, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/ISOLATE HOST": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 4.4876844873326406e-05, + "brier": 4.0276816585518035e-09, + "expected_brier": 4.027681743679068e-09, + "multiclass_ece": 4.487583792278649e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/NOTIFY USER": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 9.821390689225791e-06, + "brier": 1.929175354161015e-10, + "expected_brier": 1.9291745978478048e-10, + "multiclass_ece": 9.82134245952615e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/PURGE MAILBOXES": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.5373839740628415e-05, + "brier": 1.736036166015553e-09, + "expected_brier": 1.7360360926588214e-09, + "multiclass_ece": 2.5373405720507947e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/QUARANTINE FILE": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.5315581230035107e-05, + "brier": 1.2817248580757944e-09, + "expected_brier": 1.2817249483987325e-09, + "multiclass_ece": 2.531526079341262e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/QUARANTINE MESSAGE": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.3577685267229275e-05, + "brier": 3.7063084251807407e-10, + "expected_brier": 3.706308593365293e-10, + "multiclass_ece": 1.3577592608671196e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/REQUIRE REAUTH": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 9.411530633905133e-06, + "brier": 1.7715215046815144e-10, + "expected_brier": 1.7715207079049833e-10, + "multiclass_ece": 9.411486345589637e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/REVOKE ACCESS KEY": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 4.3360637344053244e-05, + "brier": 5.525104154909498e-09, + "expected_brier": 5.52510415197105e-09, + "multiclass_ece": 4.3359256002517554e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/REVOKE SESSIONS": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 9.371736102711662e-05, + "brier": 1.942748032458055e-08, + "expected_brier": 1.9427480291156485e-08, + "multiclass_ece": 9.371250379541785e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + } + }, + "calibrated_test": { + "count": 512, + "hard_count": 470, + "accuracy": 0.9170212765957447, + "expected_accuracy": 0.8705829923115078, + "nll": 0.3236699851754806, + "brier": 0.11310645000438298, + "expected_brier": 0.16439845769287523, + "multiclass_ece": 0.020281663881718474, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 456, + "coverage": 0.890625, + "accuracy": 0.9527027027027027, + "expected_accuracy": 0.9400584795321637, + "expected_risk": 0.059941520467836296 + }, + { + "threshold": 0.7, + "selected": 425, + "coverage": 0.830078125, + "accuracy": 0.966903073286052, + "expected_accuracy": 0.963921568627451, + "expected_risk": 0.03607843137254896 + }, + { + "threshold": 0.8, + "selected": 421, + "coverage": 0.822265625, + "accuracy": 0.969047619047619, + "expected_accuracy": 0.9675376088677751, + "expected_risk": 0.03246239113222493 + }, + { + "threshold": 0.9, + "selected": 394, + "coverage": 0.76953125, + "accuracy": 0.9873096446700508, + "expected_accuracy": 0.9873096446700508, + "expected_risk": 0.012690355329949221 + }, + { + "threshold": 0.95, + "selected": 365, + "coverage": 0.712890625, + "accuracy": 0.9917808219178083, + "expected_accuracy": 0.9917808219178083, + "expected_risk": 0.008219178082191747 + }, + { + "threshold": 0.99, + "selected": 269, + "coverage": 0.525390625, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ], + "mean_latency_seconds": 0.040071311304927804, + "by_kind": { + "choice": { + "count": 197, + "hard_count": 158, + "accuracy": 0.8481012658227848, + "expected_accuracy": 0.7474035130126501, + "nll": 0.6196996382762615, + "brier": 0.1733455196078818, + "expected_brier": 0.29903845329573936, + "multiclass_ece": 0.05666190934572925, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 149, + "coverage": 0.7563451776649747, + "accuracy": 0.9357142857142857, + "expected_accuracy": 0.907158836689038, + "expected_risk": 0.09284116331096204 + }, + { + "threshold": 0.7, + "selected": 127, + "coverage": 0.6446700507614214, + "accuracy": 0.96, + "expected_accuracy": 0.9501312335958004, + "expected_risk": 0.04986876640419957 + }, + { + "threshold": 0.8, + "selected": 124, + "coverage": 0.6294416243654822, + "accuracy": 0.959349593495935, + "expected_accuracy": 0.9543010752688171, + "expected_risk": 0.045698924731182866 + }, + { + "threshold": 0.9, + "selected": 113, + "coverage": 0.5736040609137056, + "accuracy": 0.9911504424778761, + "expected_accuracy": 0.9911504424778761, + "expected_risk": 0.008849557522123908 + }, + { + "threshold": 0.95, + "selected": 97, + "coverage": 0.49238578680203043, + "accuracy": 0.9896907216494846, + "expected_accuracy": 0.9896907216494846, + "expected_risk": 0.010309278350515427 + }, + { + "threshold": 0.99, + "selected": 53, + "coverage": 0.26903553299492383, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "noul": { + "count": 219, + "hard_count": 219, + "accuracy": 0.958904109589041, + "expected_accuracy": 0.958904109589041, + "nll": 0.10791754053655628, + "brier": 0.06341140144851942, + "expected_brier": 0.06341140144851942, + "multiclass_ece": 0.017397665132187427, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 219, + "coverage": 1.0, + "accuracy": 0.958904109589041, + "expected_accuracy": 0.958904109589041, + "expected_risk": 0.04109589041095896 + }, + { + "threshold": 0.7, + "selected": 213, + "coverage": 0.9726027397260274, + "accuracy": 0.971830985915493, + "expected_accuracy": 0.971830985915493, + "expected_risk": 0.028169014084507005 + }, + { + "threshold": 0.8, + "selected": 213, + "coverage": 0.9726027397260274, + "accuracy": 0.971830985915493, + "expected_accuracy": 0.971830985915493, + "expected_risk": 0.028169014084507005 + }, + { + "threshold": 0.9, + "selected": 199, + "coverage": 0.908675799086758, + "accuracy": 0.9849246231155779, + "expected_accuracy": 0.9849246231155779, + "expected_risk": 0.015075376884422065 + }, + { + "threshold": 0.95, + "selected": 191, + "coverage": 0.8721461187214612, + "accuracy": 0.9895287958115183, + "expected_accuracy": 0.9895287958115183, + "expected_risk": 0.010471204188481686 + }, + { + "threshold": 0.99, + "selected": 148, + "coverage": 0.6757990867579908, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "score": { + "count": 96, + "hard_count": 93, + "accuracy": 0.9354838709677419, + "expected_accuracy": 0.921875, + "nll": 0.20837773220746617, + "brier": 0.1028576887736007, + "expected_brier": 0.11848268877360073, + "multiclass_ece": 0.04200637204437315, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 88, + "coverage": 0.9166666666666666, + "accuracy": 0.9647058823529412, + "expected_accuracy": 0.9488636363636364, + "expected_risk": 0.051136363636363646 + }, + { + "threshold": 0.7, + "selected": 85, + "coverage": 0.8854166666666666, + "accuracy": 0.9647058823529412, + "expected_accuracy": 0.9647058823529412, + "expected_risk": 0.03529411764705881 + }, + { + "threshold": 0.8, + "selected": 84, + "coverage": 0.875, + "accuracy": 0.9761904761904762, + "expected_accuracy": 0.9761904761904762, + "expected_risk": 0.023809523809523836 + }, + { + "threshold": 0.9, + "selected": 82, + "coverage": 0.8541666666666666, + "accuracy": 0.9878048780487805, + "expected_accuracy": 0.9878048780487805, + "expected_risk": 0.012195121951219523 + }, + { + "threshold": 0.95, + "selected": 77, + "coverage": 0.8020833333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 68, + "coverage": 0.7083333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ], + "ordinal_mae": 0.09690621252646418 + } + }, + "by_source": { + "customer-control-v1": { + "count": 72, + "hard_count": 68, + "accuracy": 1.0, + "expected_accuracy": 0.9722222222222222, + "nll": 0.04214497732031297, + "brier": 0.00017291467119296647, + "expected_brier": 0.027950692448970735, + "multiclass_ece": 0.003987587549791073, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 72, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 0.9722222222222222, + "expected_risk": 0.02777777777777779 + }, + { + "threshold": 0.7, + "selected": 68, + "coverage": 0.9444444444444444, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 68, + "coverage": 0.9444444444444444, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 68, + "coverage": 0.9444444444444444, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 68, + "coverage": 0.9444444444444444, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 63, + "coverage": 0.875, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1": { + "count": 74, + "hard_count": 74, + "accuracy": 0.9864864864864865, + "expected_accuracy": 0.9864864864864865, + "nll": 0.015511148416632637, + "brier": 0.010118558249251453, + "expected_brier": 0.010118558249251444, + "multiclass_ece": 0.01101949024397686, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 74, + "coverage": 1.0, + "accuracy": 0.9864864864864865, + "expected_accuracy": 0.9864864864864865, + "expected_risk": 0.013513513513513487 + }, + { + "threshold": 0.7, + "selected": 73, + "coverage": 0.9864864864864865, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 73, + "coverage": 0.9864864864864865, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 73, + "coverage": 0.9864864864864865, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 73, + "coverage": 0.9864864864864865, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 69, + "coverage": 0.9324324324324325, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "reasoning-control-v1": { + "count": 72, + "hard_count": 72, + "accuracy": 0.8055555555555556, + "expected_accuracy": 0.8055555555555556, + "nll": 0.5299434905264135, + "brier": 0.2939494744448796, + "expected_brier": 0.2939494744448796, + "multiclass_ece": 0.09434128218323096, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 59, + "coverage": 0.8194444444444444, + "accuracy": 0.864406779661017, + "expected_accuracy": 0.864406779661017, + "expected_risk": 0.13559322033898302 + }, + { + "threshold": 0.7, + "selected": 51, + "coverage": 0.7083333333333334, + "accuracy": 0.9019607843137255, + "expected_accuracy": 0.9019607843137255, + "expected_risk": 0.0980392156862745 + }, + { + "threshold": 0.8, + "selected": 50, + "coverage": 0.6944444444444444, + "accuracy": 0.92, + "expected_accuracy": 0.92, + "expected_risk": 0.07999999999999996 + }, + { + "threshold": 0.9, + "selected": 38, + "coverage": 0.5277777777777778, + "accuracy": 0.9210526315789473, + "expected_accuracy": 0.9210526315789473, + "expected_risk": 0.07894736842105265 + }, + { + "threshold": 0.95, + "selected": 28, + "coverage": 0.3888888888888889, + "accuracy": 0.9642857142857143, + "expected_accuracy": 0.9642857142857143, + "expected_risk": 0.0357142857142857 + }, + { + "threshold": 0.99, + "selected": 13, + "coverage": 0.18055555555555555, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "snake-v1": { + "count": 48, + "hard_count": 43, + "accuracy": 0.8604651162790697, + "expected_accuracy": 0.8159722222222222, + "nll": 0.49093796146627167, + "brier": 0.23937689122196523, + "expected_brier": 0.298404668999743, + "multiclass_ece": 0.11475372469808605, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 48, + "coverage": 1.0, + "accuracy": 0.8604651162790697, + "expected_accuracy": 0.8159722222222222, + "expected_risk": 0.1840277777777778 + }, + { + "threshold": 0.7, + "selected": 39, + "coverage": 0.8125, + "accuracy": 0.8648648648648649, + "expected_accuracy": 0.8376068376068375, + "expected_risk": 0.16239316239316248 + }, + { + "threshold": 0.8, + "selected": 37, + "coverage": 0.7708333333333334, + "accuracy": 0.8611111111111112, + "expected_accuracy": 0.8468468468468469, + "expected_risk": 0.15315315315315314 + }, + { + "threshold": 0.9, + "selected": 27, + "coverage": 0.5625, + "accuracy": 0.9629629629629629, + "expected_accuracy": 0.9629629629629629, + "expected_risk": 0.03703703703703709 + }, + { + "threshold": 0.95, + "selected": 18, + "coverage": 0.375, + "accuracy": 0.9444444444444444, + "expected_accuracy": 0.9444444444444444, + "expected_risk": 0.05555555555555558 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 0.0625, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "tic_tac_toe-v1": { + "count": 24, + "hard_count": 7, + "accuracy": 0.2857142857142857, + "expected_accuracy": 0.3323412698412699, + "nll": 1.2458173047655068, + "brier": 0.261608911733578, + "expected_brier": 0.6792676418923081, + "multiclass_ece": 0.11376480895084877, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 0.20833333333333334, + "accuracy": 1.0, + "expected_accuracy": 0.7, + "expected_risk": 0.30000000000000004 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.041666666666666664, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "tile_platformer-v1": { + "count": 24, + "hard_count": 24, + "accuracy": 0.9583333333333334, + "expected_accuracy": 0.9583333333333334, + "nll": 0.240695125526985, + "brier": 0.0813309089515946, + "expected_brier": 0.0813309089515946, + "multiclass_ece": 0.02458689996639199, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.9583333333333334, + "expected_accuracy": 0.9583333333333334, + "expected_risk": 0.04166666666666663 + }, + { + "threshold": 0.7, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.9583333333333334, + "expected_accuracy": 0.9583333333333334, + "expected_risk": 0.04166666666666663 + }, + { + "threshold": 0.8, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.9583333333333334, + "expected_accuracy": 0.9583333333333334, + "expected_risk": 0.04166666666666663 + }, + { + "threshold": 0.9, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.9583333333333334, + "expected_accuracy": 0.9583333333333334, + "expected_risk": 0.04166666666666663 + }, + { + "threshold": 0.95, + "selected": 23, + "coverage": 0.9583333333333334, + "accuracy": 0.9565217391304348, + "expected_accuracy": 0.9565217391304348, + "expected_risk": 0.04347826086956519 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "trex_runner-v1": { + "count": 4, + "hard_count": 4, + "accuracy": 0.25, + "expected_accuracy": 0.25, + "nll": 1.3801817199977613, + "brier": 0.8197811047574942, + "expected_brier": 0.8197811047574941, + "multiclass_ece": 0.47413478912915286, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.25, + "expected_accuracy": 0.25, + "expected_risk": 0.75 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 0.5, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 0.5, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 0.25, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "vizdoom-basic-v1": { + "count": 73, + "hard_count": 73, + "accuracy": 0.9726027397260274, + "expected_accuracy": 0.9726027397260274, + "nll": 0.08887681097035764, + "brier": 0.04891785248951932, + "expected_brier": 0.04891785248951928, + "multiclass_ece": 0.04803096222074723, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 73, + "coverage": 1.0, + "accuracy": 0.9726027397260274, + "expected_accuracy": 0.9726027397260274, + "expected_risk": 0.0273972602739726 + }, + { + "threshold": 0.7, + "selected": 71, + "coverage": 0.9726027397260274, + "accuracy": 0.971830985915493, + "expected_accuracy": 0.971830985915493, + "expected_risk": 0.028169014084507005 + }, + { + "threshold": 0.8, + "selected": 71, + "coverage": 0.9726027397260274, + "accuracy": 0.971830985915493, + "expected_accuracy": 0.971830985915493, + "expected_risk": 0.028169014084507005 + }, + { + "threshold": 0.9, + "selected": 68, + "coverage": 0.9315068493150684, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 61, + "coverage": 0.8356164383561644, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 42, + "coverage": 0.5753424657534246, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "wikispeedia-v1": { + "count": 24, + "hard_count": 8, + "accuracy": 0.125, + "expected_accuracy": 0.23315145502645504, + "nll": 2.1288767388309613, + "brier": 0.3799896473866204, + "expected_brier": 0.8551715256934985, + "multiclass_ece": 0.073122762799339, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ], + "optimal_action_hit": 0.7083333333333334, + "optimal_set_probability_mass": 0.4243562210617238 + }, + "workflow-controls-v1/agent_trace_observability": { + "count": 24, + "hard_count": 24, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.02153580379299558, + "brier": 0.010521537388120887, + "expected_brier": 0.010521537388120875, + "multiclass_ece": 0.01805894542813694, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 23, + "coverage": 0.9583333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 23, + "coverage": 0.9583333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 23, + "coverage": 0.9583333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 23, + "coverage": 0.9583333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 19, + "coverage": 0.7916666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service": { + "count": 24, + "hard_count": 24, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.008111690823792544, + "brier": 0.00032806089893701464, + "expected_brier": 0.0003280608989369993, + "multiclass_ece": 0.008028099565298893, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 18, + "coverage": 0.75, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing": { + "count": 24, + "hard_count": 24, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.01074202761475687, + "brier": 0.0013203427445843156, + "expected_brier": 0.0013203427445843037, + "multiclass_ece": 0.010391769441288237, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 23, + "coverage": 0.9583333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 22, + "coverage": 0.9166666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 18, + "coverage": 0.75, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents": { + "count": 25, + "hard_count": 25, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0027357167329119737, + "brier": 9.251310162535411e-05, + "expected_brier": 9.251310162534665e-05, + "multiclass_ece": 0.0027121243687646766, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 25, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 25, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 25, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 25, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 25, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 24, + "coverage": 0.96, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "by_question": { + "customer-control-v1/bug_severity": { + "count": 8, + "hard_count": 8, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0012872105233844317, + "brier": 9.055580527662673e-06, + "expected_brier": 9.055580527661533e-06, + "multiclass_ece": 0.0012847822906639772, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 8, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 8, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 8, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 8, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 8, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 8, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "customer-control-v1/category": { + "count": 24, + "hard_count": 23, + "accuracy": 1.0, + "expected_accuracy": 0.9791666666666666, + "nll": 0.031010176634357147, + "brier": 5.415941220925496e-05, + "expected_brier": 0.02088749274554258, + "multiclass_ece": 0.0014246891072468338, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 0.9791666666666666, + "expected_risk": 0.02083333333333337 + }, + { + "threshold": 0.7, + "selected": 23, + "coverage": 0.9583333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 23, + "coverage": 0.9583333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 23, + "coverage": 0.9583333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 23, + "coverage": 0.9583333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 23, + "coverage": 0.9583333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "customer-control-v1/churn_likelihood_level": { + "count": 7, + "hard_count": 7, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0025593942933517253, + "brier": 2.7942462056376566e-05, + "expected_brier": 2.7942462056427886e-05, + "multiclass_ece": 0.0025518894633927503, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 7, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 7, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 7, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 7, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 7, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 7, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "customer-control-v1/frustration": { + "count": 9, + "hard_count": 6, + "accuracy": 1.0, + "expected_accuracy": 0.8333333333333334, + "nll": 0.2429598520272759, + "brier": 0.0008518054305189014, + "expected_brier": 0.16751847209718557, + "multiclass_ece": 0.016695006091616102, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 9, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.7, + "selected": 6, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 6, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 6, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 6, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 0.2222222222222222, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "customer-control-v1/has_reproducible_steps": { + "count": 14, + "hard_count": 14, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.004722639412256341, + "brier": 0.00022835147892207296, + "expected_brier": 0.00022835147892205247, + "multiclass_ece": 0.004664062171303028, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 14, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 14, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 14, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 14, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 14, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 13, + "coverage": 0.9285714285714286, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "customer-control-v1/refund_requested": { + "count": 10, + "hard_count": 10, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0009225063580352112, + "brier": 1.8818974676391858e-06, + "expected_brier": 1.8818974676593215e-06, + "multiclass_ece": 0.0009220355219188292, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 10, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 10, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 10, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 10, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 10, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 10, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x0_y0_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.000218241563230912, + "brier": 8.052480166785841e-08, + "expected_brier": 8.052480171549803e-08, + "multiclass_ece": 0.00021821775027330848, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x0_y1_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 8.598294450672216e-05, + "brier": 1.3628170189533595e-08, + "expected_brier": 1.3628170147939045e-08, + "multiclass_ece": 8.597924807929314e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x0_y2": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00010111584452534208, + "brier": 1.3232042354748071e-08, + "expected_brier": 1.3232042350708184e-08, + "multiclass_ece": 0.00010111073249063907, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x0_y2_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.001058937011001407, + "brier": 2.088678959238411e-06, + "expected_brier": 2.088678959344392e-06, + "multiclass_ece": 0.0010583765350584695, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x0_y2_red": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 0.9400498769560791, + "brier": 0.7427163600440653, + "expected_brier": 0.7427163600440654, + "multiclass_ece": 0.6093916474829899, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y3_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0037054212657513094, + "brier": 2.7358761145411853e-05, + "expected_brier": 2.7358761145457322e-05, + "multiclass_ece": 0.003698564663853543, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x0_y3_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0012294025176477332, + "brier": 2.8449449877317385e-06, + "expected_brier": 2.8449449878120703e-06, + "multiclass_ece": 0.001228647111970127, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x0_y5": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.000845512711410592, + "brier": 1.4285751862307861e-06, + "expected_brier": 1.4285751861464746e-06, + "multiclass_ece": 0.0008451553662583988, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x0_y5_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 5.343664645550871e-05, + "brier": 5.2119507768595014e-09, + "expected_brier": 5.211950870886994e-09, + "multiclass_ece": 5.3435218743347335e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x0_y6_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 4.584030085351549e-06, + "brier": 3.152902180992188e-11, + "expected_brier": 3.1529001631724896e-11, + "multiclass_ece": 4.584019578701692e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x0_y7_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0001326840279423973, + "brier": 3.520543108537415e-08, + "expected_brier": 3.5205431014873056e-08, + "multiclass_ece": 0.00013267522580606705, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x0_y7_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.003852143417150225, + "brier": 2.814895472119881e-05, + "expected_brier": 2.8148954721141095e-05, + "multiclass_ece": 0.0038447334305277137, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y0": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0012366259785851386, + "brier": 2.7191810525322213e-06, + "expected_brier": 2.7191810525462046e-06, + "multiclass_ece": 0.0012358616717660365, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y0_lightness": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.02409253378935903, + "brier": 0.0017524857299658781, + "expected_brier": 0.0017524857299658914, + "multiclass_ece": 0.02356138429014587, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y1_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00016073718257124557, + "brier": 5.166457874684933e-08, + "expected_brier": 5.166457883731823e-08, + "multiclass_ece": 0.00016072426504243342, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y2": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0007418822070420736, + "brier": 1.2998836963282568e-06, + "expected_brier": 1.2998836962374583e-06, + "multiclass_ece": 0.0007414519712093023, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y4_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.3537839081614708e-05, + "brier": 6.737135958367594e-10, + "expected_brier": 6.737135294088148e-10, + "multiclass_ece": 2.3537562068853823e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y4_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.001246870929154414, + "brier": 2.9475673633743613e-06, + "expected_brier": 2.9475673632983757e-06, + "multiclass_ece": 0.0012460939085791045, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y5_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0007407736048810041, + "brier": 1.0149820644604937e-06, + "expected_brier": 1.014982064395653e-06, + "multiclass_ece": 0.000740499299850983, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y6_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.002882696852076714, + "brier": 1.187195555618995e-05, + "expected_brier": 1.1871955556186364e-05, + "multiclass_ece": 0.00287854587113745, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y6_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 3.1502155074564334e-06, + "brier": 1.490340688812961e-11, + "expected_brier": 1.4903411837963176e-11, + "multiclass_ece": 3.1502105455327722e-06, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y0": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0009149866993678658, + "brier": 1.6728700825049182e-06, + "expected_brier": 1.672870082458644e-06, + "multiclass_ece": 0.0009145682266799016, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y0_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00022451273967773582, + "brier": 6.815004479700268e-08, + "expected_brier": 6.815004471860675e-08, + "multiclass_ece": 0.00022448753857862158, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y1": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0008255353401846587, + "brier": 1.5619996698096252e-06, + "expected_brier": 1.5619996697191496e-06, + "multiclass_ece": 0.0008251287212711622, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y1_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.011188082774445688, + "brier": 0.00024730929748284784, + "expected_brier": 0.000247309297482845, + "multiclass_ece": 0.011125728932986134, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y3_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0024339893723389757, + "brier": 8.955887340425834e-06, + "expected_brier": 8.955887340400537e-06, + "multiclass_ece": 0.0024310296220272365, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y3_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00024088369342886454, + "brier": 1.0146670596520899e-07, + "expected_brier": 1.014667059262564e-07, + "multiclass_ece": 0.00024085468328138848, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y4_red": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.002120901206628444, + "brier": 8.97738687281802e-06, + "expected_brier": 8.97738687277716e-06, + "multiclass_ece": 0.002118653684868965, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y5": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0008678139233717089, + "brier": 1.5048955695607922e-06, + "expected_brier": 1.5048955694929589e-06, + "multiclass_ece": 0.0008674374817705388, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y5_green": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00444637222217718, + "brier": 3.936512621932658e-05, + "expected_brier": 3.9365126219270685e-05, + "multiclass_ece": 0.004436501736138743, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y6": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.023216399136101084, + "brier": 0.0010533106412324081, + "expected_brier": 0.0010533106412324855, + "multiclass_ece": 0.02294897210369573, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y6_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 6.696635682389915e-05, + "brier": 6.086643189272156e-09, + "expected_brier": 6.086643189462393e-09, + "multiclass_ece": 6.696411462747687e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x3_y2_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.005805176357467484, + "brier": 6.701019704137307e-05, + "expected_brier": 6.701019704147271e-05, + "multiclass_ece": 0.005788358879741828, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x3_y4": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00012618029728905567, + "brier": 2.132059775999337e-08, + "expected_brier": 2.132059773174433e-08, + "multiclass_ece": 0.00012617233689016238, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x3_y5_red": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0017434254173144573, + "brier": 6.068476746637455e-06, + "expected_brier": 6.0684767466678835e-06, + "multiclass_ece": 0.001741906534036386, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x3_y6_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.979586076856112e-05, + "brier": 1.120638830409826e-09, + "expected_brier": 1.1206389149975848e-09, + "multiclass_ece": 2.9795416876310377e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x3_y7_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00044578710558551993, + "brier": 2.670799669654775e-07, + "expected_brier": 2.6707996703123627e-07, + "multiclass_ece": 0.0004456877572770468, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x4_y0_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00028107624903405727, + "brier": 1.5796331060710616e-07, + "expected_brier": 1.5796331065409674e-07, + "multiclass_ece": 0.0002810367508059297, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x4_y0_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.001387238440011486, + "brier": 3.1782344439022646e-06, + "expected_brier": 3.1782344438457955e-06, + "multiclass_ece": 0.0013862766695531548, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x4_y2": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0059615071680223525, + "brier": 6.437062918672752e-05, + "expected_brier": 6.437062918673853e-05, + "multiclass_ece": 0.005943772643164924, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x4_y4_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.004102570441874282, + "brier": 3.220672064581633e-05, + "expected_brier": 3.220672064574437e-05, + "multiclass_ece": 0.004094166396416576, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x4_y5_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.10380769038208e-05, + "brier": 5.423422768415186e-10, + "expected_brier": 5.423421711725496e-10, + "multiclass_ece": 2.1037855605032796e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x4_y6_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.009557683201579107, + "brier": 0.00016904207809084046, + "expected_brier": 0.00016904207809087524, + "multiclass_ece": 0.009512153715080851, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x4_y7_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0010511231458168173, + "brier": 1.965975152354866e-06, + "expected_brier": 1.965975152451449e-06, + "multiclass_ece": 0.0010505709093894255, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x5_y2": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.007660677090541381, + "brier": 9.324039794654835e-05, + "expected_brier": 9.324039794655903e-05, + "multiclass_ece": 0.00763140888956515, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x5_y2_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.013537068137772645, + "brier": 0.0003615819841535841, + "expected_brier": 0.00036158198415359166, + "multiclass_ece": 0.013445854085062492, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y3_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0006514059214654716, + "brier": 8.481067373215779e-07, + "expected_brier": 8.481067372834161e-07, + "multiclass_ece": 0.0006511938026891873, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x5_y5_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.6070185516887456e-05, + "brier": 4.6003006641940284e-10, + "expected_brier": 4.6003001408223554e-10, + "multiclass_ece": 1.607005639214787e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x5_y6_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00034015929333770713, + "brier": 2.3133798676796574e-07, + "expected_brier": 2.3133798676688855e-07, + "multiclass_ece": 0.00034010144572460543, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x5_y6_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.005640897378568504, + "brier": 5.9914393592434655e-05, + "expected_brier": 5.991439359243067e-05, + "multiclass_ece": 0.005625017390109899, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x5_y7_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.003486348857580535, + "brier": 1.7808592608305536e-05, + "expected_brier": 1.780859260835932e-05, + "multiclass_ece": 0.0034802785997966046, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x5_y7_red": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0015459797153931504, + "brier": 4.7727232730341525e-06, + "expected_brier": 4.7727232730387925e-06, + "multiclass_ece": 0.0015447853043439874, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x5_y7_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 4.6717559649238614e-05, + "brier": 3.649790517007409e-09, + "expected_brier": 3.6497904698507e-09, + "multiclass_ece": 4.671646840104238e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x6_y1_red": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0008305799368383249, + "brier": 1.3785806452024822e-06, + "expected_brier": 1.3785806451949156e-06, + "multiclass_ece": 0.0008302351008004916, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x6_y2_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00021909639288699426, + "brier": 9.598542677148943e-08, + "expected_brier": 9.59854267179594e-08, + "multiclass_ece": 0.0002190723930250993, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x6_y2_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0037322342922921567, + "brier": 2.4703964191233572e-05, + "expected_brier": 2.4703964191186145e-05, + "multiclass_ece": 0.00372527816254542, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x6_y2_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.005086606136375883, + "brier": 4.907267886409957e-05, + "expected_brier": 4.9072678864003905e-05, + "multiclass_ece": 0.005073691262287405, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x6_y4": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0002077978154383708, + "brier": 8.160181467797362e-08, + "expected_brier": 8.160181463878757e-08, + "multiclass_ece": 0.00020776555111789996, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x6_y5": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 7.878936845495954e-05, + "brier": 7.820382279576814e-09, + "expected_brier": 7.820382341705567e-09, + "multiclass_ece": 7.878626465418481e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x6_y6_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 2.3270172315006762e-05, + "brier": 6.791594322253329e-10, + "expected_brier": 6.791593953892061e-10, + "multiclass_ece": 2.32699015666471e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x6_y6_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0002239686961516742, + "brier": 8.566619872175475e-08, + "expected_brier": 8.566619880134851e-08, + "multiclass_ece": 0.0002239436170355935, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x7_y0_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0017792761359933273, + "brier": 6.155669426329723e-06, + "expected_brier": 6.155669426410881e-06, + "multiclass_ece": 0.0017776941626042042, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x7_y1_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.004958735340983621, + "brier": 4.893495501550082e-05, + "expected_brier": 4.893495501545608e-05, + "multiclass_ece": 0.00494646110949537, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x7_y2_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.004848193456745518, + "brier": 4.678268943729667e-05, + "expected_brier": 4.67826894372525e-05, + "multiclass_ece": 0.004836459936632176, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x7_y5_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 1.889944589976293e-05, + "brier": 4.6291421019923875e-10, + "expected_brier": 4.6291415145560677e-10, + "multiclass_ece": 1.889926730636038e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x7_y5_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0010961026718298975, + "brier": 1.9437531445684305e-06, + "expected_brier": 1.943753144661997e-06, + "multiclass_ece": 0.0010955021707199641, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x7_y7": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0006074562436410244, + "brier": 8.382374469179818e-07, + "expected_brier": 8.382374468984821e-07, + "multiclass_ece": 0.0006072351622752326, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "reasoning-control-v1/decision": { + "count": 24, + "hard_count": 24, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "nll": 0.44036328917180384, + "brier": 0.22659672724871238, + "expected_brier": 0.22659672724871238, + "multiclass_ece": 0.10989364944575757, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 19, + "coverage": 0.7916666666666666, + "accuracy": 0.9473684210526315, + "expected_accuracy": 0.9473684210526315, + "expected_risk": 0.052631578947368474 + }, + { + "threshold": 0.7, + "selected": 14, + "coverage": 0.5833333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 14, + "coverage": 0.5833333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 12, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 11, + "coverage": 0.4583333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 5, + "coverage": 0.20833333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "reasoning-control-v1/ordinal": { + "count": 24, + "hard_count": 24, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "nll": 0.735011673059514, + "brier": 0.410916519590808, + "expected_brier": 0.410916519590808, + "multiclass_ece": 0.17537330275027707, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 16, + "coverage": 0.6666666666666666, + "accuracy": 0.8125, + "expected_accuracy": 0.8125, + "expected_risk": 0.1875 + }, + { + "threshold": 0.7, + "selected": 16, + "coverage": 0.6666666666666666, + "accuracy": 0.8125, + "expected_accuracy": 0.8125, + "expected_risk": 0.1875 + }, + { + "threshold": 0.8, + "selected": 15, + "coverage": 0.625, + "accuracy": 0.8666666666666667, + "expected_accuracy": 0.8666666666666667, + "expected_risk": 0.1333333333333333 + }, + { + "threshold": 0.9, + "selected": 13, + "coverage": 0.5416666666666666, + "accuracy": 0.9230769230769231, + "expected_accuracy": 0.9230769230769231, + "expected_risk": 0.07692307692307687 + }, + { + "threshold": 0.95, + "selected": 8, + "coverage": 0.3333333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 5, + "coverage": 0.20833333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "reasoning-control-v1/verification": { + "count": 24, + "hard_count": 24, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "nll": 0.4144555093479229, + "brier": 0.24433517649511874, + "expected_brier": 0.24433517649511863, + "multiclass_ece": 0.09572696881932889, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.7, + "selected": 21, + "coverage": 0.875, + "accuracy": 0.9047619047619048, + "expected_accuracy": 0.9047619047619048, + "expected_risk": 0.09523809523809523 + }, + { + "threshold": 0.8, + "selected": 21, + "coverage": 0.875, + "accuracy": 0.9047619047619048, + "expected_accuracy": 0.9047619047619048, + "expected_risk": 0.09523809523809523 + }, + { + "threshold": 0.9, + "selected": 13, + "coverage": 0.5416666666666666, + "accuracy": 0.8461538461538461, + "expected_accuracy": 0.8461538461538461, + "expected_risk": 0.15384615384615385 + }, + { + "threshold": 0.95, + "selected": 9, + "coverage": 0.375, + "accuracy": 0.8888888888888888, + "expected_accuracy": 0.8888888888888888, + "expected_risk": 0.11111111111111116 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 0.125, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "snake-v1/choice": { + "count": 24, + "hard_count": 19, + "accuracy": 0.8947368421052632, + "expected_accuracy": 0.7986111111111112, + "nll": 0.5145538442333625, + "brier": 0.1891400664145071, + "expected_brier": 0.3071956219700626, + "multiclass_ece": 0.16173213389940258, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.8947368421052632, + "expected_accuracy": 0.7986111111111112, + "expected_risk": 0.20138888888888884 + }, + { + "threshold": 0.7, + "selected": 16, + "coverage": 0.6666666666666666, + "accuracy": 0.9285714285714286, + "expected_accuracy": 0.8541666666666667, + "expected_risk": 0.14583333333333326 + }, + { + "threshold": 0.8, + "selected": 14, + "coverage": 0.5833333333333334, + "accuracy": 0.9230769230769231, + "expected_accuracy": 0.880952380952381, + "expected_risk": 0.11904761904761896 + }, + { + "threshold": 0.9, + "selected": 9, + "coverage": 0.375, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 0.08333333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "snake-v1/noul": { + "count": 24, + "hard_count": 24, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "nll": 0.4673220786991807, + "brier": 0.28961371602942326, + "expected_brier": 0.28961371602942326, + "multiclass_ece": 0.13549784863367967, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.7, + "selected": 23, + "coverage": 0.9583333333333334, + "accuracy": 0.8260869565217391, + "expected_accuracy": 0.8260869565217391, + "expected_risk": 0.17391304347826086 + }, + { + "threshold": 0.8, + "selected": 23, + "coverage": 0.9583333333333334, + "accuracy": 0.8260869565217391, + "expected_accuracy": 0.8260869565217391, + "expected_risk": 0.17391304347826086 + }, + { + "threshold": 0.9, + "selected": 18, + "coverage": 0.75, + "accuracy": 0.9444444444444444, + "expected_accuracy": 0.9444444444444444, + "expected_risk": 0.05555555555555558 + }, + { + "threshold": 0.95, + "selected": 16, + "coverage": 0.6666666666666666, + "accuracy": 0.9375, + "expected_accuracy": 0.9375, + "expected_risk": 0.0625 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 0.125, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "tic_tac_toe-v1/choice": { + "count": 24, + "hard_count": 7, + "accuracy": 0.2857142857142857, + "expected_accuracy": 0.3323412698412699, + "nll": 1.2458173047655068, + "brier": 0.261608911733578, + "expected_brier": 0.6792676418923081, + "multiclass_ece": 0.11376480895084877, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 0.20833333333333334, + "accuracy": 1.0, + "expected_accuracy": 0.7, + "expected_risk": 0.30000000000000004 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 0.041666666666666664, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "tile_platformer-v1/choice": { + "count": 24, + "hard_count": 24, + "accuracy": 0.9583333333333334, + "expected_accuracy": 0.9583333333333334, + "nll": 0.240695125526985, + "brier": 0.0813309089515946, + "expected_brier": 0.0813309089515946, + "multiclass_ece": 0.02458689996639199, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.9583333333333334, + "expected_accuracy": 0.9583333333333334, + "expected_risk": 0.04166666666666663 + }, + { + "threshold": 0.7, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.9583333333333334, + "expected_accuracy": 0.9583333333333334, + "expected_risk": 0.04166666666666663 + }, + { + "threshold": 0.8, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.9583333333333334, + "expected_accuracy": 0.9583333333333334, + "expected_risk": 0.04166666666666663 + }, + { + "threshold": 0.9, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.9583333333333334, + "expected_accuracy": 0.9583333333333334, + "expected_risk": 0.04166666666666663 + }, + { + "threshold": 0.95, + "selected": 23, + "coverage": 0.9583333333333334, + "accuracy": 0.9565217391304348, + "expected_accuracy": 0.9565217391304348, + "expected_risk": 0.04347826086956519 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "trex_runner-v1/choice": { + "count": 4, + "hard_count": 4, + "accuracy": 0.25, + "expected_accuracy": 0.25, + "nll": 1.3801817199977613, + "brier": 0.8197811047574942, + "expected_brier": 0.8197811047574941, + "multiclass_ece": 0.47413478912915286, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.25, + "expected_accuracy": 0.25, + "expected_risk": 0.75 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 0.5, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 0.5, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 0.25, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "vizdoom-basic-v1/choice": { + "count": 24, + "hard_count": 24, + "accuracy": 0.9166666666666666, + "expected_accuracy": 0.9166666666666666, + "nll": 0.254042995042915, + "brier": 0.14751772030464744, + "expected_brier": 0.14751772030464738, + "multiclass_ece": 0.13013473135157544, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 0.9166666666666666, + "expected_accuracy": 0.9166666666666666, + "expected_risk": 0.08333333333333337 + }, + { + "threshold": 0.7, + "selected": 22, + "coverage": 0.9166666666666666, + "accuracy": 0.9090909090909091, + "expected_accuracy": 0.9090909090909091, + "expected_risk": 0.09090909090909094 + }, + { + "threshold": 0.8, + "selected": 22, + "coverage": 0.9166666666666666, + "accuracy": 0.9090909090909091, + "expected_accuracy": 0.9090909090909091, + "expected_risk": 0.09090909090909094 + }, + { + "threshold": 0.9, + "selected": 19, + "coverage": 0.7916666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 13, + "coverage": 0.5416666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "vizdoom-basic-v1/noul": { + "count": 25, + "hard_count": 25, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.013588914526404256, + "brier": 0.0012113439443411202, + "expected_brier": 0.0012113439443411256, + "multiclass_ece": 0.013275429450876043, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 25, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 25, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 25, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 25, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 24, + "coverage": 0.96, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 18, + "coverage": 0.72, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "vizdoom-basic-v1/score": { + "count": 24, + "hard_count": 24, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0021355190269184763, + "brier": 1.2264408951774791e-05, + "expected_brier": 1.2264408951763913e-05, + "multiclass_ece": 0.0021308730585349878, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 24, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "wikispeedia-v1/choice": { + "count": 24, + "hard_count": 8, + "accuracy": 0.125, + "expected_accuracy": 0.23315145502645504, + "nll": 2.1288767388309613, + "brier": 0.3799896473866204, + "expected_brier": 0.8551715256934985, + "multiclass_ece": 0.073122762799339, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/AUTO-CLOSE": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0010989186228561217, + "brier": 2.6001003645713702e-06, + "expected_brier": 2.6001003645692577e-06, + "multiclass_ece": 0.001098268050525153, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/agent_trace_observability/COUNT ONLY": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0006867715936909412, + "brier": 1.0071673253367672e-06, + "expected_brier": 1.0071673253915965e-06, + "multiclass_ece": 0.0006865196713296365, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/agent_trace_observability/FILE ISSUE ยท ROUTE": { + "count": 5, + "hard_count": 5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0010406987489463748, + "brier": 2.2704455141877487e-06, + "expected_brier": 2.2704455142186218e-06, + "multiclass_ece": 0.001040130704974196, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/agent_trace_observability/HUMAN REVIEW": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.001101845968489853, + "brier": 2.5960148490941432e-06, + "expected_brier": 2.59601484908778e-06, + "multiclass_ece": 0.0011011964220879467, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/agent_trace_observability/NOT A BUG": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00040871537036469945, + "brier": 3.339599901169524e-07, + "expected_brier": 3.3395999010643607e-07, + "multiclass_ece": 0.00040863185761574883, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/agent_trace_observability/PAGE ON-CALL": { + "count": 5, + "hard_count": 5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.009523417976595696, + "brier": 0.00024950413084922463, + "expected_brier": 0.00024950413084914305, + "multiclass_ece": 0.009460434176379451, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 0.4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/agent_trace_observability/PRIORITY REVIEW": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.1512885125034061, + "brier": 0.08374549487020937, + "expected_brier": 0.08374549487020928, + "multiclass_ece": 0.12358136171899503, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 0.3333333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service/CANCEL": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0009121638681814813, + "brier": 1.6626039522816305e-06, + "expected_brier": 1.6626039522904534e-06, + "multiclass_ece": 0.0009117479643638848, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service/CLOSE": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00518135817729126, + "brier": 6.510601557656118e-05, + "expected_brier": 6.510601557652151e-05, + "multiclass_ece": 0.005165008372302567, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service/FLAG FOR REVIEW": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.012839939879918239, + "brier": 0.0004779390892691936, + "expected_brier": 0.00047793908926922296, + "multiclass_ece": 0.012718884126595498, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 0.3333333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service/FREEZE CARD": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0005783300751288652, + "brier": 6.686593430617063e-07, + "expected_brier": 6.686593430060839e-07, + "multiclass_ece": 0.0005781628458107294, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service/HAND OFF": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.005332959903268297, + "brier": 5.657852049688965e-05, + "expected_brier": 5.6578520496897156e-05, + "multiclass_ece": 0.005318764917576768, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service/REFUND": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0006575557742752308, + "brier": 8.64525520856754e-07, + "expected_brier": 8.645255208753847e-07, + "multiclass_ece": 0.0006573395480608349, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service/SAY": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.007818247394437192, + "brier": 0.0001221958295181014, + "expected_brier": 0.00012219582951805696, + "multiclass_ece": 0.007787536608491963, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service/SET INTENT": { + "count": 5, + "hard_count": 5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.01969575016676005, + "brier": 0.0011382079317770947, + "expected_brier": 0.00113820793177708, + "multiclass_ece": 0.019404759521718697, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 0.2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/DISPUTE LINES": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0004080892675768517, + "brier": 3.329378091715434e-07, + "expected_brier": 3.329378090999313e-07, + "multiclass_ece": 0.00040800601047752405, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/DUPLICATE": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0004942981450790142, + "brier": 4.884198377091359e-07, + "expected_brier": 4.884198376853277e-07, + "multiclass_ece": 0.0004941759998771111, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/FRAUD REVIEW": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0005777794141664189, + "brier": 6.888878501890544e-07, + "expected_brier": 6.88887850153197e-07, + "multiclass_ece": 0.0005776071212669764, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/HOLD": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.017053282896678554, + "brier": 0.0006326897088602082, + "expected_brier": 0.0006326897088601569, + "multiclass_ece": 0.01689294217630055, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/HOLD FOR DOCUMENTS": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0005178357346687126, + "brier": 5.360300607821365e-07, + "expected_brier": 5.360300607826218e-07, + "multiclass_ece": 0.0005177016808849411, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/PAY": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.03636925685729642, + "brier": 0.006885067243695091, + "expected_brier": 0.006885067243695087, + "multiclass_ece": 0.03452173513394415, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/REQUEST CORRECTED INVOICE": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0007294130314709884, + "brier": 1.4235206266035047e-06, + "expected_brier": 1.4235206265977851e-06, + "multiclass_ece": 0.00072905689051872, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/ROUTE FOR APPROVAL": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.017554235265073766, + "brier": 0.002120371928854703, + "expected_brier": 0.002120371928854725, + "multiclass_ece": 0.016999967881545786, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 0.75, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 0.75, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/SCHEDULE": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0014168545780722257, + "brier": 4.015858087671432e-06, + "expected_brier": 4.015858087624835e-06, + "multiclass_ece": 0.0014158496617789895, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/SHORT PAY": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0011234717560948209, + "brier": 2.89294577084466e-06, + "expected_brier": 2.8929457709381268e-06, + "multiclass_ece": 0.0011227478383988299, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/BLOCK EGRESS ASSET": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0010478341814348922, + "brier": 2.353811903385862e-06, + "expected_brier": 2.3538119033143326e-06, + "multiclass_ece": 0.0010472452665183003, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/BLOCK SENDER": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0015381079443832647, + "brier": 4.7242809846904296e-06, + "expected_brier": 4.7242809846359535e-06, + "multiclass_ece": 0.0015369256625956673, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/DISABLE ACCOUNT": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0008983301425278225, + "brier": 1.799677740787705e-06, + "expected_brier": 1.7996777407214282e-06, + "multiclass_ece": 0.0008978798974407232, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/ESCALATE TIER2": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0012416980666806574, + "brier": 3.0971744961121014e-06, + "expected_brier": 3.0971744960917036e-06, + "multiclass_ece": 0.0012409231246889307, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/ESCALATE URGENT": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.011694361771501035, + "brier": 0.0006897902203353569, + "expected_brier": 0.000689790220335329, + "multiclass_ece": 0.011518143589805518, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/ISOLATE HOST": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0011056934465739495, + "brier": 2.780395022008406e-06, + "expected_brier": 2.780395022069134e-06, + "multiclass_ece": 0.0011049977221690988, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/KILL PROCESS": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0013686750748221372, + "brier": 3.7414192125695984e-06, + "expected_brier": 3.7414192126661305e-06, + "multiclass_ece": 0.0013677388662624024, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/NOTIFY USER": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0006987632196573655, + "brier": 1.0297004954907962e-06, + "expected_brier": 1.02970049553619e-06, + "multiclass_ece": 0.000698505662033555, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/QUARANTINE FILE": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0009403651532668032, + "brier": 1.76691105023072e-06, + "expected_brier": 1.7669110501383756e-06, + "multiclass_ece": 0.0009399231485155424, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/QUARANTINE MESSAGE": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.005035006803622096, + "brier": 8.60435340074367e-05, + "expected_brier": 8.60435340075405e-05, + "multiclass_ece": 0.005013363320549802, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/REMOVE FORWARDING RULES": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.002569828758201726, + "brier": 1.5345580700663495e-05, + "expected_brier": 1.534558070059866e-05, + "multiclass_ece": 0.0025659839162491815, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/REVOKE ACCESS KEY": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.000576420123263373, + "brier": 6.705872999049699e-07, + "expected_brier": 6.705872999113183e-07, + "multiclass_ece": 0.0005762524107649769, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/REVOKE SESSIONS": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0009620077238914374, + "brier": 1.849138123317009e-06, + "expected_brier": 1.849138123288796e-06, + "multiclass_ece": 0.0009615451428084398, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + } + }, + "calibrated_ood": { + "count": 512, + "hard_count": 484, + "accuracy": 0.878099173553719, + "expected_accuracy": 0.8438422309027778, + "nll": 0.5359452791103922, + "brier": 0.19770581826822115, + "expected_brier": 0.23491827486544328, + "multiclass_ece": 0.07848831403978263, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 467, + "coverage": 0.912109375, + "accuracy": 0.8978260869565218, + "expected_accuracy": 0.8918629550321199, + "expected_risk": 0.10813704496788012 + }, + { + "threshold": 0.7, + "selected": 451, + "coverage": 0.880859375, + "accuracy": 0.9068736141906873, + "expected_accuracy": 0.9068736141906873, + "expected_risk": 0.09312638580931265 + }, + { + "threshold": 0.8, + "selected": 438, + "coverage": 0.85546875, + "accuracy": 0.9223744292237442, + "expected_accuracy": 0.9223744292237442, + "expected_risk": 0.07762557077625576 + }, + { + "threshold": 0.9, + "selected": 419, + "coverage": 0.818359375, + "accuracy": 0.9236276849642004, + "expected_accuracy": 0.9236276849642004, + "expected_risk": 0.07637231503579955 + }, + { + "threshold": 0.95, + "selected": 382, + "coverage": 0.74609375, + "accuracy": 0.9424083769633508, + "expected_accuracy": 0.9424083769633508, + "expected_risk": 0.05759162303664922 + }, + { + "threshold": 0.99, + "selected": 285, + "coverage": 0.556640625, + "accuracy": 0.9614035087719298, + "expected_accuracy": 0.9614035087719298, + "expected_risk": 0.038596491228070184 + } + ], + "mean_latency_seconds": 0.038995993836579146, + "by_kind": { + "choice": { + "count": 162, + "hard_count": 136, + "accuracy": 0.8235294117647058, + "expected_accuracy": 0.7307441700960221, + "nll": 0.8468075975538774, + "brier": 0.25056108033831115, + "expected_brier": 0.3599403670324125, + "multiclass_ece": 0.09590698229597418, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 126, + "coverage": 0.7777777777777778, + "accuracy": 0.8823529411764706, + "expected_accuracy": 0.8611111111111112, + "expected_risk": 0.13888888888888884 + }, + { + "threshold": 0.7, + "selected": 115, + "coverage": 0.7098765432098766, + "accuracy": 0.8956521739130435, + "expected_accuracy": 0.8956521739130435, + "expected_risk": 0.10434782608695647 + }, + { + "threshold": 0.8, + "selected": 108, + "coverage": 0.6666666666666666, + "accuracy": 0.9259259259259259, + "expected_accuracy": 0.9259259259259259, + "expected_risk": 0.07407407407407407 + }, + { + "threshold": 0.9, + "selected": 101, + "coverage": 0.6234567901234568, + "accuracy": 0.9207920792079208, + "expected_accuracy": 0.9207920792079208, + "expected_risk": 0.07920792079207917 + }, + { + "threshold": 0.95, + "selected": 88, + "coverage": 0.5432098765432098, + "accuracy": 0.9318181818181818, + "expected_accuracy": 0.9318181818181818, + "expected_risk": 0.06818181818181823 + }, + { + "threshold": 0.99, + "selected": 56, + "coverage": 0.345679012345679, + "accuracy": 0.9464285714285714, + "expected_accuracy": 0.9464285714285714, + "expected_risk": 0.0535714285714286 + } + ] + }, + "noul": { + "count": 243, + "hard_count": 243, + "accuracy": 0.8930041152263375, + "expected_accuracy": 0.8930041152263375, + "nll": 0.3619478932713637, + "brier": 0.1781279493659772, + "expected_brier": 0.1781279493659772, + "multiclass_ece": 0.08315592856424382, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 243, + "coverage": 1.0, + "accuracy": 0.8930041152263375, + "expected_accuracy": 0.8930041152263375, + "expected_risk": 0.10699588477366251 + }, + { + "threshold": 0.7, + "selected": 239, + "coverage": 0.9835390946502057, + "accuracy": 0.9037656903765691, + "expected_accuracy": 0.9037656903765691, + "expected_risk": 0.09623430962343094 + }, + { + "threshold": 0.8, + "selected": 234, + "coverage": 0.9629629629629629, + "accuracy": 0.9145299145299145, + "expected_accuracy": 0.9145299145299145, + "expected_risk": 0.0854700854700855 + }, + { + "threshold": 0.9, + "selected": 229, + "coverage": 0.9423868312757202, + "accuracy": 0.9213973799126638, + "expected_accuracy": 0.9213973799126638, + "expected_risk": 0.0786026200873362 + }, + { + "threshold": 0.95, + "selected": 218, + "coverage": 0.897119341563786, + "accuracy": 0.9403669724770642, + "expected_accuracy": 0.9403669724770642, + "expected_risk": 0.059633027522935755 + }, + { + "threshold": 0.99, + "selected": 171, + "coverage": 0.7037037037037037, + "accuracy": 0.9649122807017544, + "expected_accuracy": 0.9649122807017544, + "expected_risk": 0.03508771929824561 + } + ] + }, + "score": { + "count": 107, + "hard_count": 105, + "accuracy": 0.9142857142857143, + "expected_accuracy": 0.9034267912772586, + "nll": 0.4604468601481423, + "brier": 0.16214385273448972, + "expected_brier": 0.17460491192452085, + "multiclass_ece": 0.07826528899195538, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 98, + "coverage": 0.9158878504672897, + "accuracy": 0.9285714285714286, + "expected_accuracy": 0.9285714285714286, + "expected_risk": 0.0714285714285714 + }, + { + "threshold": 0.7, + "selected": 97, + "coverage": 0.9065420560747663, + "accuracy": 0.9278350515463918, + "expected_accuracy": 0.9278350515463918, + "expected_risk": 0.07216494845360821 + }, + { + "threshold": 0.8, + "selected": 96, + "coverage": 0.897196261682243, + "accuracy": 0.9375, + "expected_accuracy": 0.9375, + "expected_risk": 0.0625 + }, + { + "threshold": 0.9, + "selected": 89, + "coverage": 0.8317757009345794, + "accuracy": 0.9325842696629213, + "expected_accuracy": 0.9325842696629213, + "expected_risk": 0.0674157303370787 + }, + { + "threshold": 0.95, + "selected": 76, + "coverage": 0.7102803738317757, + "accuracy": 0.9605263157894737, + "expected_accuracy": 0.9605263157894737, + "expected_risk": 0.03947368421052633 + }, + { + "threshold": 0.99, + "selected": 58, + "coverage": 0.5420560747663551, + "accuracy": 0.9655172413793104, + "expected_accuracy": 0.9655172413793104, + "expected_risk": 0.03448275862068961 + } + ], + "ordinal_mae": 0.1692529081409676 + } + }, + "by_source": { + "customer-control-v1": { + "count": 81, + "hard_count": 77, + "accuracy": 0.922077922077922, + "expected_accuracy": 0.8971193415637859, + "nll": 0.36209434937184415, + "brier": 0.1461303721410015, + "expected_brier": 0.17493695650314145, + "multiclass_ece": 0.08037632143222284, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 77, + "coverage": 0.9506172839506173, + "accuracy": 0.9210526315789473, + "expected_accuracy": 0.9155844155844156, + "expected_risk": 0.08441558441558439 + }, + { + "threshold": 0.7, + "selected": 76, + "coverage": 0.9382716049382716, + "accuracy": 0.9210526315789473, + "expected_accuracy": 0.9210526315789473, + "expected_risk": 0.07894736842105265 + }, + { + "threshold": 0.8, + "selected": 76, + "coverage": 0.9382716049382716, + "accuracy": 0.9210526315789473, + "expected_accuracy": 0.9210526315789473, + "expected_risk": 0.07894736842105265 + }, + { + "threshold": 0.9, + "selected": 75, + "coverage": 0.9259259259259259, + "accuracy": 0.92, + "expected_accuracy": 0.92, + "expected_risk": 0.07999999999999996 + }, + { + "threshold": 0.95, + "selected": 73, + "coverage": 0.9012345679012346, + "accuracy": 0.9452054794520548, + "expected_accuracy": 0.9452054794520548, + "expected_risk": 0.0547945205479452 + }, + { + "threshold": 0.99, + "selected": 65, + "coverage": 0.8024691358024691, + "accuracy": 0.9692307692307692, + "expected_accuracy": 0.9692307692307692, + "expected_risk": 0.03076923076923077 + } + ] + }, + "painting-geometry-v1": { + "count": 81, + "hard_count": 81, + "accuracy": 0.8765432098765432, + "expected_accuracy": 0.8765432098765432, + "nll": 0.8644029546339671, + "brier": 0.2432393409722228, + "expected_brier": 0.24323934097222288, + "multiclass_ece": 0.11049760383721956, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 81, + "coverage": 1.0, + "accuracy": 0.8765432098765432, + "expected_accuracy": 0.8765432098765432, + "expected_risk": 0.12345679012345678 + }, + { + "threshold": 0.7, + "selected": 81, + "coverage": 1.0, + "accuracy": 0.8765432098765432, + "expected_accuracy": 0.8765432098765432, + "expected_risk": 0.12345679012345678 + }, + { + "threshold": 0.8, + "selected": 81, + "coverage": 1.0, + "accuracy": 0.8765432098765432, + "expected_accuracy": 0.8765432098765432, + "expected_risk": 0.12345679012345678 + }, + { + "threshold": 0.9, + "selected": 79, + "coverage": 0.9753086419753086, + "accuracy": 0.8734177215189873, + "expected_accuracy": 0.8734177215189873, + "expected_risk": 0.12658227848101267 + }, + { + "threshold": 0.95, + "selected": 73, + "coverage": 0.9012345679012346, + "accuracy": 0.8767123287671232, + "expected_accuracy": 0.8767123287671232, + "expected_risk": 0.12328767123287676 + }, + { + "threshold": 0.99, + "selected": 61, + "coverage": 0.7530864197530864, + "accuracy": 0.8688524590163934, + "expected_accuracy": 0.8688524590163934, + "expected_risk": 0.1311475409836066 + } + ] + }, + "reasoning-control-v1": { + "count": 80, + "hard_count": 80, + "accuracy": 0.675, + "expected_accuracy": 0.675, + "nll": 0.8574726225169125, + "brier": 0.48196296595493787, + "expected_brier": 0.48196296595493787, + "multiclass_ece": 0.17487304367327164, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 66, + "coverage": 0.825, + "accuracy": 0.7272727272727273, + "expected_accuracy": 0.7272727272727273, + "expected_risk": 0.2727272727272727 + }, + { + "threshold": 0.7, + "selected": 62, + "coverage": 0.775, + "accuracy": 0.7419354838709677, + "expected_accuracy": 0.7419354838709677, + "expected_risk": 0.25806451612903225 + }, + { + "threshold": 0.8, + "selected": 53, + "coverage": 0.6625, + "accuracy": 0.7924528301886793, + "expected_accuracy": 0.7924528301886793, + "expected_risk": 0.2075471698113207 + }, + { + "threshold": 0.9, + "selected": 46, + "coverage": 0.575, + "accuracy": 0.8043478260869565, + "expected_accuracy": 0.8043478260869565, + "expected_risk": 0.19565217391304346 + }, + { + "threshold": 0.95, + "selected": 36, + "coverage": 0.45, + "accuracy": 0.8888888888888888, + "expected_accuracy": 0.8888888888888888, + "expected_risk": 0.11111111111111116 + }, + { + "threshold": 0.99, + "selected": 18, + "coverage": 0.225, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "snake-v1": { + "count": 54, + "hard_count": 48, + "accuracy": 0.9166666666666666, + "expected_accuracy": 0.8703703703703703, + "nll": 0.33487800609716084, + "brier": 0.13345654167556611, + "expected_brier": 0.18901209723112172, + "multiclass_ece": 0.07203390722559798, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 54, + "coverage": 1.0, + "accuracy": 0.9166666666666666, + "expected_accuracy": 0.8703703703703703, + "expected_risk": 0.12962962962962965 + }, + { + "threshold": 0.7, + "selected": 46, + "coverage": 0.8518518518518519, + "accuracy": 0.9347826086956522, + "expected_accuracy": 0.9347826086956522, + "expected_risk": 0.06521739130434778 + }, + { + "threshold": 0.8, + "selected": 46, + "coverage": 0.8518518518518519, + "accuracy": 0.9347826086956522, + "expected_accuracy": 0.9347826086956522, + "expected_risk": 0.06521739130434778 + }, + { + "threshold": 0.9, + "selected": 41, + "coverage": 0.7592592592592593, + "accuracy": 0.926829268292683, + "expected_accuracy": 0.926829268292683, + "expected_risk": 0.07317073170731703 + }, + { + "threshold": 0.95, + "selected": 29, + "coverage": 0.5370370370370371, + "accuracy": 0.9310344827586207, + "expected_accuracy": 0.9310344827586207, + "expected_risk": 0.06896551724137934 + }, + { + "threshold": 0.99, + "selected": 8, + "coverage": 0.14814814814814814, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "vizdoom-basic-v1": { + "count": 81, + "hard_count": 81, + "accuracy": 0.9135802469135802, + "expected_accuracy": 0.9135802469135802, + "nll": 0.2595076732983789, + "brier": 0.12942038247069337, + "expected_brier": 0.12942038247069335, + "multiclass_ece": 0.05732790794961664, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 81, + "coverage": 1.0, + "accuracy": 0.9135802469135802, + "expected_accuracy": 0.9135802469135802, + "expected_risk": 0.0864197530864198 + }, + { + "threshold": 0.7, + "selected": 78, + "coverage": 0.9629629629629629, + "accuracy": 0.9358974358974359, + "expected_accuracy": 0.9358974358974359, + "expected_risk": 0.0641025641025641 + }, + { + "threshold": 0.8, + "selected": 74, + "coverage": 0.9135802469135802, + "accuracy": 0.972972972972973, + "expected_accuracy": 0.972972972972973, + "expected_risk": 0.027027027027026973 + }, + { + "threshold": 0.9, + "selected": 71, + "coverage": 0.8765432098765432, + "accuracy": 0.971830985915493, + "expected_accuracy": 0.971830985915493, + "expected_risk": 0.028169014084507005 + }, + { + "threshold": 0.95, + "selected": 66, + "coverage": 0.8148148148148148, + "accuracy": 0.9848484848484849, + "expected_accuracy": 0.9848484848484849, + "expected_risk": 0.015151515151515138 + }, + { + "threshold": 0.99, + "selected": 46, + "coverage": 0.5679012345679012, + "accuracy": 0.9782608695652174, + "expected_accuracy": 0.9782608695652174, + "expected_risk": 0.021739130434782594 + } + ] + }, + "wikispeedia-v1": { + "count": 27, + "hard_count": 9, + "accuracy": 0.5555555555555556, + "expected_accuracy": 0.27335390946502053, + "nll": 2.21792246668928, + "brier": 0.35760252451643976, + "expected_brier": 0.8657300965329005, + "multiclass_ece": 0.15134543733309497, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ], + "optimal_action_hit": 0.6666666666666666, + "optimal_set_probability_mass": 0.3990037955619308 + }, + "workflow-controls-v1/agent_trace_observability": { + "count": 27, + "hard_count": 27, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0033803454342630733, + "brier": 7.61918821998959e-05, + "expected_brier": 7.619188219988017e-05, + "multiclass_ece": 0.003361086984585671, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 24, + "coverage": 0.8888888888888888, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service": { + "count": 27, + "hard_count": 27, + "accuracy": 0.9259259259259259, + "expected_accuracy": 0.9259259259259259, + "nll": 0.267069201636269, + "brier": 0.139881675493404, + "expected_brier": 0.13988167549340402, + "multiclass_ece": 0.07393707191836439, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.9259259259259259, + "expected_accuracy": 0.9259259259259259, + "expected_risk": 0.07407407407407407 + }, + { + "threshold": 0.7, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.9259259259259259, + "expected_accuracy": 0.9259259259259259, + "expected_risk": 0.07407407407407407 + }, + { + "threshold": 0.8, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.9259259259259259, + "expected_accuracy": 0.9259259259259259, + "expected_risk": 0.07407407407407407 + }, + { + "threshold": 0.9, + "selected": 26, + "coverage": 0.9629629629629629, + "accuracy": 0.9230769230769231, + "expected_accuracy": 0.9230769230769231, + "expected_risk": 0.07692307692307687 + }, + { + "threshold": 0.95, + "selected": 24, + "coverage": 0.8888888888888888, + "accuracy": 0.9166666666666666, + "expected_accuracy": 0.9166666666666666, + "expected_risk": 0.08333333333333337 + }, + { + "threshold": 0.99, + "selected": 15, + "coverage": 0.5555555555555556, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing": { + "count": 27, + "hard_count": 27, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0045240194592861, + "brier": 0.00017215208623737115, + "expected_brier": 0.00017215208623735215, + "multiclass_ece": 0.004480215263243615, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 22, + "coverage": 0.8148148148148148, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents": { + "count": 27, + "hard_count": 27, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0017838783468913614, + "brier": 3.377765713728113e-05, + "expected_brier": 3.3777657137285185e-05, + "multiclass_ece": 0.0017753259282021307, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 26, + "coverage": 0.9629629629629629, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "by_question": { + "customer-control-v1/bug_severity": { + "count": 8, + "hard_count": 8, + "accuracy": 0.875, + "expected_accuracy": 0.875, + "nll": 0.5031613577428611, + "brier": 0.23076052709666706, + "expected_brier": 0.23076052709666703, + "multiclass_ece": 0.11631322528025367, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 8, + "coverage": 1.0, + "accuracy": 0.875, + "expected_accuracy": 0.875, + "expected_risk": 0.125 + }, + { + "threshold": 0.7, + "selected": 8, + "coverage": 1.0, + "accuracy": 0.875, + "expected_accuracy": 0.875, + "expected_risk": 0.125 + }, + { + "threshold": 0.8, + "selected": 8, + "coverage": 1.0, + "accuracy": 0.875, + "expected_accuracy": 0.875, + "expected_risk": 0.125 + }, + { + "threshold": 0.9, + "selected": 8, + "coverage": 1.0, + "accuracy": 0.875, + "expected_accuracy": 0.875, + "expected_risk": 0.125 + }, + { + "threshold": 0.95, + "selected": 7, + "coverage": 0.875, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 7, + "coverage": 0.875, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "customer-control-v1/category": { + "count": 27, + "hard_count": 25, + "accuracy": 1.0, + "expected_accuracy": 0.9629629629629629, + "nll": 0.058439409721856066, + "brier": 0.001087817794321304, + "expected_brier": 0.03812485483135833, + "multiclass_ece": 0.007475497850178635, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 26, + "coverage": 0.9629629629629629, + "accuracy": 1.0, + "expected_accuracy": 0.9807692307692307, + "expected_risk": 0.019230769230769273 + }, + { + "threshold": 0.7, + "selected": 25, + "coverage": 0.9259259259259259, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 25, + "coverage": 0.9259259259259259, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 25, + "coverage": 0.9259259259259259, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 25, + "coverage": 0.9259259259259259, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 23, + "coverage": 0.8518518518518519, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "customer-control-v1/churn_likelihood_level": { + "count": 9, + "hard_count": 7, + "accuracy": 1.0, + "expected_accuracy": 0.8518518518518517, + "nll": 0.34498713571468725, + "brier": 0.0525999846025417, + "expected_brier": 0.20074813275068984, + "multiclass_ece": 0.09261389607789626, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 6, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 6, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 6, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 5, + "coverage": 0.5555555555555556, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 5, + "coverage": 0.5555555555555556, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 5, + "coverage": 0.5555555555555556, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "customer-control-v1/frustration": { + "count": 10, + "hard_count": 10, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.010145630204588254, + "brier": 0.00027803713653107656, + "expected_brier": 0.000278037136531073, + "multiclass_ece": 0.010072591650383345, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 10, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 10, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 10, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 10, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 10, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 6, + "coverage": 0.6, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "customer-control-v1/has_reproducible_steps": { + "count": 15, + "hard_count": 15, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "nll": 0.6556572226538439, + "brier": 0.36824316899895726, + "expected_brier": 0.3682431689989573, + "multiclass_ece": 0.19076033468922268, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 15, + "coverage": 1.0, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.7, + "selected": 15, + "coverage": 1.0, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.8, + "selected": 15, + "coverage": 1.0, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.9, + "selected": 15, + "coverage": 1.0, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.95, + "selected": 14, + "coverage": 0.9333333333333333, + "accuracy": 0.8571428571428571, + "expected_accuracy": 0.8571428571428571, + "expected_risk": 0.1428571428571429 + }, + { + "threshold": 0.99, + "selected": 12, + "coverage": 0.8, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "customer-control-v1/refund_requested": { + "count": 12, + "hard_count": 12, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "nll": 0.8904407092833865, + "brier": 0.3301064232023802, + "expected_brier": 0.3301064232023803, + "multiclass_ece": 0.1651356172787913, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 12, + "coverage": 1.0, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.7, + "selected": 12, + "coverage": 1.0, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.8, + "selected": 12, + "coverage": 1.0, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.9, + "selected": 12, + "coverage": 1.0, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.95, + "selected": 12, + "coverage": 1.0, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.99, + "selected": 12, + "coverage": 1.0, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + } + ] + }, + "painting-geometry-v1/x0_y11_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00017408586529914683, + "brier": 6.060122641168731e-08, + "expected_brier": 6.060122648143818e-08, + "multiclass_ece": 0.00017407071323416456, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x0_y11_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.08054897407373712, + "brier": 0.008583177609755303, + "expected_brier": 0.008583177609755355, + "multiclass_ece": 0.07739028147906812, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x0_y3_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00014276364272728406, + "brier": 4.075709639186337e-08, + "expected_brier": 4.0757096364529843e-08, + "multiclass_ece": 0.00014275345248337956, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x0_y8": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0029874300501322613, + "brier": 1.779624535332517e-05, + "expected_brier": 1.7796245353363105e-05, + "multiclass_ece": 0.002982972121335137, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x0_y8_red": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.004411536510380496, + "brier": 3.875203821492971e-05, + "expected_brier": 3.875203821501749e-05, + "multiclass_ece": 0.0044018199767215105, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x10_y0_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.1346397240080695, + "brier": 0.02171794971930731, + "expected_brier": 0.021717949719307406, + "multiclass_ece": 0.12596925273477222, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x10_y11": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0007178445909014128, + "brier": 6.499789499461788e-07, + "expected_brier": 6.499789499958553e-07, + "multiclass_ece": 0.000717587002112996, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x10_y4_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.000318873163494841, + "brier": 2.0329535474800538e-07, + "expected_brier": 2.0329535477259242e-07, + "multiclass_ece": 0.0003188223288510539, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x10_y4_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0007151352837912693, + "brier": 9.137042813840454e-07, + "expected_brier": 9.137042813467389e-07, + "multiclass_ece": 0.0007148796354988773, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x10_y5_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 9.89273355511167, + "brier": 1.9994284393661956, + "expected_brier": 1.9994284393661959, + "multiclass_ece": 0.9997647142623245, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + } + ] + }, + "painting-geometry-v1/x10_y7_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0180909608722001, + "brier": 0.0004736499460429417, + "expected_brier": 0.0004736499460429311, + "multiclass_ece": 0.01792830180282534, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x10_y9_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0008026897447410669, + "brier": 1.2875877733987983e-06, + "expected_brier": 1.2875877732909657e-06, + "multiclass_ece": 0.0008023676755075648, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x10_y9_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0005581277450500179, + "brier": 4.6912378999479106e-07, + "expected_brier": 4.6912379003671845e-07, + "multiclass_ece": 0.000557972020732822, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x11_y0_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 7.004756817067593e-05, + "brier": 6.29510843771281e-09, + "expected_brier": 6.295108434528629e-09, + "multiclass_ece": 7.00451148970549e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x11_y10": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 9.159895109842266e-05, + "brier": 9.947467909452428e-09, + "expected_brier": 9.947467960458312e-09, + "multiclass_ece": 9.159475604259004e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x11_y1_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.06169033094472852, + "brier": 0.006409768135490834, + "expected_brier": 0.006409768135490879, + "multiclass_ece": 0.05982601549364164, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y0": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.003307040017781848, + "brier": 1.777080020409896e-05, + "expected_brier": 1.7770800204042203e-05, + "multiclass_ece": 0.0033015777838766303, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y10_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0053190396913161435, + "brier": 5.62843237150599e-05, + "expected_brier": 5.6284323715072304e-05, + "multiclass_ece": 0.005304918647588308, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y3_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.02851564388618157, + "brier": 0.001578850192772401, + "expected_brier": 0.0015788501927724852, + "multiclass_ece": 0.028112910063805896, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y4_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00013890616884204834, + "brier": 3.025637299020543e-08, + "expected_brier": 3.025637296261152e-08, + "multiclass_ece": 0.00013889652182685897, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y6_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.021920222809948084, + "brier": 0.0009081751667739688, + "expected_brier": 0.0009081751667738835, + "multiclass_ece": 0.02168172057860296, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y7_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.011416006316916055, + "brier": 0.00021512270180724973, + "expected_brier": 0.00021512270180723547, + "multiclass_ece": 0.011351090976272293, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x1_y7_red": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00587838098088239, + "brier": 6.8705856318798e-05, + "expected_brier": 6.870585631890691e-05, + "multiclass_ece": 0.005861137104640979, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y8_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 6.496367307346462e-05, + "brier": 5.8505517477196136e-09, + "expected_brier": 5.850551820785199e-09, + "multiclass_ece": 6.496156297974842e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y9_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.004041352233012237, + "brier": 3.253335541835517e-05, + "expected_brier": 3.253335541830715e-05, + "multiclass_ece": 0.004033196958887264, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x1_y9_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0003574477114489097, + "brier": 2.001188374507725e-07, + "expected_brier": 2.001188375011509e-07, + "multiclass_ece": 0.000357383834626801, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y10_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.031781128128237794, + "brier": 0.0014079458865382762, + "expected_brier": 0.0014079458865383643, + "multiclass_ece": 0.03128141587257249, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y11_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0030561561873356157, + "brier": 1.8623193374848484e-05, + "expected_brier": 1.8623193374756042e-05, + "multiclass_ece": 0.0030514908958448883, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y4_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.032259533371785776, + "brier": 0.0014384553788028042, + "expected_brier": 0.0014384553788027166, + "multiclass_ece": 0.03174474508434233, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x2_y6_red": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0023186166638966216, + "brier": 1.0727070464158755e-05, + "expected_brier": 1.072707046412269e-05, + "multiclass_ece": 0.002315930748549966, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y6_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00034374061167763843, + "brier": 2.1948728125209747e-07, + "expected_brier": 2.1948728123888372e-07, + "multiclass_ece": 0.0003436815396422599, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x2_y8": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0014648250741152633, + "brier": 6.542199306979801e-06, + "expected_brier": 6.542199306958096e-06, + "multiclass_ece": 0.0014627361529970795, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x3_y0_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.03380525472577725, + "brier": 0.0016218788757316302, + "expected_brier": 0.0016218788757316194, + "multiclass_ece": 0.03324024179961593, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x3_y11_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0003117554457796769, + "brier": 1.94322327033078e-07, + "expected_brier": 1.9432232711835695e-07, + "multiclass_ece": 0.00031170685510029106, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x3_y1_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0002227990461434138, + "brier": 9.92567135706775e-08, + "expected_brier": 9.925671351496135e-08, + "multiclass_ece": 0.00022277422827909898, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x3_y6": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 9.960312109419946e-05, + "brier": 1.351596830511459e-08, + "expected_brier": 1.3515968344535167e-08, + "multiclass_ece": 9.959816086801965e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x3_y7_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00032437840612811, + "brier": 2.1037445057155043e-07, + "expected_brier": 2.1037445052662918e-07, + "multiclass_ece": 0.0003243258011410566, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x3_y7_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0004171968069686142, + "brier": 1.963723239807423e-07, + "expected_brier": 1.9637232395730564e-07, + "multiclass_ece": 0.0004171097924818845, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x4_y10_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0016970598017416844, + "brier": 4.901642804852906e-06, + "expected_brier": 4.901642804755468e-06, + "multiclass_ece": 0.0016956206100029458, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x4_y11_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.010954940182284484, + "brier": 0.00019287287364145117, + "expected_brier": 0.00019287287364155858, + "multiclass_ece": 0.010895153344657293, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y1_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0013940738502926594, + "brier": 3.149058586210884e-06, + "expected_brier": 3.149058586116027e-06, + "multiclass_ece": 0.0013931025807355502, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x4_y1_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.002142821013676754, + "brier": 9.078734538377765e-06, + "expected_brier": 9.07873453837027e-06, + "multiclass_ece": 0.0021405268117090648, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x4_y4_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.08119847538683407, + "brier": 0.012148739561105376, + "expected_brier": 0.012148739561105293, + "multiclass_ece": 0.07798932314249107, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x4_y5_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00036413690705318146, + "brier": 2.4149100763535685e-07, + "expected_brier": 2.414910076353749e-07, + "multiclass_ece": 0.0003640706172560737, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x4_y6_red": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.003202778549176705, + "brier": 2.0449996598790872e-05, + "expected_brier": 2.044999659878144e-05, + "multiclass_ece": 0.003197655125149579, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x5_y1_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.09294665536286804, + "brier": 0.011079230386460423, + "expected_brier": 0.011079230386460415, + "multiclass_ece": 0.08875789112073706, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y2": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 9.95339394559801, + "brier": 1.9996852784816506, + "expected_brier": 1.9996852784816506, + "multiclass_ece": 0.9998901976807382, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + } + ] + }, + "painting-geometry-v1/x5_y4_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0011726960365928967, + "brier": 1.7473321037978151e-06, + "expected_brier": 1.7473321038963263e-06, + "multiclass_ece": 0.0011720086973020516, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x5_y4_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.02867402627737055, + "brier": 0.0011268868179075903, + "expected_brier": 0.001126886817907513, + "multiclass_ece": 0.028266827675834216, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x5_y7": { + "count": 2, + "hard_count": 2, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "nll": 4.084989772853176, + "brier": 0.9992514586255856, + "expected_brier": 0.9992514586255857, + "multiclass_ece": 0.4992063538899436, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 0.5, + "expected_accuracy": 0.5, + "expected_risk": 0.5 + } + ] + }, + "painting-geometry-v1/x5_y8": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 6.01212467796658, + "brier": 1.990216475747884, + "expected_brier": 1.990216475747884, + "multiclass_ece": 0.9975511204314003, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + } + ] + }, + "painting-geometry-v1/x5_y9": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00014422523791354503, + "brier": 2.521377353198005e-08, + "expected_brier": 2.5213773513499405e-08, + "multiclass_ece": 0.00014421483795390433, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x5_y9_red": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0008265495388906856, + "brier": 1.3652394537465062e-06, + "expected_brier": 1.365239453710032e-06, + "multiclass_ece": 0.0008262080409153771, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x6_y0_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00010465654015825381, + "brier": 1.5361235886603984e-08, + "expected_brier": 1.5361235838895482e-08, + "multiclass_ece": 0.00010465106385360023, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x6_y1_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.025962402613956066, + "brier": 0.0012851508570936108, + "expected_brier": 0.0012851508570936865, + "multiclass_ece": 0.02562827725009964, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y3": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 4.598798602336495e-05, + "brier": 2.4927730156011822e-09, + "expected_brier": 2.4927730990498276e-09, + "multiclass_ece": 4.598692859214548e-05, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x6_y3_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.000215940223906848, + "brier": 9.324022447524861e-08, + "expected_brier": 9.32402244302466e-08, + "multiclass_ece": 0.00021591691049482886, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x6_y6_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 10.303745993843604, + "brier": 1.9998064971788665, + "expected_brier": 1.9998064971788665, + "multiclass_ece": 0.9999367531562324, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + } + ] + }, + "painting-geometry-v1/x6_y7_lightness": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.030687149044726133, + "brier": 0.0013657535180190174, + "expected_brier": 0.0013657535180189306, + "multiclass_ece": 0.030221078115814537, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x6_y8_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 6.637564096782521, + "brier": 1.9947625736600154, + "expected_brier": 1.9947625736600152, + "multiclass_ece": 0.9986897850834401, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + } + ] + }, + "painting-geometry-v1/x7_y1_lightness": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0334368756341766, + "brier": 0.0035018425045524816, + "expected_brier": 0.0035018425045524326, + "multiclass_ece": 0.03234680250487232, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x7_y2": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.005236602457824964, + "brier": 5.455768965065393e-05, + "expected_brier": 5.455768965068053e-05, + "multiclass_ece": 0.005222915356898561, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x7_y5_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 4.823191700920016, + "brier": 1.967964992334591, + "expected_brier": 1.9679649923345908, + "multiclass_ece": 0.9919589185885147, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + } + ] + }, + "painting-geometry-v1/x7_y6_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.004681691165093955, + "brier": 3.546693860429264e-05, + "expected_brier": 3.546693860423389e-05, + "multiclass_ece": 0.0046707491314119665, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x7_y7": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 2.5577924215063272, + "brier": 1.7021025916823174, + "expected_brier": 1.7021025916823174, + "multiclass_ece": 0.9225244147669799, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x7_y7_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00035765451514144643, + "brier": 2.5574202348100827e-07, + "expected_brier": 2.557420235582697e-07, + "multiclass_ece": 0.00035759056438966486, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x7_y8": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0023769950302316794, + "brier": 1.1273387409100797e-05, + "expected_brier": 1.1273387409138635e-05, + "multiclass_ece": 0.002374172214594017, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x7_y9_saturation": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.11372360235385968, + "brier": 0.02309697147689805, + "expected_brier": 0.023096971476898043, + "multiclass_ece": 0.10749539203337832, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x8_y0_green": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0009088779818975207, + "brier": 1.6506175937370657e-06, + "expected_brier": 1.6506175937180956e-06, + "multiclass_ece": 0.0009084650774072722, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x8_y11_blue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0003132128844100487, + "brier": 1.9614317933190255e-07, + "expected_brier": 1.9614317925409352e-07, + "multiclass_ece": 0.0003131638383753188, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x8_y3": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 6.17208862852384, + "brier": 1.9913363837664002, + "expected_brier": 1.9913363837664004, + "multiclass_ece": 0.9977503553142871, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + } + ] + }, + "painting-geometry-v1/x9_y0": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0012497641972180297, + "brier": 2.2272428205252976e-06, + "expected_brier": 2.227242820618258e-06, + "multiclass_ece": 0.0012489835671787297, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x9_y0_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.002408335873991389, + "brier": 9.778254214000045e-06, + "expected_brier": 9.77825421411005e-06, + "multiclass_ece": 0.0024054381598401786, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x9_y1_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0003732461146552982, + "brier": 2.108141433828455e-07, + "expected_brier": 2.1081414347534633e-07, + "multiclass_ece": 0.00037317646698975526, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x9_y2": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.001143837387141169, + "brier": 1.7192375915504163e-06, + "expected_brier": 1.7192375916241787e-06, + "multiclass_ece": 0.0011431792389777762, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "painting-geometry-v1/x9_y6_red": { + "count": 1, + "hard_count": 1, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "nll": 4.515458136562205, + "brier": 1.956484935178565, + "expected_brier": 1.956484935178565, + "multiclass_ece": 0.989061407390503, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "painting-geometry-v1/x9_y9_hue": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0012517004928135674, + "brier": 2.056753662906301e-06, + "expected_brier": 2.0567536629378935e-06, + "multiclass_ece": 0.0012509174425006098, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "reasoning-control-v1/decision": { + "count": 27, + "hard_count": 27, + "accuracy": 0.5925925925925926, + "expected_accuracy": 0.5925925925925926, + "nll": 1.153127157742573, + "brier": 0.5862613474158151, + "expected_brier": 0.5862613474158151, + "multiclass_ece": 0.21290927882488817, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 19, + "coverage": 0.7037037037037037, + "accuracy": 0.7368421052631579, + "expected_accuracy": 0.7368421052631579, + "expected_risk": 0.26315789473684215 + }, + { + "threshold": 0.7, + "selected": 19, + "coverage": 0.7037037037037037, + "accuracy": 0.7368421052631579, + "expected_accuracy": 0.7368421052631579, + "expected_risk": 0.26315789473684215 + }, + { + "threshold": 0.8, + "selected": 16, + "coverage": 0.5925925925925926, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.9, + "selected": 15, + "coverage": 0.5555555555555556, + "accuracy": 0.7333333333333333, + "expected_accuracy": 0.7333333333333333, + "expected_risk": 0.2666666666666667 + }, + { + "threshold": 0.95, + "selected": 14, + "coverage": 0.5185185185185185, + "accuracy": 0.7857142857142857, + "expected_accuracy": 0.7857142857142857, + "expected_risk": 0.2142857142857143 + }, + { + "threshold": 0.99, + "selected": 8, + "coverage": 0.2962962962962963, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "reasoning-control-v1/ordinal": { + "count": 26, + "hard_count": 26, + "accuracy": 0.7692307692307693, + "expected_accuracy": 0.7692307692307693, + "nll": 0.796582467406562, + "brier": 0.4185420177274819, + "expected_brier": 0.4185420177274819, + "multiclass_ece": 0.21843402519068247, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 20, + "coverage": 0.7692307692307693, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.7, + "selected": 19, + "coverage": 0.7307692307692307, + "accuracy": 0.7894736842105263, + "expected_accuracy": 0.7894736842105263, + "expected_risk": 0.21052631578947367 + }, + { + "threshold": 0.8, + "selected": 18, + "coverage": 0.6923076923076923, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.9, + "selected": 15, + "coverage": 0.5769230769230769, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.95, + "selected": 9, + "coverage": 0.34615384615384615, + "accuracy": 0.8888888888888888, + "expected_accuracy": 0.8888888888888888, + "expected_risk": 0.11111111111111116 + }, + { + "threshold": 0.99, + "selected": 4, + "coverage": 0.15384615384615385, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "reasoning-control-v1/verification": { + "count": 27, + "hard_count": 27, + "accuracy": 0.6666666666666666, + "expected_accuracy": 0.6666666666666666, + "nll": 0.6204530514715896, + "brier": 0.43873660871309145, + "expected_brier": 0.4387366087130915, + "multiclass_ece": 0.2084572905485984, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.6666666666666666, + "expected_accuracy": 0.6666666666666666, + "expected_risk": 0.33333333333333337 + }, + { + "threshold": 0.7, + "selected": 24, + "coverage": 0.8888888888888888, + "accuracy": 0.7083333333333334, + "expected_accuracy": 0.7083333333333334, + "expected_risk": 0.29166666666666663 + }, + { + "threshold": 0.8, + "selected": 19, + "coverage": 0.7037037037037037, + "accuracy": 0.7894736842105263, + "expected_accuracy": 0.7894736842105263, + "expected_risk": 0.21052631578947367 + }, + { + "threshold": 0.9, + "selected": 16, + "coverage": 0.5925925925925926, + "accuracy": 0.875, + "expected_accuracy": 0.875, + "expected_risk": 0.125 + }, + { + "threshold": 0.95, + "selected": 13, + "coverage": 0.48148148148148145, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 6, + "coverage": 0.2222222222222222, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "snake-v1/choice": { + "count": 27, + "hard_count": 21, + "accuracy": 1.0, + "expected_accuracy": 0.8888888888888888, + "nll": 0.23810749290289118, + "brier": 0.025412196660988203, + "expected_brier": 0.1365233077720993, + "multiclass_ece": 0.07220241217442877, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 0.8888888888888888, + "expected_risk": 0.11111111111111116 + }, + { + "threshold": 0.7, + "selected": 20, + "coverage": 0.7407407407407407, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 20, + "coverage": 0.7407407407407407, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 15, + "coverage": 0.5555555555555556, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 7, + "coverage": 0.25925925925925924, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "snake-v1/noul": { + "count": 27, + "hard_count": 27, + "accuracy": 0.8518518518518519, + "expected_accuracy": 0.8518518518518519, + "nll": 0.4316485192914303, + "brier": 0.24150088669014402, + "expected_brier": 0.24150088669014405, + "multiclass_ece": 0.11647797242821019, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.8518518518518519, + "expected_accuracy": 0.8518518518518519, + "expected_risk": 0.14814814814814814 + }, + { + "threshold": 0.7, + "selected": 26, + "coverage": 0.9629629629629629, + "accuracy": 0.8846153846153846, + "expected_accuracy": 0.8846153846153846, + "expected_risk": 0.11538461538461542 + }, + { + "threshold": 0.8, + "selected": 26, + "coverage": 0.9629629629629629, + "accuracy": 0.8846153846153846, + "expected_accuracy": 0.8846153846153846, + "expected_risk": 0.11538461538461542 + }, + { + "threshold": 0.9, + "selected": 26, + "coverage": 0.9629629629629629, + "accuracy": 0.8846153846153846, + "expected_accuracy": 0.8846153846153846, + "expected_risk": 0.11538461538461542 + }, + { + "threshold": 0.95, + "selected": 22, + "coverage": 0.8148148148148148, + "accuracy": 0.9090909090909091, + "expected_accuracy": 0.9090909090909091, + "expected_risk": 0.09090909090909094 + }, + { + "threshold": 0.99, + "selected": 8, + "coverage": 0.2962962962962963, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "vizdoom-basic-v1/choice": { + "count": 27, + "hard_count": 27, + "accuracy": 0.7777777777777778, + "expected_accuracy": 0.7777777777777778, + "nll": 0.510881722478577, + "brier": 0.3110848977402862, + "expected_brier": 0.3110848977402862, + "multiclass_ece": 0.18076080302939543, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.7777777777777778, + "expected_accuracy": 0.7777777777777778, + "expected_risk": 0.2222222222222222 + }, + { + "threshold": 0.7, + "selected": 24, + "coverage": 0.8888888888888888, + "accuracy": 0.8333333333333334, + "expected_accuracy": 0.8333333333333334, + "expected_risk": 0.16666666666666663 + }, + { + "threshold": 0.8, + "selected": 20, + "coverage": 0.7407407407407407, + "accuracy": 0.95, + "expected_accuracy": 0.95, + "expected_risk": 0.050000000000000044 + }, + { + "threshold": 0.9, + "selected": 19, + "coverage": 0.7037037037037037, + "accuracy": 0.9473684210526315, + "expected_accuracy": 0.9473684210526315, + "expected_risk": 0.052631578947368474 + }, + { + "threshold": 0.95, + "selected": 15, + "coverage": 0.5555555555555556, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "vizdoom-basic-v1/noul": { + "count": 27, + "hard_count": 27, + "accuracy": 0.9629629629629629, + "expected_accuracy": 0.9629629629629629, + "nll": 0.2579303069966674, + "brier": 0.07546167612897833, + "expected_brier": 0.07546167612897833, + "multiclass_ece": 0.03305960701154611, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.9629629629629629, + "expected_accuracy": 0.9629629629629629, + "expected_risk": 0.03703703703703709 + }, + { + "threshold": 0.7, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.9629629629629629, + "expected_accuracy": 0.9629629629629629, + "expected_risk": 0.03703703703703709 + }, + { + "threshold": 0.8, + "selected": 27, + "coverage": 1.0, + "accuracy": 0.9629629629629629, + "expected_accuracy": 0.9629629629629629, + "expected_risk": 0.03703703703703709 + }, + { + "threshold": 0.9, + "selected": 26, + "coverage": 0.9629629629629629, + "accuracy": 0.9615384615384616, + "expected_accuracy": 0.9615384615384616, + "expected_risk": 0.038461538461538436 + }, + { + "threshold": 0.95, + "selected": 26, + "coverage": 0.9629629629629629, + "accuracy": 0.9615384615384616, + "expected_accuracy": 0.9615384615384616, + "expected_risk": 0.038461538461538436 + }, + { + "threshold": 0.99, + "selected": 21, + "coverage": 0.7777777777777778, + "accuracy": 0.9523809523809523, + "expected_accuracy": 0.9523809523809523, + "expected_risk": 0.04761904761904767 + } + ] + }, + "vizdoom-basic-v1/score": { + "count": 27, + "hard_count": 27, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.009710990419892294, + "brier": 0.0017145735428157382, + "expected_brier": 0.0017145735428157277, + "multiclass_ece": 0.009131062885368831, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 27, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 26, + "coverage": 0.9629629629629629, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 25, + "coverage": 0.9259259259259259, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 25, + "coverage": 0.9259259259259259, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "wikispeedia-v1/choice": { + "count": 27, + "hard_count": 9, + "accuracy": 0.5555555555555556, + "expected_accuracy": 0.27335390946502053, + "nll": 2.21792246668928, + "brier": 0.35760252451643976, + "expected_brier": 0.8657300965329005, + "multiclass_ece": 0.15134543733309497, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/agent_trace_observability/AUTO-CLOSE": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0013970655108291486, + "brier": 4.243623541093808e-06, + "expected_brier": 4.24362354112251e-06, + "multiclass_ece": 0.0013960034400172239, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/agent_trace_observability/COUNT ONLY": { + "count": 6, + "hard_count": 6, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0006494540634950175, + "brier": 8.902746207117703e-07, + "expected_brier": 8.902746206906897e-07, + "multiclass_ece": 0.0006492313871933794, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 6, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 6, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 6, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 6, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 6, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 6, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/agent_trace_observability/FILE ISSUE ยท ROUTE": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0011888481860308944, + "brier": 3.030292954343521e-06, + "expected_brier": 3.0302929543424284e-06, + "multiclass_ece": 0.00118808992997943, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/agent_trace_observability/HUMAN REVIEW": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.000857696059778229, + "brier": 1.4700237776508242e-06, + "expected_brier": 1.4700237775944203e-06, + "multiclass_ece": 0.0008573283436498214, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/agent_trace_observability/NOT A BUG": { + "count": 6, + "hard_count": 6, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00616565545610085, + "brier": 0.0002057435319516677, + "expected_brier": 0.00020574353195164838, + "multiclass_ece": 0.006113553786638655, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 6, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 6, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 6, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 6, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 6, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 4, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/agent_trace_observability/PAGE ON-CALL": { + "count": 5, + "hard_count": 5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00728235917843832, + "brier": 0.00015396445584723158, + "expected_brier": 0.00015396445584721087, + "multiclass_ece": 0.007243533694272841, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 4, + "coverage": 0.8, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/agent_trace_observability/PRIORITY REVIEW": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0017144064133931887, + "brier": 7.683532292013932e-06, + "expected_brier": 7.683532291968831e-06, + "multiclass_ece": 0.0017124823685925605, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service/CANCEL": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.01235837867420699, + "brier": 0.001060139305173496, + "expected_brier": 0.0010601393051734576, + "multiclass_ece": 0.012084926890261416, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 0.75, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service/CLOSE": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00231884950865475, + "brier": 2.3024367245839342e-05, + "expected_brier": 2.3024367245808957e-05, + "multiclass_ece": 0.0023130713372191045, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service/FLAG FOR REVIEW": { + "count": 5, + "hard_count": 5, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "nll": 0.6217342825938899, + "brier": 0.3649801130098105, + "expected_brier": 0.3649801130098105, + "multiclass_ece": 0.19051612331284673, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 1.0, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.7, + "selected": 5, + "coverage": 1.0, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.8, + "selected": 5, + "coverage": 1.0, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.9, + "selected": 5, + "coverage": 1.0, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.95, + "selected": 5, + "coverage": 1.0, + "accuracy": 0.8, + "expected_accuracy": 0.8, + "expected_risk": 0.19999999999999996 + }, + { + "threshold": 0.99, + "selected": 4, + "coverage": 0.8, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service/FREEZE CARD": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.010744808252229237, + "brier": 0.000609007336816701, + "expected_brier": 0.0006090073368167174, + "multiclass_ece": 0.01058942359999282, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service/HAND OFF": { + "count": 4, + "hard_count": 4, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "nll": 0.9779793852450871, + "brier": 0.48550392886586813, + "expected_brier": 0.4855039288658682, + "multiclass_ece": 0.30786781103921323, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 0.75, + "accuracy": 0.6666666666666666, + "expected_accuracy": 0.6666666666666666, + "expected_risk": 0.33333333333333337 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 0.25, + "accuracy": 0.0, + "expected_accuracy": 0.0, + "expected_risk": 1.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/customer_service/REFUND": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0006769509461687102, + "brier": 9.159049693926481e-07, + "expected_brier": 9.159049694229537e-07, + "multiclass_ece": 0.00067672186657175, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service/SAY": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.007015759020364116, + "brier": 0.00014726888735118666, + "expected_brier": 0.00014726888735124474, + "multiclass_ece": 0.0069786571097332955, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 0.6666666666666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/customer_service/SET INTENT": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.019982693560681164, + "brier": 0.0008273957273696928, + "expected_brier": 0.000827395727369673, + "multiclass_ece": 0.01977275833919767, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/DISPUTE LINES": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0008600789918185208, + "brier": 1.483255117916939e-06, + "expected_brier": 1.4832551179141025e-06, + "multiclass_ece": 0.0008597079639258709, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/DUPLICATE": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.005519075314967984, + "brier": 0.00010944298576525374, + "expected_brier": 0.00010944298576520683, + "multiclass_ece": 0.005491522966827089, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/FRAUD REVIEW": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0006013554770268364, + "brier": 7.487403138576091e-07, + "expected_brier": 7.487403138236223e-07, + "multiclass_ece": 0.0006011682114070993, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/HOLD": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.016707100677437053, + "brier": 0.0007209642918357847, + "expected_brier": 0.0007209642918357684, + "multiclass_ece": 0.016523649892849452, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 0.3333333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/HOLD FOR DOCUMENTS": { + "count": 5, + "hard_count": 5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0004634428160928183, + "brier": 4.3897287517243745e-07, + "expected_brier": 4.389728751608146e-07, + "multiclass_ece": 0.0004633330373366107, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 5, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/PAY": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0007291188912891053, + "brier": 1.0624538247402956e-06, + "expected_brier": 1.0624538246961635e-06, + "multiclass_ece": 0.0007288531487001926, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/REQUEST CORRECTED INVOICE": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0005777696890279705, + "brier": 6.98925096595535e-07, + "expected_brier": 6.9892509653909e-07, + "multiclass_ece": 0.0005775948847608348, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/ROUTE FOR APPROVAL": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.012723123679879617, + "brier": 0.0003196669737490899, + "expected_brier": 0.0003196669737490776, + "multiclass_ece": 0.01264252691808665, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "workflow-controls-v1/invoice_processing/SCHEDULE": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.017233692259840422, + "brier": 0.0009590728986326729, + "expected_brier": 0.000959072898632618, + "multiclass_ece": 0.01698893072472407, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/invoice_processing/SHORT PAY": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0011586547733859853, + "brier": 4.11615764275265e-06, + "expected_brier": 4.116157642786167e-06, + "multiclass_ece": 0.0011576241530439901, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/AUTO CLOSE": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0008222368481454345, + "brier": 1.558797047947924e-06, + "expected_brier": 1.5587970479587672e-06, + "multiclass_ece": 0.0008218468706910897, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/BLOCK DESTINATION": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0007088377701160579, + "brier": 1.0263983272496134e-06, + "expected_brier": 1.0263983273306465e-06, + "multiclass_ece": 0.0007085810439991791, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/BLOCK EGRESS ASSET": { + "count": 4, + "hard_count": 4, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.000764726666907159, + "brier": 1.1996841159419712e-06, + "expected_brier": 1.199684115937849e-06, + "multiclass_ece": 0.0007644265847994713, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 4, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/BLOCK SENDER": { + "count": 3, + "hard_count": 3, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.001395454523105344, + "brier": 4.161522487468866e-06, + "expected_brier": 4.161522487426221e-06, + "multiclass_ece": 0.0013944130577948262, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/ESCALATE TIER2": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0008313581055031088, + "brier": 1.381163959502572e-06, + "expected_brier": 1.3811639594241143e-06, + "multiclass_ece": 0.0008310126230998138, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/ESCALATE URGENT": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.011469377073177607, + "brier": 0.0004233135761193105, + "expected_brier": 0.00042331357611924414, + "multiclass_ece": 0.011362099469704345, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 0.5, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/ISOLATE HOST": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0013706612538375482, + "brier": 3.752278504706391e-06, + "expected_brier": 3.752278504798312e-06, + "multiclass_ece": 0.0013697223267338243, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/NOTIFY USER": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0005042638702921357, + "brier": 5.083077266813369e-07, + "expected_brier": 5.083077265766178e-07, + "multiclass_ece": 0.000504136750634876, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/PURGE MAILBOXES": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0009008090705143226, + "brier": 1.9052234845194915e-06, + "expected_brier": 1.9052234845329252e-06, + "multiclass_ece": 0.0009003323931723317, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/QUARANTINE FILE": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0009404070577906259, + "brier": 1.767068453777562e-06, + "expected_brier": 1.7670684537840486e-06, + "multiclass_ece": 0.0009399650136514559, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/QUARANTINE MESSAGE": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0006237122490353072, + "brier": 7.793124437410299e-07, + "expected_brier": 7.793124437993626e-07, + "multiclass_ece": 0.0006235173395334748, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/REQUIRE REAUTH": { + "count": 1, + "hard_count": 1, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0004903111236311378, + "brier": 4.805743168364158e-07, + "expected_brier": 4.805743167857202e-07, + "multiclass_ece": 0.0004901909407752925, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/REVOKE ACCESS KEY": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0012576295631115512, + "brier": 3.953374030578835e-06, + "expected_brier": 3.953374030585266e-06, + "multiclass_ece": 0.0012566400569566927, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "workflow-controls-v1/security_incidents/REVOKE SESSIONS": { + "count": 2, + "hard_count": 2, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.002197500860860765, + "brier": 1.0095942932134542e-05, + "expected_brier": 1.0095942932220048e-05, + "multiclass_ece": 0.0021949728375881827, + "ece_bins": 15, + "coverage": [ + { + "threshold": 0.5, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 2, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + } + } +}