| { |
| "baseline": { |
| "n": 3000, |
| "accuracy": 0.9676666666666667, |
| "balanced_accuracy": 0.9677240727274675, |
| "f1_deny": 0.9654927072216293, |
| "auroc": 0.9943784012045359, |
| "nll": 0.13015823493686085, |
| "false_approve_rate": 0.03140613847251963, |
| "false_deny_rate": 0.03314571607254534, |
| "false_approve_count": 44, |
| "false_deny_count": 53, |
| "deny_count": 1401, |
| "approve_count": 1599, |
| "threshold": 0.5, |
| "brier": 0.02713664612681924, |
| "accuracy_ci95": [ |
| 0.9607148392773621, |
| 0.973422344214883 |
| ] |
| }, |
| "shortlist": [ |
| { |
| "name": "soup-greedy", |
| "path": "/home/user/checkpoints/auto04-v3/selection/soup-greedy", |
| "sha256": "cdee2a2c80eba404f39661e0cc8805543380437a2adf23e75ba312ee0324fc2d", |
| "accuracy": 0.9676666666666667, |
| "correct": 2903, |
| "false_approve_count": 40, |
| "false_deny_count": 57, |
| "auroc": 0.9948765712331807, |
| "nll": 0.09408753117300124, |
| "long_accuracy": 0.9539748953974896, |
| "audit_accuracy": 0.9807321772639692, |
| "audit_false_approve_count": 27, |
| "validation_nll": 0.06444159020538713, |
| "validation_balanced_accuracy": 0.9790919571932201, |
| "gates": { |
| "benchmark_accuracy_improved": false, |
| "benchmark_false_approvals_not_worse": true, |
| "benchmark_false_denials_not_worse": false, |
| "benchmark_long_accuracy_not_worse": true, |
| "audit_accuracy_not_worse": true, |
| "audit_false_approvals_not_worse": true |
| }, |
| "passes_score_gates": false |
| }, |
| { |
| "name": "kd_b_aug-step-870", |
| "path": "/home/user/checkpoints/auto04-v3/kd_b_aug/step-870", |
| "sha256": "9569c1cfea42c8fa07f583f8249aac310cd7a2ab8dba0e441cbe6243cec4c158", |
| "accuracy": 0.9676666666666667, |
| "correct": 2903, |
| "false_approve_count": 38, |
| "false_deny_count": 59, |
| "auroc": 0.994768545115858, |
| "nll": 0.09642005100613404, |
| "long_accuracy": 0.9497907949790795, |
| "audit_accuracy": 0.9799614643545279, |
| "audit_false_approve_count": 26, |
| "validation_nll": 0.06539374741470408, |
| "validation_balanced_accuracy": 0.9793324740954146, |
| "gates": { |
| "benchmark_accuracy_improved": false, |
| "benchmark_false_approvals_not_worse": true, |
| "benchmark_false_denials_not_worse": false, |
| "benchmark_long_accuracy_not_worse": true, |
| "audit_accuracy_not_worse": true, |
| "audit_false_approvals_not_worse": true |
| }, |
| "passes_score_gates": false |
| }, |
| { |
| "name": "kd_b_aug-step-1305", |
| "path": "/home/user/checkpoints/auto04-v3/kd_b_aug/step-1305", |
| "sha256": "5fcdb2c6a461ce10defc379ac1a3a40111cd05e7e71692a4e9921909ac5a7935", |
| "accuracy": 0.9696666666666667, |
| "correct": 2909, |
| "false_approve_count": 37, |
| "false_deny_count": 54, |
| "auroc": 0.9948267988692077, |
| "nll": 0.09550500018377822, |
| "long_accuracy": 0.9581589958158996, |
| "audit_accuracy": 0.9803468208092485, |
| "audit_false_approve_count": 28, |
| "validation_nll": 0.06564422695544463, |
| "validation_balanced_accuracy": 0.9790881664376963, |
| "gates": { |
| "benchmark_accuracy_improved": true, |
| "benchmark_false_approvals_not_worse": true, |
| "benchmark_false_denials_not_worse": false, |
| "benchmark_long_accuracy_not_worse": true, |
| "audit_accuracy_not_worse": true, |
| "audit_false_approvals_not_worse": true |
| }, |
| "passes_score_gates": false |
| }, |
| { |
| "name": "soup-kd_b_aug-last3", |
| "path": "/home/user/checkpoints/auto04-v3/selection/soup-kd_b_aug-last3", |
| "sha256": "d18e95a658932bac26cb1ec6c0edd0cd53c4520e0aaeb7eb0591e32381b9a484", |
| "accuracy": 0.9693333333333334, |
| "correct": 2908, |
| "false_approve_count": 36, |
| "false_deny_count": 56, |
| "auroc": 0.9948676434548894, |
| "nll": 0.09592511571143014, |
| "long_accuracy": 0.9581589958158996, |
| "audit_accuracy": 0.9803468208092485, |
| "audit_false_approve_count": 28, |
| "validation_nll": 0.06588619902033178, |
| "validation_balanced_accuracy": 0.9788211142468357, |
| "gates": { |
| "benchmark_accuracy_improved": true, |
| "benchmark_false_approvals_not_worse": true, |
| "benchmark_false_denials_not_worse": false, |
| "benchmark_long_accuracy_not_worse": true, |
| "audit_accuracy_not_worse": true, |
| "audit_false_approvals_not_worse": true |
| }, |
| "passes_score_gates": false |
| }, |
| { |
| "name": "soup-kd_a0.3+kd_b_aug0.7", |
| "path": "/home/user/checkpoints/auto04-v3/selection/soup-kd_a0.3+kd_b_aug0.7", |
| "sha256": "0cb85318efc06bbd3dc5afdd7255b0e7539f2fba65db996497cb9dc78e5ca9bd", |
| "accuracy": 0.9696666666666667, |
| "correct": 2909, |
| "false_approve_count": 36, |
| "false_deny_count": 55, |
| "auroc": 0.995031021797617, |
| "nll": 0.09734198486148768, |
| "long_accuracy": 0.9581589958158996, |
| "audit_accuracy": 0.9803468208092485, |
| "audit_false_approve_count": 27, |
| "validation_nll": 0.06668469289284482, |
| "validation_balanced_accuracy": 0.9784432804936871, |
| "gates": { |
| "benchmark_accuracy_improved": true, |
| "benchmark_false_approvals_not_worse": true, |
| "benchmark_false_denials_not_worse": false, |
| "benchmark_long_accuracy_not_worse": true, |
| "audit_accuracy_not_worse": true, |
| "audit_false_approvals_not_worse": true |
| }, |
| "passes_score_gates": false |
| }, |
| { |
| "name": "soup-kd_a0.4+kd_b_aug0.6", |
| "path": "/home/user/checkpoints/auto04-v3/selection/soup-kd_a0.4+kd_b_aug0.6", |
| "sha256": "24e6ed966ea62dabae209cb88419ed4ea702afa854435bd37064b64bc524eb6b", |
| "accuracy": 0.969, |
| "correct": 2907, |
| "false_approve_count": 38, |
| "false_deny_count": 55, |
| "auroc": 0.9950127198521204, |
| "nll": 0.09768559681259002, |
| "long_accuracy": 0.9581589958158996, |
| "audit_accuracy": 0.9803468208092485, |
| "audit_false_approve_count": 27, |
| "validation_nll": 0.06669131243679578, |
| "validation_balanced_accuracy": 0.978954640342266, |
| "gates": { |
| "benchmark_accuracy_improved": true, |
| "benchmark_false_approvals_not_worse": true, |
| "benchmark_false_denials_not_worse": false, |
| "benchmark_long_accuracy_not_worse": true, |
| "audit_accuracy_not_worse": true, |
| "audit_false_approvals_not_worse": true |
| }, |
| "passes_score_gates": false |
| }, |
| { |
| "name": "kd_b_aug-step-1737", |
| "path": "/home/user/checkpoints/auto04-v3/kd_b_aug/step-1737", |
| "sha256": "bca2d2614055c7dc43d6d9f059446691c343b5bf0ca5271cfe6f29f507cc201d", |
| "accuracy": 0.9683333333333334, |
| "correct": 2905, |
| "false_approve_count": 34, |
| "false_deny_count": 61, |
| "auroc": 0.9949238884581236, |
| "nll": 0.09748464604243104, |
| "long_accuracy": 0.9623430962343096, |
| "audit_accuracy": 0.9803468208092485, |
| "audit_false_approve_count": 27, |
| "validation_nll": 0.06671769760595667, |
| "validation_balanced_accuracy": 0.9788097419802646, |
| "gates": { |
| "benchmark_accuracy_improved": true, |
| "benchmark_false_approvals_not_worse": true, |
| "benchmark_false_denials_not_worse": false, |
| "benchmark_long_accuracy_not_worse": true, |
| "audit_accuracy_not_worse": true, |
| "audit_false_approvals_not_worse": true |
| }, |
| "passes_score_gates": false |
| }, |
| { |
| "name": "soup-all-finals", |
| "path": "/home/user/checkpoints/auto04-v3/selection/soup-all-finals", |
| "sha256": "075d68c851eac223b91ab8b42767763b152bfb8bfa2adbcd127973aa46358899", |
| "accuracy": 0.969, |
| "correct": 2907, |
| "false_approve_count": 48, |
| "false_deny_count": 45, |
| "auroc": 0.9952513147269507, |
| "nll": 0.1001646734032178, |
| "long_accuracy": 0.9623430962343096, |
| "audit_accuracy": 0.981888246628131, |
| "audit_false_approve_count": 26, |
| "validation_nll": 0.06923167073800253, |
| "validation_balanced_accuracy": 0.979988732305995, |
| "gates": { |
| "benchmark_accuracy_improved": true, |
| "benchmark_false_approvals_not_worse": false, |
| "benchmark_false_denials_not_worse": true, |
| "benchmark_long_accuracy_not_worse": true, |
| "audit_accuracy_not_worse": true, |
| "audit_false_approvals_not_worse": true |
| }, |
| "passes_score_gates": false |
| }, |
| { |
| "name": "soup-all-last2", |
| "path": "/home/user/checkpoints/auto04-v3/selection/soup-all-last2", |
| "sha256": "8a9b8aa93546eddb71d0407762b15c236342ca630ad3c3b60dda73fffb42cddc", |
| "accuracy": 0.9703333333333334, |
| "correct": 2911, |
| "false_approve_count": 47, |
| "false_deny_count": 42, |
| "auroc": 0.9953059973689838, |
| "nll": 0.09961677208299435, |
| "long_accuracy": 0.9623430962343096, |
| "audit_accuracy": 0.9811175337186898, |
| "audit_false_approve_count": 26, |
| "validation_nll": 0.06959238149003957, |
| "validation_balanced_accuracy": 0.9803779383257147, |
| "gates": { |
| "benchmark_accuracy_improved": true, |
| "benchmark_false_approvals_not_worse": false, |
| "benchmark_false_denials_not_worse": true, |
| "benchmark_long_accuracy_not_worse": true, |
| "audit_accuracy_not_worse": true, |
| "audit_false_approvals_not_worse": true |
| }, |
| "passes_score_gates": false |
| }, |
| { |
| "name": "soup-kd_b_aug+kd_c", |
| "path": "/home/user/checkpoints/auto04-v3/selection/soup-kd_b_aug+kd_c", |
| "sha256": "016e227636321df3955da589bd60d3fa3452946a21f61bda98f28ee24853c52e", |
| "accuracy": 0.9673333333333334, |
| "correct": 2902, |
| "false_approve_count": 41, |
| "false_deny_count": 57, |
| "auroc": 0.9951778837505061, |
| "nll": 0.10191145911240229, |
| "long_accuracy": 0.9539748953974896, |
| "audit_accuracy": 0.9799614643545279, |
| "audit_false_approve_count": 26, |
| "validation_nll": 0.07110769054353236, |
| "validation_balanced_accuracy": 0.9803551937925725, |
| "gates": { |
| "benchmark_accuracy_improved": false, |
| "benchmark_false_approvals_not_worse": true, |
| "benchmark_false_denials_not_worse": false, |
| "benchmark_long_accuracy_not_worse": true, |
| "audit_accuracy_not_worse": true, |
| "audit_false_approvals_not_worse": true |
| }, |
| "passes_score_gates": false |
| }, |
| { |
| "name": "soup-kd_a0.45+kd_b_aug0.55", |
| "path": "/home/user/checkpoints/auto04-v3/selection/soup-kd_a0.45+kd_b_aug0.55", |
| "sha256": "bcb137cf2e7dd4599593893a89c82d77bb31050223fa42acd37d4e5e6f0b12f9", |
| "accuracy": 0.9693333333333334, |
| "correct": 2908, |
| "false_approve_count": 37, |
| "false_deny_count": 55, |
| "auroc": 0.9950352624923053, |
| "nll": 0.09734451421267853, |
| "long_accuracy": 0.9581589958158996, |
| "audit_accuracy": 0.9803468208092485, |
| "audit_false_approve_count": 27, |
| "validation_nll": 0.0667321780183361, |
| "validation_balanced_accuracy": 0.9784432804936871, |
| "gates": { |
| "benchmark_accuracy_improved": true, |
| "benchmark_false_approvals_not_worse": true, |
| "benchmark_false_denials_not_worse": false, |
| "benchmark_long_accuracy_not_worse": true, |
| "audit_accuracy_not_worse": true, |
| "audit_false_approvals_not_worse": true |
| }, |
| "passes_score_gates": false |
| }, |
| { |
| "name": "soup-kd_a0.5+kd_b_aug0.5", |
| "path": "/home/user/checkpoints/auto04-v3/selection/soup-kd_a0.5+kd_b_aug0.5", |
| "sha256": "8d4a125e417a804e301c6d3542d292bfad57dc709d0b7df3c9aab326a1d6aafa", |
| "accuracy": 0.9693333333333334, |
| "correct": 2908, |
| "false_approve_count": 45, |
| "false_deny_count": 47, |
| "auroc": 0.9951743126391897, |
| "nll": 0.09802602189660843, |
| "long_accuracy": 0.9581589958158996, |
| "audit_accuracy": 0.9815028901734104, |
| "audit_false_approve_count": 27, |
| "validation_nll": 0.06728941604016982, |
| "validation_balanced_accuracy": 0.9797368431372293, |
| "gates": { |
| "benchmark_accuracy_improved": true, |
| "benchmark_false_approvals_not_worse": false, |
| "benchmark_false_denials_not_worse": true, |
| "benchmark_long_accuracy_not_worse": true, |
| "audit_accuracy_not_worse": true, |
| "audit_false_approvals_not_worse": true |
| }, |
| "passes_score_gates": false |
| }, |
| { |
| "name": "soup-kd_a0.35+kd_b_aug0.45+kd_c0.2", |
| "path": "/home/user/checkpoints/auto04-v3/selection/soup-kd_a0.35+kd_b_aug0.45+kd_c0.2", |
| "sha256": "8c4c3ca037a8bf289d8447f796d2ae46788de3ac97281328c9efcb0ffa15803b", |
| "accuracy": 0.97, |
| "correct": 2910, |
| "false_approve_count": 46, |
| "false_deny_count": 44, |
| "auroc": 0.9951789997227926, |
| "nll": 0.09976799853938363, |
| "long_accuracy": 0.9623430962343096, |
| "audit_accuracy": 0.981888246628131, |
| "audit_false_approve_count": 26, |
| "validation_nll": 0.06828081469723489, |
| "validation_balanced_accuracy": 0.9802482029858082, |
| "gates": { |
| "benchmark_accuracy_improved": true, |
| "benchmark_false_approvals_not_worse": false, |
| "benchmark_false_denials_not_worse": true, |
| "benchmark_long_accuracy_not_worse": true, |
| "audit_accuracy_not_worse": true, |
| "audit_false_approvals_not_worse": true |
| }, |
| "passes_score_gates": false |
| }, |
| { |
| "name": "soup-kd_b_aug-last2", |
| "path": "/home/user/checkpoints/auto04-v3/selection/soup-kd_b_aug-last2", |
| "sha256": "08791b8c4064779a00e89615e6b13f55abb3cd5536acc5de0f418d00bd127b0c", |
| "accuracy": 0.97, |
| "correct": 2910, |
| "false_approve_count": 36, |
| "false_deny_count": 54, |
| "auroc": 0.9948738928996933, |
| "nll": 0.09660789025587242, |
| "long_accuracy": 0.9581589958158996, |
| "audit_accuracy": 0.9803468208092485, |
| "audit_false_approve_count": 28, |
| "validation_nll": 0.06612161509400427, |
| "validation_balanced_accuracy": 0.9786913789069291, |
| "gates": { |
| "benchmark_accuracy_improved": true, |
| "benchmark_false_approvals_not_worse": true, |
| "benchmark_false_denials_not_worse": false, |
| "benchmark_long_accuracy_not_worse": true, |
| "audit_accuracy_not_worse": true, |
| "audit_false_approvals_not_worse": true |
| }, |
| "passes_score_gates": false |
| } |
| ] |
| } |
|
|
