{
  "baseline": {
    "n": 3000,
    "accuracy": 0.9676666666666667,
    "balanced_accuracy": 0.9677240727274675,
    "f1_deny": 0.9654927072216293,
    "auroc": 0.9943784012045359,
    "nll": 0.13015823493686085,
    "false_approve_rate": 0.03140613847251963,
    "false_deny_rate": 0.03314571607254534,
    "false_approve_count": 44,
    "false_deny_count": 53,
    "deny_count": 1401,
    "approve_count": 1599,
    "threshold": 0.5,
    "brier": 0.02713664612681924,
    "accuracy_ci95": [
      0.9607148392773621,
      0.973422344214883
    ]
  },
  "shortlist": [
    {
      "name": "soup-greedy",
      "path": "/home/user/checkpoints/auto04-v3/selection/soup-greedy",
      "sha256": "cdee2a2c80eba404f39661e0cc8805543380437a2adf23e75ba312ee0324fc2d",
      "accuracy": 0.9676666666666667,
      "correct": 2903,
      "false_approve_count": 40,
      "false_deny_count": 57,
      "auroc": 0.9948765712331807,
      "nll": 0.09408753117300124,
      "long_accuracy": 0.9539748953974896,
      "audit_accuracy": 0.9807321772639692,
      "audit_false_approve_count": 27,
      "validation_nll": 0.06444159020538713,
      "validation_balanced_accuracy": 0.9790919571932201,
      "gates": {
        "benchmark_accuracy_improved": false,
        "benchmark_false_approvals_not_worse": true,
        "benchmark_false_denials_not_worse": false,
        "benchmark_long_accuracy_not_worse": true,
        "audit_accuracy_not_worse": true,
        "audit_false_approvals_not_worse": true
      },
      "passes_score_gates": false
    },
    {
      "name": "kd_b_aug-step-870",
      "path": "/home/user/checkpoints/auto04-v3/kd_b_aug/step-870",
      "sha256": "9569c1cfea42c8fa07f583f8249aac310cd7a2ab8dba0e441cbe6243cec4c158",
      "accuracy": 0.9676666666666667,
      "correct": 2903,
      "false_approve_count": 38,
      "false_deny_count": 59,
      "auroc": 0.994768545115858,
      "nll": 0.09642005100613404,
      "long_accuracy": 0.9497907949790795,
      "audit_accuracy": 0.9799614643545279,
      "audit_false_approve_count": 26,
      "validation_nll": 0.06539374741470408,
      "validation_balanced_accuracy": 0.9793324740954146,
      "gates": {
        "benchmark_accuracy_improved": false,
        "benchmark_false_approvals_not_worse": true,
        "benchmark_false_denials_not_worse": false,
        "benchmark_long_accuracy_not_worse": true,
        "audit_accuracy_not_worse": true,
        "audit_false_approvals_not_worse": true
      },
      "passes_score_gates": false
    },
    {
      "name": "kd_b_aug-step-1305",
      "path": "/home/user/checkpoints/auto04-v3/kd_b_aug/step-1305",
      "sha256": "5fcdb2c6a461ce10defc379ac1a3a40111cd05e7e71692a4e9921909ac5a7935",
      "accuracy": 0.9696666666666667,
      "correct": 2909,
      "false_approve_count": 37,
      "false_deny_count": 54,
      "auroc": 0.9948267988692077,
      "nll": 0.09550500018377822,
      "long_accuracy": 0.9581589958158996,
      "audit_accuracy": 0.9803468208092485,
      "audit_false_approve_count": 28,
      "validation_nll": 0.06564422695544463,
      "validation_balanced_accuracy": 0.9790881664376963,
      "gates": {
        "benchmark_accuracy_improved": true,
        "benchmark_false_approvals_not_worse": true,
        "benchmark_false_denials_not_worse": false,
        "benchmark_long_accuracy_not_worse": true,
        "audit_accuracy_not_worse": true,
        "audit_false_approvals_not_worse": true
      },
      "passes_score_gates": false
    },
    {
      "name": "soup-kd_b_aug-last3",
      "path": "/home/user/checkpoints/auto04-v3/selection/soup-kd_b_aug-last3",
      "sha256": "d18e95a658932bac26cb1ec6c0edd0cd53c4520e0aaeb7eb0591e32381b9a484",
      "accuracy": 0.9693333333333334,
      "correct": 2908,
      "false_approve_count": 36,
      "false_deny_count": 56,
      "auroc": 0.9948676434548894,
      "nll": 0.09592511571143014,
      "long_accuracy": 0.9581589958158996,
      "audit_accuracy": 0.9803468208092485,
      "audit_false_approve_count": 28,
      "validation_nll": 0.06588619902033178,
      "validation_balanced_accuracy": 0.9788211142468357,
      "gates": {
        "benchmark_accuracy_improved": true,
        "benchmark_false_approvals_not_worse": true,
        "benchmark_false_denials_not_worse": false,
        "benchmark_long_accuracy_not_worse": true,
        "audit_accuracy_not_worse": true,
        "audit_false_approvals_not_worse": true
      },
      "passes_score_gates": false
    },
    {
      "name": "soup-kd_a0.3+kd_b_aug0.7",
      "path": "/home/user/checkpoints/auto04-v3/selection/soup-kd_a0.3+kd_b_aug0.7",
      "sha256": "0cb85318efc06bbd3dc5afdd7255b0e7539f2fba65db996497cb9dc78e5ca9bd",
      "accuracy": 0.9696666666666667,
      "correct": 2909,
      "false_approve_count": 36,
      "false_deny_count": 55,
      "auroc": 0.995031021797617,
      "nll": 0.09734198486148768,
      "long_accuracy": 0.9581589958158996,
      "audit_accuracy": 0.9803468208092485,
      "audit_false_approve_count": 27,
      "validation_nll": 0.06668469289284482,
      "validation_balanced_accuracy": 0.9784432804936871,
      "gates": {
        "benchmark_accuracy_improved": true,
        "benchmark_false_approvals_not_worse": true,
        "benchmark_false_denials_not_worse": false,
        "benchmark_long_accuracy_not_worse": true,
        "audit_accuracy_not_worse": true,
        "audit_false_approvals_not_worse": true
      },
      "passes_score_gates": false
    },
    {
      "name": "soup-kd_a0.4+kd_b_aug0.6",
      "path": "/home/user/checkpoints/auto04-v3/selection/soup-kd_a0.4+kd_b_aug0.6",
      "sha256": "24e6ed966ea62dabae209cb88419ed4ea702afa854435bd37064b64bc524eb6b",
      "accuracy": 0.969,
      "correct": 2907,
      "false_approve_count": 38,
      "false_deny_count": 55,
      "auroc": 0.9950127198521204,
      "nll": 0.09768559681259002,
      "long_accuracy": 0.9581589958158996,
      "audit_accuracy": 0.9803468208092485,
      "audit_false_approve_count": 27,
      "validation_nll": 0.06669131243679578,
      "validation_balanced_accuracy": 0.978954640342266,
      "gates": {
        "benchmark_accuracy_improved": true,
        "benchmark_false_approvals_not_worse": true,
        "benchmark_false_denials_not_worse": false,
        "benchmark_long_accuracy_not_worse": true,
        "audit_accuracy_not_worse": true,
        "audit_false_approvals_not_worse": true
      },
      "passes_score_gates": false
    },
    {
      "name": "kd_b_aug-step-1737",
      "path": "/home/user/checkpoints/auto04-v3/kd_b_aug/step-1737",
      "sha256": "bca2d2614055c7dc43d6d9f059446691c343b5bf0ca5271cfe6f29f507cc201d",
      "accuracy": 0.9683333333333334,
      "correct": 2905,
      "false_approve_count": 34,
      "false_deny_count": 61,
      "auroc": 0.9949238884581236,
      "nll": 0.09748464604243104,
      "long_accuracy": 0.9623430962343096,
      "audit_accuracy": 0.9803468208092485,
      "audit_false_approve_count": 27,
      "validation_nll": 0.06671769760595667,
      "validation_balanced_accuracy": 0.9788097419802646,
      "gates": {
        "benchmark_accuracy_improved": true,
        "benchmark_false_approvals_not_worse": true,
        "benchmark_false_denials_not_worse": false,
        "benchmark_long_accuracy_not_worse": true,
        "audit_accuracy_not_worse": true,
        "audit_false_approvals_not_worse": true
      },
      "passes_score_gates": false
    },
    {
      "name": "soup-all-finals",
      "path": "/home/user/checkpoints/auto04-v3/selection/soup-all-finals",
      "sha256": "075d68c851eac223b91ab8b42767763b152bfb8bfa2adbcd127973aa46358899",
      "accuracy": 0.969,
      "correct": 2907,
      "false_approve_count": 48,
      "false_deny_count": 45,
      "auroc": 0.9952513147269507,
      "nll": 0.1001646734032178,
      "long_accuracy": 0.9623430962343096,
      "audit_accuracy": 0.981888246628131,
      "audit_false_approve_count": 26,
      "validation_nll": 0.06923167073800253,
      "validation_balanced_accuracy": 0.979988732305995,
      "gates": {
        "benchmark_accuracy_improved": true,
        "benchmark_false_approvals_not_worse": false,
        "benchmark_false_denials_not_worse": true,
        "benchmark_long_accuracy_not_worse": true,
        "audit_accuracy_not_worse": true,
        "audit_false_approvals_not_worse": true
      },
      "passes_score_gates": false
    },
    {
      "name": "soup-all-last2",
      "path": "/home/user/checkpoints/auto04-v3/selection/soup-all-last2",
      "sha256": "8a9b8aa93546eddb71d0407762b15c236342ca630ad3c3b60dda73fffb42cddc",
      "accuracy": 0.9703333333333334,
      "correct": 2911,
      "false_approve_count": 47,
      "false_deny_count": 42,
      "auroc": 0.9953059973689838,
      "nll": 0.09961677208299435,
      "long_accuracy": 0.9623430962343096,
      "audit_accuracy": 0.9811175337186898,
      "audit_false_approve_count": 26,
      "validation_nll": 0.06959238149003957,
      "validation_balanced_accuracy": 0.9803779383257147,
      "gates": {
        "benchmark_accuracy_improved": true,
        "benchmark_false_approvals_not_worse": false,
        "benchmark_false_denials_not_worse": true,
        "benchmark_long_accuracy_not_worse": true,
        "audit_accuracy_not_worse": true,
        "audit_false_approvals_not_worse": true
      },
      "passes_score_gates": false
    },
    {
      "name": "soup-kd_b_aug+kd_c",
      "path": "/home/user/checkpoints/auto04-v3/selection/soup-kd_b_aug+kd_c",
      "sha256": "016e227636321df3955da589bd60d3fa3452946a21f61bda98f28ee24853c52e",
      "accuracy": 0.9673333333333334,
      "correct": 2902,
      "false_approve_count": 41,
      "false_deny_count": 57,
      "auroc": 0.9951778837505061,
      "nll": 0.10191145911240229,
      "long_accuracy": 0.9539748953974896,
      "audit_accuracy": 0.9799614643545279,
      "audit_false_approve_count": 26,
      "validation_nll": 0.07110769054353236,
      "validation_balanced_accuracy": 0.9803551937925725,
      "gates": {
        "benchmark_accuracy_improved": false,
        "benchmark_false_approvals_not_worse": true,
        "benchmark_false_denials_not_worse": false,
        "benchmark_long_accuracy_not_worse": true,
        "audit_accuracy_not_worse": true,
        "audit_false_approvals_not_worse": true
      },
      "passes_score_gates": false
    },
    {
      "name": "soup-kd_a0.45+kd_b_aug0.55",
      "path": "/home/user/checkpoints/auto04-v3/selection/soup-kd_a0.45+kd_b_aug0.55",
      "sha256": "bcb137cf2e7dd4599593893a89c82d77bb31050223fa42acd37d4e5e6f0b12f9",
      "accuracy": 0.9693333333333334,
      "correct": 2908,
      "false_approve_count": 37,
      "false_deny_count": 55,
      "auroc": 0.9950352624923053,
      "nll": 0.09734451421267853,
      "long_accuracy": 0.9581589958158996,
      "audit_accuracy": 0.9803468208092485,
      "audit_false_approve_count": 27,
      "validation_nll": 0.0667321780183361,
      "validation_balanced_accuracy": 0.9784432804936871,
      "gates": {
        "benchmark_accuracy_improved": true,
        "benchmark_false_approvals_not_worse": true,
        "benchmark_false_denials_not_worse": false,
        "benchmark_long_accuracy_not_worse": true,
        "audit_accuracy_not_worse": true,
        "audit_false_approvals_not_worse": true
      },
      "passes_score_gates": false
    },
    {
      "name": "soup-kd_a0.5+kd_b_aug0.5",
      "path": "/home/user/checkpoints/auto04-v3/selection/soup-kd_a0.5+kd_b_aug0.5",
      "sha256": "8d4a125e417a804e301c6d3542d292bfad57dc709d0b7df3c9aab326a1d6aafa",
      "accuracy": 0.9693333333333334,
      "correct": 2908,
      "false_approve_count": 45,
      "false_deny_count": 47,
      "auroc": 0.9951743126391897,
      "nll": 0.09802602189660843,
      "long_accuracy": 0.9581589958158996,
      "audit_accuracy": 0.9815028901734104,
      "audit_false_approve_count": 27,
      "validation_nll": 0.06728941604016982,
      "validation_balanced_accuracy": 0.9797368431372293,
      "gates": {
        "benchmark_accuracy_improved": true,
        "benchmark_false_approvals_not_worse": false,
        "benchmark_false_denials_not_worse": true,
        "benchmark_long_accuracy_not_worse": true,
        "audit_accuracy_not_worse": true,
        "audit_false_approvals_not_worse": true
      },
      "passes_score_gates": false
    },
    {
      "name": "soup-kd_a0.35+kd_b_aug0.45+kd_c0.2",
      "path": "/home/user/checkpoints/auto04-v3/selection/soup-kd_a0.35+kd_b_aug0.45+kd_c0.2",
      "sha256": "8c4c3ca037a8bf289d8447f796d2ae46788de3ac97281328c9efcb0ffa15803b",
      "accuracy": 0.97,
      "correct": 2910,
      "false_approve_count": 46,
      "false_deny_count": 44,
      "auroc": 0.9951789997227926,
      "nll": 0.09976799853938363,
      "long_accuracy": 0.9623430962343096,
      "audit_accuracy": 0.981888246628131,
      "audit_false_approve_count": 26,
      "validation_nll": 0.06828081469723489,
      "validation_balanced_accuracy": 0.9802482029858082,
      "gates": {
        "benchmark_accuracy_improved": true,
        "benchmark_false_approvals_not_worse": false,
        "benchmark_false_denials_not_worse": true,
        "benchmark_long_accuracy_not_worse": true,
        "audit_accuracy_not_worse": true,
        "audit_false_approvals_not_worse": true
      },
      "passes_score_gates": false
    },
    {
      "name": "soup-kd_b_aug-last2",
      "path": "/home/user/checkpoints/auto04-v3/selection/soup-kd_b_aug-last2",
      "sha256": "08791b8c4064779a00e89615e6b13f55abb3cd5536acc5de0f418d00bd127b0c",
      "accuracy": 0.97,
      "correct": 2910,
      "false_approve_count": 36,
      "false_deny_count": 54,
      "auroc": 0.9948738928996933,
      "nll": 0.09660789025587242,
      "long_accuracy": 0.9581589958158996,
      "audit_accuracy": 0.9803468208092485,
      "audit_false_approve_count": 28,
      "validation_nll": 0.06612161509400427,
      "validation_balanced_accuracy": 0.9786913789069291,
      "gates": {
        "benchmark_accuracy_improved": true,
        "benchmark_false_approvals_not_worse": true,
        "benchmark_false_denials_not_worse": false,
        "benchmark_long_accuracy_not_worse": true,
        "audit_accuracy_not_worse": true,
        "audit_false_approvals_not_worse": true
      },
      "passes_score_gates": false
    }
  ]
}