File size: 14,336 Bytes
1c297a0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 | {
"baseline": {
"n": 3000,
"accuracy": 0.9676666666666667,
"balanced_accuracy": 0.9677240727274675,
"f1_deny": 0.9654927072216293,
"auroc": 0.9943784012045359,
"nll": 0.13015823493686085,
"false_approve_rate": 0.03140613847251963,
"false_deny_rate": 0.03314571607254534,
"false_approve_count": 44,
"false_deny_count": 53,
"deny_count": 1401,
"approve_count": 1599,
"threshold": 0.5,
"brier": 0.02713664612681924,
"accuracy_ci95": [
0.9607148392773621,
0.973422344214883
]
},
"shortlist": [
{
"name": "soup-greedy",
"path": "/home/user/checkpoints/auto04-v3/selection/soup-greedy",
"sha256": "cdee2a2c80eba404f39661e0cc8805543380437a2adf23e75ba312ee0324fc2d",
"accuracy": 0.9676666666666667,
"correct": 2903,
"false_approve_count": 40,
"false_deny_count": 57,
"auroc": 0.9948765712331807,
"nll": 0.09408753117300124,
"long_accuracy": 0.9539748953974896,
"audit_accuracy": 0.9807321772639692,
"audit_false_approve_count": 27,
"validation_nll": 0.06444159020538713,
"validation_balanced_accuracy": 0.9790919571932201,
"gates": {
"benchmark_accuracy_improved": false,
"benchmark_false_approvals_not_worse": true,
"benchmark_false_denials_not_worse": false,
"benchmark_long_accuracy_not_worse": true,
"audit_accuracy_not_worse": true,
"audit_false_approvals_not_worse": true
},
"passes_score_gates": false
},
{
"name": "kd_b_aug-step-870",
"path": "/home/user/checkpoints/auto04-v3/kd_b_aug/step-870",
"sha256": "9569c1cfea42c8fa07f583f8249aac310cd7a2ab8dba0e441cbe6243cec4c158",
"accuracy": 0.9676666666666667,
"correct": 2903,
"false_approve_count": 38,
"false_deny_count": 59,
"auroc": 0.994768545115858,
"nll": 0.09642005100613404,
"long_accuracy": 0.9497907949790795,
"audit_accuracy": 0.9799614643545279,
"audit_false_approve_count": 26,
"validation_nll": 0.06539374741470408,
"validation_balanced_accuracy": 0.9793324740954146,
"gates": {
"benchmark_accuracy_improved": false,
"benchmark_false_approvals_not_worse": true,
"benchmark_false_denials_not_worse": false,
"benchmark_long_accuracy_not_worse": true,
"audit_accuracy_not_worse": true,
"audit_false_approvals_not_worse": true
},
"passes_score_gates": false
},
{
"name": "kd_b_aug-step-1305",
"path": "/home/user/checkpoints/auto04-v3/kd_b_aug/step-1305",
"sha256": "5fcdb2c6a461ce10defc379ac1a3a40111cd05e7e71692a4e9921909ac5a7935",
"accuracy": 0.9696666666666667,
"correct": 2909,
"false_approve_count": 37,
"false_deny_count": 54,
"auroc": 0.9948267988692077,
"nll": 0.09550500018377822,
"long_accuracy": 0.9581589958158996,
"audit_accuracy": 0.9803468208092485,
"audit_false_approve_count": 28,
"validation_nll": 0.06564422695544463,
"validation_balanced_accuracy": 0.9790881664376963,
"gates": {
"benchmark_accuracy_improved": true,
"benchmark_false_approvals_not_worse": true,
"benchmark_false_denials_not_worse": false,
"benchmark_long_accuracy_not_worse": true,
"audit_accuracy_not_worse": true,
"audit_false_approvals_not_worse": true
},
"passes_score_gates": false
},
{
"name": "soup-kd_b_aug-last3",
"path": "/home/user/checkpoints/auto04-v3/selection/soup-kd_b_aug-last3",
"sha256": "d18e95a658932bac26cb1ec6c0edd0cd53c4520e0aaeb7eb0591e32381b9a484",
"accuracy": 0.9693333333333334,
"correct": 2908,
"false_approve_count": 36,
"false_deny_count": 56,
"auroc": 0.9948676434548894,
"nll": 0.09592511571143014,
"long_accuracy": 0.9581589958158996,
"audit_accuracy": 0.9803468208092485,
"audit_false_approve_count": 28,
"validation_nll": 0.06588619902033178,
"validation_balanced_accuracy": 0.9788211142468357,
"gates": {
"benchmark_accuracy_improved": true,
"benchmark_false_approvals_not_worse": true,
"benchmark_false_denials_not_worse": false,
"benchmark_long_accuracy_not_worse": true,
"audit_accuracy_not_worse": true,
"audit_false_approvals_not_worse": true
},
"passes_score_gates": false
},
{
"name": "soup-kd_a0.3+kd_b_aug0.7",
"path": "/home/user/checkpoints/auto04-v3/selection/soup-kd_a0.3+kd_b_aug0.7",
"sha256": "0cb85318efc06bbd3dc5afdd7255b0e7539f2fba65db996497cb9dc78e5ca9bd",
"accuracy": 0.9696666666666667,
"correct": 2909,
"false_approve_count": 36,
"false_deny_count": 55,
"auroc": 0.995031021797617,
"nll": 0.09734198486148768,
"long_accuracy": 0.9581589958158996,
"audit_accuracy": 0.9803468208092485,
"audit_false_approve_count": 27,
"validation_nll": 0.06668469289284482,
"validation_balanced_accuracy": 0.9784432804936871,
"gates": {
"benchmark_accuracy_improved": true,
"benchmark_false_approvals_not_worse": true,
"benchmark_false_denials_not_worse": false,
"benchmark_long_accuracy_not_worse": true,
"audit_accuracy_not_worse": true,
"audit_false_approvals_not_worse": true
},
"passes_score_gates": false
},
{
"name": "soup-kd_a0.4+kd_b_aug0.6",
"path": "/home/user/checkpoints/auto04-v3/selection/soup-kd_a0.4+kd_b_aug0.6",
"sha256": "24e6ed966ea62dabae209cb88419ed4ea702afa854435bd37064b64bc524eb6b",
"accuracy": 0.969,
"correct": 2907,
"false_approve_count": 38,
"false_deny_count": 55,
"auroc": 0.9950127198521204,
"nll": 0.09768559681259002,
"long_accuracy": 0.9581589958158996,
"audit_accuracy": 0.9803468208092485,
"audit_false_approve_count": 27,
"validation_nll": 0.06669131243679578,
"validation_balanced_accuracy": 0.978954640342266,
"gates": {
"benchmark_accuracy_improved": true,
"benchmark_false_approvals_not_worse": true,
"benchmark_false_denials_not_worse": false,
"benchmark_long_accuracy_not_worse": true,
"audit_accuracy_not_worse": true,
"audit_false_approvals_not_worse": true
},
"passes_score_gates": false
},
{
"name": "kd_b_aug-step-1737",
"path": "/home/user/checkpoints/auto04-v3/kd_b_aug/step-1737",
"sha256": "bca2d2614055c7dc43d6d9f059446691c343b5bf0ca5271cfe6f29f507cc201d",
"accuracy": 0.9683333333333334,
"correct": 2905,
"false_approve_count": 34,
"false_deny_count": 61,
"auroc": 0.9949238884581236,
"nll": 0.09748464604243104,
"long_accuracy": 0.9623430962343096,
"audit_accuracy": 0.9803468208092485,
"audit_false_approve_count": 27,
"validation_nll": 0.06671769760595667,
"validation_balanced_accuracy": 0.9788097419802646,
"gates": {
"benchmark_accuracy_improved": true,
"benchmark_false_approvals_not_worse": true,
"benchmark_false_denials_not_worse": false,
"benchmark_long_accuracy_not_worse": true,
"audit_accuracy_not_worse": true,
"audit_false_approvals_not_worse": true
},
"passes_score_gates": false
},
{
"name": "soup-all-finals",
"path": "/home/user/checkpoints/auto04-v3/selection/soup-all-finals",
"sha256": "075d68c851eac223b91ab8b42767763b152bfb8bfa2adbcd127973aa46358899",
"accuracy": 0.969,
"correct": 2907,
"false_approve_count": 48,
"false_deny_count": 45,
"auroc": 0.9952513147269507,
"nll": 0.1001646734032178,
"long_accuracy": 0.9623430962343096,
"audit_accuracy": 0.981888246628131,
"audit_false_approve_count": 26,
"validation_nll": 0.06923167073800253,
"validation_balanced_accuracy": 0.979988732305995,
"gates": {
"benchmark_accuracy_improved": true,
"benchmark_false_approvals_not_worse": false,
"benchmark_false_denials_not_worse": true,
"benchmark_long_accuracy_not_worse": true,
"audit_accuracy_not_worse": true,
"audit_false_approvals_not_worse": true
},
"passes_score_gates": false
},
{
"name": "soup-all-last2",
"path": "/home/user/checkpoints/auto04-v3/selection/soup-all-last2",
"sha256": "8a9b8aa93546eddb71d0407762b15c236342ca630ad3c3b60dda73fffb42cddc",
"accuracy": 0.9703333333333334,
"correct": 2911,
"false_approve_count": 47,
"false_deny_count": 42,
"auroc": 0.9953059973689838,
"nll": 0.09961677208299435,
"long_accuracy": 0.9623430962343096,
"audit_accuracy": 0.9811175337186898,
"audit_false_approve_count": 26,
"validation_nll": 0.06959238149003957,
"validation_balanced_accuracy": 0.9803779383257147,
"gates": {
"benchmark_accuracy_improved": true,
"benchmark_false_approvals_not_worse": false,
"benchmark_false_denials_not_worse": true,
"benchmark_long_accuracy_not_worse": true,
"audit_accuracy_not_worse": true,
"audit_false_approvals_not_worse": true
},
"passes_score_gates": false
},
{
"name": "soup-kd_b_aug+kd_c",
"path": "/home/user/checkpoints/auto04-v3/selection/soup-kd_b_aug+kd_c",
"sha256": "016e227636321df3955da589bd60d3fa3452946a21f61bda98f28ee24853c52e",
"accuracy": 0.9673333333333334,
"correct": 2902,
"false_approve_count": 41,
"false_deny_count": 57,
"auroc": 0.9951778837505061,
"nll": 0.10191145911240229,
"long_accuracy": 0.9539748953974896,
"audit_accuracy": 0.9799614643545279,
"audit_false_approve_count": 26,
"validation_nll": 0.07110769054353236,
"validation_balanced_accuracy": 0.9803551937925725,
"gates": {
"benchmark_accuracy_improved": false,
"benchmark_false_approvals_not_worse": true,
"benchmark_false_denials_not_worse": false,
"benchmark_long_accuracy_not_worse": true,
"audit_accuracy_not_worse": true,
"audit_false_approvals_not_worse": true
},
"passes_score_gates": false
},
{
"name": "soup-kd_a0.45+kd_b_aug0.55",
"path": "/home/user/checkpoints/auto04-v3/selection/soup-kd_a0.45+kd_b_aug0.55",
"sha256": "bcb137cf2e7dd4599593893a89c82d77bb31050223fa42acd37d4e5e6f0b12f9",
"accuracy": 0.9693333333333334,
"correct": 2908,
"false_approve_count": 37,
"false_deny_count": 55,
"auroc": 0.9950352624923053,
"nll": 0.09734451421267853,
"long_accuracy": 0.9581589958158996,
"audit_accuracy": 0.9803468208092485,
"audit_false_approve_count": 27,
"validation_nll": 0.0667321780183361,
"validation_balanced_accuracy": 0.9784432804936871,
"gates": {
"benchmark_accuracy_improved": true,
"benchmark_false_approvals_not_worse": true,
"benchmark_false_denials_not_worse": false,
"benchmark_long_accuracy_not_worse": true,
"audit_accuracy_not_worse": true,
"audit_false_approvals_not_worse": true
},
"passes_score_gates": false
},
{
"name": "soup-kd_a0.5+kd_b_aug0.5",
"path": "/home/user/checkpoints/auto04-v3/selection/soup-kd_a0.5+kd_b_aug0.5",
"sha256": "8d4a125e417a804e301c6d3542d292bfad57dc709d0b7df3c9aab326a1d6aafa",
"accuracy": 0.9693333333333334,
"correct": 2908,
"false_approve_count": 45,
"false_deny_count": 47,
"auroc": 0.9951743126391897,
"nll": 0.09802602189660843,
"long_accuracy": 0.9581589958158996,
"audit_accuracy": 0.9815028901734104,
"audit_false_approve_count": 27,
"validation_nll": 0.06728941604016982,
"validation_balanced_accuracy": 0.9797368431372293,
"gates": {
"benchmark_accuracy_improved": true,
"benchmark_false_approvals_not_worse": false,
"benchmark_false_denials_not_worse": true,
"benchmark_long_accuracy_not_worse": true,
"audit_accuracy_not_worse": true,
"audit_false_approvals_not_worse": true
},
"passes_score_gates": false
},
{
"name": "soup-kd_a0.35+kd_b_aug0.45+kd_c0.2",
"path": "/home/user/checkpoints/auto04-v3/selection/soup-kd_a0.35+kd_b_aug0.45+kd_c0.2",
"sha256": "8c4c3ca037a8bf289d8447f796d2ae46788de3ac97281328c9efcb0ffa15803b",
"accuracy": 0.97,
"correct": 2910,
"false_approve_count": 46,
"false_deny_count": 44,
"auroc": 0.9951789997227926,
"nll": 0.09976799853938363,
"long_accuracy": 0.9623430962343096,
"audit_accuracy": 0.981888246628131,
"audit_false_approve_count": 26,
"validation_nll": 0.06828081469723489,
"validation_balanced_accuracy": 0.9802482029858082,
"gates": {
"benchmark_accuracy_improved": true,
"benchmark_false_approvals_not_worse": false,
"benchmark_false_denials_not_worse": true,
"benchmark_long_accuracy_not_worse": true,
"audit_accuracy_not_worse": true,
"audit_false_approvals_not_worse": true
},
"passes_score_gates": false
},
{
"name": "soup-kd_b_aug-last2",
"path": "/home/user/checkpoints/auto04-v3/selection/soup-kd_b_aug-last2",
"sha256": "08791b8c4064779a00e89615e6b13f55abb3cd5536acc5de0f418d00bd127b0c",
"accuracy": 0.97,
"correct": 2910,
"false_approve_count": 36,
"false_deny_count": 54,
"auroc": 0.9948738928996933,
"nll": 0.09660789025587242,
"long_accuracy": 0.9581589958158996,
"audit_accuracy": 0.9803468208092485,
"audit_false_approve_count": 28,
"validation_nll": 0.06612161509400427,
"validation_balanced_accuracy": 0.9786913789069291,
"gates": {
"benchmark_accuracy_improved": true,
"benchmark_false_approvals_not_worse": true,
"benchmark_false_denials_not_worse": false,
"benchmark_long_accuracy_not_worse": true,
"audit_accuracy_not_worse": true,
"audit_false_approvals_not_worse": true
},
"passes_score_gates": false
}
]
}
|