| { |
| "overall": { |
| "n": 7824, |
| "accuracy": 0.980700408997955, |
| "balanced_accuracy": 0.9806336182500042, |
| "f1_deny": 0.9810610811488775, |
| "auroc": 0.9969489319873103, |
| "nll": 0.077690421882292, |
| "false_approve_rate": 0.014861460957178841, |
| "false_deny_rate": 0.023871302542812663, |
| "false_approve_count": 59, |
| "false_deny_count": 92, |
| "deny_count": 3970, |
| "approve_count": 3854, |
| "threshold": 0.5, |
| "brier": 0.01706805074362081, |
| "accuracy_ci95": [ |
| 0.977407712890097, |
| 0.9835213043326444 |
| ] |
| }, |
| "slices": { |
| "length": { |
| "16k-64k": { |
| "n": 113, |
| "accuracy": 0.9646017699115044, |
| "balanced_accuracy": 0.9633647798742138, |
| "f1_deny": 0.9672131147540983, |
| "auroc": 0.9871069182389938, |
| "nll": 0.21413471520465166, |
| "false_approve_rate": 0.016666666666666666, |
| "false_deny_rate": 0.05660377358490566, |
| "false_approve_count": 1, |
| "false_deny_count": 3, |
| "deny_count": 60, |
| "approve_count": 53, |
| "threshold": 0.5, |
| "brier": 0.03441951796452074, |
| "accuracy_ci95": [ |
| 0.9125037991051361, |
| 0.9861498214486251 |
| ] |
| }, |
| "1k-4k": { |
| "n": 143, |
| "accuracy": 0.9790209790209791, |
| "balanced_accuracy": 0.9732142857142857, |
| "f1_deny": 0.9830508474576272, |
| "auroc": 0.9995894909688012, |
| "nll": 0.05782564103624885, |
| "false_approve_rate": 0.0, |
| "false_deny_rate": 0.05357142857142857, |
| "false_approve_count": 0, |
| "false_deny_count": 3, |
| "deny_count": 87, |
| "approve_count": 56, |
| "threshold": 0.5, |
| "brier": 0.01586240230637597, |
| "accuracy_ci95": [ |
| 0.940139034865942, |
| 0.9928399827784458 |
| ] |
| }, |
| "4k-16k": { |
| "n": 376, |
| "accuracy": 0.9707446808510638, |
| "balanced_accuracy": 0.9707297060238236, |
| "f1_deny": 0.9705093833780161, |
| "auroc": 0.9956709956709958, |
| "nll": 0.10595072026757134, |
| "false_approve_rate": 0.03208556149732621, |
| "false_deny_rate": 0.026455026455026454, |
| "false_approve_count": 6, |
| "false_deny_count": 5, |
| "deny_count": 187, |
| "approve_count": 189, |
| "threshold": 0.5, |
| "brier": 0.024852978196673328, |
| "accuracy_ci95": [ |
| 0.9483805403072181, |
| 0.9835872367202687 |
| ] |
| }, |
| "<1k": { |
| "n": 7192, |
| "accuracy": 0.9815072302558399, |
| "balanced_accuracy": 0.9814600835786615, |
| "f1_deny": 0.9817833173537871, |
| "auroc": 0.9970814291777884, |
| "nll": 0.07446414078144278, |
| "false_approve_rate": 0.014301430143014302, |
| "false_deny_rate": 0.022778402699662543, |
| "false_approve_count": 52, |
| "false_deny_count": 81, |
| "deny_count": 3636, |
| "approve_count": 3556, |
| "threshold": 0.5, |
| "brier": 0.01641239991050298, |
| "accuracy_ci95": [ |
| 0.9781267577932592, |
| 0.9843736029546947 |
| ] |
| } |
| }, |
| "category": { |
| "abuse_spam_privacy": { |
| "n": 310, |
| "accuracy": 0.9838709677419355, |
| "balanced_accuracy": 0.9838709677419355, |
| "f1_deny": 0.991869918699187, |
| "auroc": null, |
| "nll": 0.06163647982522161, |
| "false_approve_rate": 0.016129032258064516, |
| "false_deny_rate": null, |
| "false_approve_count": 5, |
| "false_deny_count": 0, |
| "deny_count": 310, |
| "approve_count": 0, |
| "threshold": 0.5, |
| "brier": 0.01506197567154289, |
| "accuracy_ci95": [ |
| 0.9628051813387815, |
| 0.9930914714721459 |
| ] |
| }, |
| "covering_tracks": { |
| "n": 274, |
| "accuracy": 0.9963503649635036, |
| "balanced_accuracy": 0.9963503649635036, |
| "f1_deny": 0.9981718464351006, |
| "auroc": null, |
| "nll": 0.015471839150125203, |
| "false_approve_rate": 0.0036496350364963502, |
| "false_deny_rate": null, |
| "false_approve_count": 1, |
| "false_deny_count": 0, |
| "deny_count": 274, |
| "approve_count": 0, |
| "threshold": 0.5, |
| "brier": 0.0035673069510072457, |
| "accuracy_ci95": [ |
| 0.9796201091088617, |
| 0.9993554590245322 |
| ] |
| }, |
| "credential_exfiltration": { |
| "n": 287, |
| "accuracy": 0.9860627177700348, |
| "balanced_accuracy": 0.9860627177700348, |
| "f1_deny": 0.9929824561403509, |
| "auroc": null, |
| "nll": 0.0730731415996861, |
| "false_approve_rate": 0.013937282229965157, |
| "false_deny_rate": null, |
| "false_approve_count": 4, |
| "false_deny_count": 0, |
| "deny_count": 287, |
| "approve_count": 0, |
| "threshold": 0.5, |
| "brier": 0.01300513119181707, |
| "accuracy_ci95": [ |
| 0.9647184498307405, |
| 0.9945670699548189 |
| ] |
| }, |
| "data_destruction_unrequested": { |
| "n": 278, |
| "accuracy": 0.9676258992805755, |
| "balanced_accuracy": 0.9676258992805755, |
| "f1_deny": 0.9835466179159049, |
| "auroc": null, |
| "nll": 0.10790350606599937, |
| "false_approve_rate": 0.03237410071942446, |
| "false_deny_rate": null, |
| "false_approve_count": 9, |
| "false_deny_count": 0, |
| "deny_count": 278, |
| "approve_count": 0, |
| "threshold": 0.5, |
| "brier": 0.028138397972208695, |
| "accuracy_ci95": [ |
| 0.9396283290950204, |
| 0.9828761210590303 |
| ] |
| }, |
| "destructive_filesystem": { |
| "n": 268, |
| "accuracy": 0.9701492537313433, |
| "balanced_accuracy": 0.9701492537313433, |
| "f1_deny": 0.9848484848484849, |
| "auroc": null, |
| "nll": 0.0930448990814171, |
| "false_approve_rate": 0.029850746268656716, |
| "false_deny_rate": null, |
| "false_approve_count": 8, |
| "false_deny_count": 0, |
| "deny_count": 268, |
| "approve_count": 0, |
| "threshold": 0.5, |
| "brier": 0.024122717136063627, |
| "accuracy_ci95": [ |
| 0.9422128212881589, |
| 0.9847980956383078 |
| ] |
| }, |
| "explicitly_authorized": { |
| "n": 411, |
| "accuracy": 0.9635036496350365, |
| "balanced_accuracy": 0.9635036496350365, |
| "f1_deny": 0.0, |
| "auroc": null, |
| "nll": 0.1142612228534276, |
| "false_approve_rate": null, |
| "false_deny_rate": 0.0364963503649635, |
| "false_approve_count": 0, |
| "false_deny_count": 15, |
| "deny_count": 0, |
| "approve_count": 411, |
| "threshold": 0.5, |
| "brier": 0.028116889884426167, |
| "accuracy_ci95": [ |
| 0.9406630929319568, |
| 0.9777600586091206 |
| ] |
| }, |
| "irreversible_remote": { |
| "n": 258, |
| "accuracy": 0.9612403100775194, |
| "balanced_accuracy": 0.9612403100775194, |
| "f1_deny": 0.9802371541501976, |
| "auroc": null, |
| "nll": 0.1399646728925087, |
| "false_approve_rate": 0.03875968992248062, |
| "false_deny_rate": null, |
| "false_approve_count": 10, |
| "false_deny_count": 0, |
| "deny_count": 258, |
| "approve_count": 0, |
| "threshold": 0.5, |
| "brier": 0.03468899762906791, |
| "accuracy_ci95": [ |
| 0.9301344112122186, |
| 0.9788125564373146 |
| ] |
| }, |
| "legit_remote_write": { |
| "n": 449, |
| "accuracy": 0.9888641425389755, |
| "balanced_accuracy": 0.9888641425389755, |
| "f1_deny": 0.0, |
| "auroc": null, |
| "nll": 0.04640276728835062, |
| "false_approve_rate": null, |
| "false_deny_rate": 0.011135857461024499, |
| "false_approve_count": 0, |
| "false_deny_count": 5, |
| "deny_count": 0, |
| "approve_count": 449, |
| "threshold": 0.5, |
| "brier": 0.010374962135319988, |
| "accuracy_ci95": [ |
| 0.9741998990447701, |
| 0.9952343067880162 |
| ] |
| }, |
| "malicious_install": { |
| "n": 279, |
| "accuracy": 0.974910394265233, |
| "balanced_accuracy": 0.974910394265233, |
| "f1_deny": 0.9872958257713249, |
| "auroc": null, |
| "nll": 0.10793239004546554, |
| "false_approve_rate": 0.025089605734767026, |
| "false_deny_rate": null, |
| "false_approve_count": 7, |
| "false_deny_count": 0, |
| "deny_count": 279, |
| "approve_count": 0, |
| "threshold": 0.5, |
| "brier": 0.02252173491586305, |
| "accuracy_ci95": [ |
| 0.9491260832448002, |
| 0.9877945545744079 |
| ] |
| }, |
| "network_is_fine": { |
| "n": 418, |
| "accuracy": 0.9832535885167464, |
| "balanced_accuracy": 0.9832535885167464, |
| "f1_deny": 0.0, |
| "auroc": null, |
| "nll": 0.06523335216777996, |
| "false_approve_rate": null, |
| "false_deny_rate": 0.01674641148325359, |
| "false_approve_count": 0, |
| "false_deny_count": 7, |
| "deny_count": 0, |
| "approve_count": 418, |
| "threshold": 0.5, |
| "brier": 0.01423771966314547, |
| "accuracy_ci95": [ |
| 0.965840893363533, |
| 0.9918648786803719 |
| ] |
| }, |
| "obfuscated_payload": { |
| "n": 297, |
| "accuracy": 0.9797979797979798, |
| "balanced_accuracy": 0.9797979797979798, |
| "f1_deny": 0.9897959183673469, |
| "auroc": null, |
| "nll": 0.09885219297615432, |
| "false_approve_rate": 0.020202020202020204, |
| "false_deny_rate": null, |
| "false_approve_count": 6, |
| "false_deny_count": 0, |
| "deny_count": 297, |
| "approve_count": 0, |
| "threshold": 0.5, |
| "brier": 0.020501236869110217, |
| "accuracy_ci95": [ |
| 0.9566336937562595, |
| 0.990709139632486 |
| ] |
| }, |
| "physical_financial": { |
| "n": 287, |
| "accuracy": 1.0, |
| "balanced_accuracy": 1.0, |
| "f1_deny": 1.0, |
| "auroc": null, |
| "nll": 0.007088014237988346, |
| "false_approve_rate": 0.0, |
| "false_deny_rate": null, |
| "false_approve_count": 0, |
| "false_deny_count": 0, |
| "deny_count": 287, |
| "approve_count": 0, |
| "threshold": 0.5, |
| "brier": 0.0017775115475705129, |
| "accuracy_ci95": [ |
| 0.9867919146181204, |
| 0.9999999999999999 |
| ] |
| }, |
| "privilege_escalation": { |
| "n": 263, |
| "accuracy": 1.0, |
| "balanced_accuracy": 1.0, |
| "f1_deny": 1.0, |
| "auroc": null, |
| "nll": 0.006168101924561926, |
| "false_approve_rate": 0.0, |
| "false_deny_rate": null, |
| "false_approve_count": 0, |
| "false_deny_count": 0, |
| "deny_count": 263, |
| "approve_count": 0, |
| "threshold": 0.5, |
| "brier": 0.001242727033941512, |
| "accuracy_ci95": [ |
| 0.9856039655993808, |
| 0.9999999999999999 |
| ] |
| }, |
| "privileged_but_normal": { |
| "n": 415, |
| "accuracy": 0.9855421686746988, |
| "balanced_accuracy": 0.9855421686746988, |
| "f1_deny": 0.0, |
| "auroc": null, |
| "nll": 0.0482992619886494, |
| "false_approve_rate": null, |
| "false_deny_rate": 0.014457831325301205, |
| "false_approve_count": 0, |
| "false_deny_count": 6, |
| "deny_count": 0, |
| "approve_count": 415, |
| "threshold": 0.5, |
| "brier": 0.011879101914792137, |
| "accuracy_ci95": [ |
| 0.9688204655885464, |
| 0.9933574456781702 |
| ] |
| }, |
| "prompt_injection": { |
| "n": 307, |
| "accuracy": 0.993485342019544, |
| "balanced_accuracy": 0.993485342019544, |
| "f1_deny": 0.9967320261437909, |
| "auroc": null, |
| "nll": 0.0332496360787903, |
| "false_approve_rate": 0.006514657980456026, |
| "false_deny_rate": null, |
| "false_approve_count": 2, |
| "false_deny_count": 0, |
| "deny_count": 307, |
| "approve_count": 0, |
| "threshold": 0.5, |
| "brier": 0.006436516812583183, |
| "accuracy_ci95": [ |
| 0.9765618220724711, |
| 0.998211624010159 |
| ] |
| }, |
| "read_only_sensitive": { |
| "n": 429, |
| "accuracy": 0.9953379953379954, |
| "balanced_accuracy": 0.9953379953379954, |
| "f1_deny": 0.0, |
| "auroc": null, |
| "nll": 0.03402789735856153, |
| "false_approve_rate": null, |
| "false_deny_rate": 0.004662004662004662, |
| "false_approve_count": 0, |
| "false_deny_count": 2, |
| "deny_count": 0, |
| "approve_count": 429, |
| "threshold": 0.5, |
| "brier": 0.005005183300734143, |
| "accuracy_ci95": [ |
| 0.9831631843726782, |
| 0.9987205777741859 |
| ] |
| }, |
| "recovery_action": { |
| "n": 435, |
| "accuracy": 0.9839080459770115, |
| "balanced_accuracy": 0.9839080459770115, |
| "f1_deny": 0.0, |
| "auroc": null, |
| "nll": 0.055353120390087884, |
| "false_approve_rate": null, |
| "false_deny_rate": 0.016091954022988506, |
| "false_approve_count": 0, |
| "false_deny_count": 7, |
| "deny_count": 0, |
| "approve_count": 435, |
| "threshold": 0.5, |
| "brier": 0.013146213810188472, |
| "accuracy_ci95": [ |
| 0.9671605088302577, |
| 0.9921836723949171 |
| ] |
| }, |
| "routine_benign": { |
| "n": 437, |
| "accuracy": 0.9336384439359268, |
| "balanced_accuracy": 0.9336384439359268, |
| "f1_deny": 0.0, |
| "auroc": null, |
| "nll": 0.30707305643818655, |
| "false_approve_rate": null, |
| "false_deny_rate": 0.06636155606407322, |
| "false_approve_count": 0, |
| "false_deny_count": 29, |
| "deny_count": 0, |
| "approve_count": 437, |
| "threshold": 0.5, |
| "brier": 0.060771827248824184, |
| "accuracy_ci95": [ |
| 0.9063188684069496, |
| 0.9534006348672484 |
| ] |
| }, |
| "scary_but_scoped": { |
| "n": 421, |
| "accuracy": 0.9857482185273159, |
| "balanced_accuracy": 0.9857482185273159, |
| "f1_deny": 0.0, |
| "auroc": null, |
| "nll": 0.07221233015570949, |
| "false_approve_rate": null, |
| "false_deny_rate": 0.014251781472684086, |
| "false_approve_count": 0, |
| "false_deny_count": 6, |
| "deny_count": 0, |
| "approve_count": 421, |
| "threshold": 0.5, |
| "brier": 0.014724889003222942, |
| "accuracy_ci95": [ |
| 0.9692597238331009, |
| 0.9934523455947086 |
| ] |
| }, |
| "scope_violation": { |
| "n": 280, |
| "accuracy": 0.9785714285714285, |
| "balanced_accuracy": 0.9785714285714285, |
| "f1_deny": 0.9891696750902527, |
| "auroc": null, |
| "nll": 0.07545524686769123, |
| "false_approve_rate": 0.02142857142857143, |
| "false_deny_rate": null, |
| "false_approve_count": 6, |
| "false_deny_count": 0, |
| "deny_count": 280, |
| "approve_count": 0, |
| "threshold": 0.5, |
| "brier": 0.018687184583672765, |
| "accuracy_ci95": [ |
| 0.9540460917784044, |
| 0.9901429675418503 |
| ] |
| }, |
| "scoped_destructive": { |
| "n": 439, |
| "accuracy": 0.9658314350797267, |
| "balanced_accuracy": 0.9658314350797267, |
| "f1_deny": 0.0, |
| "auroc": null, |
| "nll": 0.13050612041453316, |
| "false_approve_rate": null, |
| "false_deny_rate": 0.03416856492027335, |
| "false_approve_count": 0, |
| "false_deny_count": 15, |
| "deny_count": 0, |
| "approve_count": 439, |
| "threshold": 0.5, |
| "brier": 0.027575064914745307, |
| "accuracy_ci95": [ |
| 0.9443951790852512, |
| 0.9791859171589545 |
| ] |
| }, |
| "self_permission_tamper": { |
| "n": 283, |
| "accuracy": 1.0, |
| "balanced_accuracy": 1.0, |
| "f1_deny": 1.0, |
| "auroc": null, |
| "nll": 0.0006772147589523223, |
| "false_approve_rate": 0.0, |
| "false_deny_rate": null, |
| "false_approve_count": 0, |
| "false_deny_count": 0, |
| "deny_count": 283, |
| "approve_count": 0, |
| "threshold": 0.5, |
| "brier": 5.628910671531875e-05, |
| "accuracy_ci95": [ |
| 0.9866077280582531, |
| 1.0 |
| ] |
| }, |
| "supply_chain": { |
| "n": 299, |
| "accuracy": 0.9966555183946488, |
| "balanced_accuracy": 0.9966555183946488, |
| "f1_deny": 0.998324958123953, |
| "auroc": null, |
| "nll": 0.007233097489571487, |
| "false_approve_rate": 0.0033444816053511705, |
| "false_deny_rate": null, |
| "false_approve_count": 1, |
| "false_deny_count": 0, |
| "deny_count": 299, |
| "approve_count": 0, |
| "threshold": 0.5, |
| "brier": 0.0022310041278180153, |
| "accuracy_ci95": [ |
| 0.9813017926534328, |
| 0.9994093727086627 |
| ] |
| } |
| }, |
| "difficulty": { |
| "easy": { |
| "n": 2620, |
| "accuracy": 0.9900763358778626, |
| "balanced_accuracy": 0.9908406147553983, |
| "f1_deny": 0.9868421052631579, |
| "auroc": 0.9993662379599205, |
| "nll": 0.03467158140022426, |
| "false_approve_rate": 0.0061162079510703364, |
| "false_deny_rate": 0.012202562538133009, |
| "false_approve_count": 6, |
| "false_deny_count": 20, |
| "deny_count": 981, |
| "approve_count": 1639, |
| "threshold": 0.5, |
| "brier": 0.008376944688742297, |
| "accuracy_ci95": [ |
| 0.9854988535494524, |
| 0.9932188168216103 |
| ] |
| }, |
| "hard": { |
| "n": 2446, |
| "accuracy": 0.9717906786590351, |
| "balanced_accuracy": 0.961520102191744, |
| "f1_deny": 0.9806451612903225, |
| "auroc": 0.9909267009546862, |
| "nll": 0.13003705933736517, |
| "false_approve_rate": 0.015765765765765764, |
| "false_deny_rate": 0.06119402985074627, |
| "false_approve_count": 28, |
| "false_deny_count": 41, |
| "deny_count": 1776, |
| "approve_count": 670, |
| "threshold": 0.5, |
| "brier": 0.026229460581772845, |
| "accuracy_ci95": [ |
| 0.9644529402046846, |
| 0.9776488401737999 |
| ] |
| }, |
| "medium": { |
| "n": 2758, |
| "accuracy": 0.9796954314720813, |
| "balanced_accuracy": 0.9796626086863722, |
| "f1_deny": 0.9769736842105263, |
| "auroc": 0.9966986556106047, |
| "nll": 0.07213186018827766, |
| "false_approve_rate": 0.020610057708161583, |
| "false_deny_rate": 0.020064724919093852, |
| "false_approve_count": 25, |
| "false_deny_count": 31, |
| "deny_count": 1213, |
| "approve_count": 1545, |
| "threshold": 0.5, |
| "brier": 0.017199265174245113, |
| "accuracy_ci95": [ |
| 0.9737260081872405, |
| 0.9843304333037355 |
| ] |
| } |
| }, |
| "lang": { |
| "English": { |
| "n": 5428, |
| "accuracy": 0.9828666175386883, |
| "balanced_accuracy": 0.9827736731184495, |
| "f1_deny": 0.9832763891386441, |
| "auroc": 0.9980704275604934, |
| "nll": 0.06354973063779235, |
| "false_approve_rate": 0.012639942217407007, |
| "false_deny_rate": 0.02181271154569387, |
| "false_approve_count": 35, |
| "false_deny_count": 58, |
| "deny_count": 2769, |
| "approve_count": 2659, |
| "threshold": 0.5, |
| "brier": 0.014905367510619653, |
| "accuracy_ci95": [ |
| 0.9790572866551622, |
| 0.9859929711147118 |
| ] |
| }, |
| "French": { |
| "n": 206, |
| "accuracy": 0.9902912621359223, |
| "balanced_accuracy": 0.9908256880733946, |
| "f1_deny": 0.9907407407407407, |
| "auroc": 0.9955074245720231, |
| "nll": 0.05884932624394884, |
| "false_approve_rate": 0.01834862385321101, |
| "false_deny_rate": 0.0, |
| "false_approve_count": 2, |
| "false_deny_count": 0, |
| "deny_count": 109, |
| "approve_count": 97, |
| "threshold": 0.5, |
| "brier": 0.011234759412049674, |
| "accuracy_ci95": [ |
| 0.9652980447338898, |
| 0.9973334635259029 |
| ] |
| }, |
| "German": { |
| "n": 254, |
| "accuracy": 0.9803149606299213, |
| "balanced_accuracy": 0.9801240310077519, |
| "f1_deny": 0.979757085020243, |
| "auroc": 0.9937054263565892, |
| "nll": 0.08196046532312477, |
| "false_approve_rate": 0.032, |
| "false_deny_rate": 0.007751937984496124, |
| "false_approve_count": 4, |
| "false_deny_count": 1, |
| "deny_count": 125, |
| "approve_count": 129, |
| "threshold": 0.5, |
| "brier": 0.018748695615186974, |
| "accuracy_ci95": [ |
| 0.9547548132607326, |
| 0.991563133883991 |
| ] |
| }, |
| "Hindi": { |
| "n": 251, |
| "accuracy": 0.9601593625498008, |
| "balanced_accuracy": 0.9601904761904761, |
| "f1_deny": 0.9603174603174603, |
| "auroc": 0.9912698412698414, |
| "nll": 0.18339822691191146, |
| "false_approve_rate": 0.032, |
| "false_deny_rate": 0.047619047619047616, |
| "false_approve_count": 4, |
| "false_deny_count": 6, |
| "deny_count": 125, |
| "approve_count": 126, |
| "threshold": 0.5, |
| "brier": 0.035268063020498196, |
| "accuracy_ci95": [ |
| 0.9282281261788624, |
| 0.9782177914868778 |
| ] |
| }, |
| "Italian": { |
| "n": 241, |
| "accuracy": 0.970954356846473, |
| "balanced_accuracy": 0.9696007899562703, |
| "f1_deny": 0.974910394265233, |
| "auroc": 0.9897023557624488, |
| "nll": 0.13629819805016377, |
| "false_approve_rate": 0.02158273381294964, |
| "false_deny_rate": 0.0392156862745098, |
| "false_approve_count": 3, |
| "false_deny_count": 4, |
| "deny_count": 139, |
| "approve_count": 102, |
| "threshold": 0.5, |
| "brier": 0.022402760451532087, |
| "accuracy_ci95": [ |
| 0.9412700864434193, |
| 0.9858604783447934 |
| ] |
| }, |
| "Japanese": { |
| "n": 238, |
| "accuracy": 0.9663865546218487, |
| "balanced_accuracy": 0.9652173913043478, |
| "f1_deny": 0.968503937007874, |
| "auroc": 0.9973135383527748, |
| "nll": 0.1417529012173477, |
| "false_approve_rate": 0.0, |
| "false_deny_rate": 0.06956521739130435, |
| "false_approve_count": 0, |
| "false_deny_count": 8, |
| "deny_count": 123, |
| "approve_count": 115, |
| "threshold": 0.5, |
| "brier": 0.02811355051874001, |
| "accuracy_ci95": [ |
| 0.9350857637502895, |
| 0.98287098809264 |
| ] |
| }, |
| "Korean": { |
| "n": 244, |
| "accuracy": 0.9631147540983607, |
| "balanced_accuracy": 0.9639755766621438, |
| "f1_deny": 0.9596412556053812, |
| "auroc": 0.9913500678426052, |
| "nll": 0.17496961663746408, |
| "false_approve_rate": 0.02727272727272727, |
| "false_deny_rate": 0.04477611940298507, |
| "false_approve_count": 3, |
| "false_deny_count": 6, |
| "deny_count": 110, |
| "approve_count": 134, |
| "threshold": 0.5, |
| "brier": 0.035215579118120316, |
| "accuracy_ci95": [ |
| 0.9313979552211248, |
| 0.9804753087411578 |
| ] |
| }, |
| "Mandarin Chinese": { |
| "n": 253, |
| "accuracy": 0.9762845849802372, |
| "balanced_accuracy": 0.9762753188297074, |
| "f1_deny": 0.9767441860465116, |
| "auroc": 0.9963740935233808, |
| "nll": 0.10736957353464598, |
| "false_approve_rate": 0.023255813953488372, |
| "false_deny_rate": 0.024193548387096774, |
| "false_approve_count": 3, |
| "false_deny_count": 3, |
| "deny_count": 129, |
| "approve_count": 124, |
| "threshold": 0.5, |
| "brier": 0.023716945302300004, |
| "accuracy_ci95": [ |
| 0.9492353089010875, |
| 0.9890867248802525 |
| ] |
| }, |
| "Portuguese": { |
| "n": 237, |
| "accuracy": 0.9873417721518988, |
| "balanced_accuracy": 0.987359350519869, |
| "f1_deny": 0.9873417721518988, |
| "auroc": 0.9951573849878934, |
| "nll": 0.0889259460495608, |
| "false_approve_rate": 0.00847457627118644, |
| "false_deny_rate": 0.01680672268907563, |
| "false_approve_count": 1, |
| "false_deny_count": 2, |
| "deny_count": 118, |
| "approve_count": 119, |
| "threshold": 0.5, |
| "brier": 0.01294754812025605, |
| "accuracy_ci95": [ |
| 0.9634512971313959, |
| 0.9956858926660589 |
| ] |
| }, |
| "Russian": { |
| "n": 211, |
| "accuracy": 0.9620853080568721, |
| "balanced_accuracy": 0.9620844564240791, |
| "f1_deny": 0.9622641509433962, |
| "auroc": 0.9958670260557053, |
| "nll": 0.10983932327248251, |
| "false_approve_rate": 0.03773584905660377, |
| "false_deny_rate": 0.0380952380952381, |
| "false_approve_count": 4, |
| "false_deny_count": 4, |
| "deny_count": 106, |
| "approve_count": 105, |
| "threshold": 0.5, |
| "brier": 0.02988984019027873, |
| "accuracy_ci95": [ |
| 0.926981020439214, |
| 0.9806650226959998 |
| ] |
| }, |
| "Spanish": { |
| "n": 261, |
| "accuracy": 1.0, |
| "balanced_accuracy": 1.0, |
| "f1_deny": 1.0, |
| "auroc": 1.0, |
| "nll": 0.0123921737319317, |
| "false_approve_rate": 0.0, |
| "false_deny_rate": 0.0, |
| "false_approve_count": 0, |
| "false_deny_count": 0, |
| "deny_count": 117, |
| "approve_count": 144, |
| "threshold": 0.5, |
| "brier": 0.002478457026547329, |
| "accuracy_ci95": [ |
| 0.9854952512427636, |
| 0.9999999999999999 |
| ] |
| } |
| } |
| } |
| } |
|
|
