| { |
| "overall": { |
| "n": 7824, |
| "accuracy": 0.9810838445807771, |
| "balanced_accuracy": 0.9810304057807715, |
| "f1_deny": 0.9814210394175245, |
| "auroc": 0.9970296489368237, |
| "nll": 0.07466706779386763, |
| "false_approve_rate": 0.015365239294710327, |
| "false_deny_rate": 0.022573949143746757, |
| "false_approve_count": 61, |
| "false_deny_count": 87, |
| "deny_count": 3970, |
| "approve_count": 3854, |
| "threshold": 0.5, |
| "brier": 0.016390781243667968, |
| "accuracy_ci95": [ |
| 0.977820692838512, |
| 0.9838748192131761 |
| ] |
| }, |
| "slices": { |
| "length": { |
| "16k-64k": { |
| "n": 113, |
| "accuracy": 0.9646017699115044, |
| "balanced_accuracy": 0.9633647798742138, |
| "f1_deny": 0.9672131147540983, |
| "auroc": 0.9867924528301887, |
| "nll": 0.1965060950318625, |
| "false_approve_rate": 0.016666666666666666, |
| "false_deny_rate": 0.05660377358490566, |
| "false_approve_count": 1, |
| "false_deny_count": 3, |
| "deny_count": 60, |
| "approve_count": 53, |
| "threshold": 0.5, |
| "brier": 0.0329039183166851, |
| "accuracy_ci95": [ |
| 0.9125037991051361, |
| 0.9861498214486251 |
| ] |
| }, |
| "1k-4k": { |
| "n": 143, |
| "accuracy": 0.986013986013986, |
| "balanced_accuracy": 0.9821428571428572, |
| "f1_deny": 0.9886363636363636, |
| "auroc": 0.9995894909688013, |
| "nll": 0.04954786688116823, |
| "false_approve_rate": 0.0, |
| "false_deny_rate": 0.03571428571428571, |
| "false_approve_count": 0, |
| "false_deny_count": 2, |
| "deny_count": 87, |
| "approve_count": 56, |
| "threshold": 0.5, |
| "brier": 0.013246689350871695, |
| "accuracy_ci95": [ |
| 0.9504430359191713, |
| 0.9961561133696599 |
| ] |
| }, |
| "4k-16k": { |
| "n": 376, |
| "accuracy": 0.973404255319149, |
| "balanced_accuracy": 0.9733752086693264, |
| "f1_deny": 0.9731182795698925, |
| "auroc": 0.9959822312763489, |
| "nll": 0.10496513153205347, |
| "false_approve_rate": 0.03208556149732621, |
| "false_deny_rate": 0.021164021164021163, |
| "false_approve_count": 6, |
| "false_deny_count": 4, |
| "deny_count": 187, |
| "approve_count": 189, |
| "threshold": 0.5, |
| "brier": 0.025859532032310437, |
| "accuracy_ci95": [ |
| 0.9517423311582142, |
| 0.9854908005417445 |
| ] |
| }, |
| "<1k": { |
| "n": 7192, |
| "accuracy": 0.9816462736373749, |
| "balanced_accuracy": 0.9816068783481273, |
| "f1_deny": 0.9819078947368421, |
| "auroc": 0.9971603951733756, |
| "nll": 0.0716682029005229, |
| "false_approve_rate": 0.01485148514851485, |
| "false_deny_rate": 0.021934758155230598, |
| "false_approve_count": 54, |
| "false_deny_count": 78, |
| "deny_count": 3636, |
| "approve_count": 3556, |
| "threshold": 0.5, |
| "brier": 0.015698813829164267, |
| "accuracy_ci95": [ |
| 0.9782771864348433, |
| 0.9845011126211521 |
| ] |
| } |
| }, |
| "category": { |
| "abuse_spam_privacy": { |
| "n": 310, |
| "accuracy": 0.9903225806451613, |
| "balanced_accuracy": 0.9903225806451613, |
| "f1_deny": 0.9951377633711507, |
| "auroc": null, |
| "nll": 0.037631711755560175, |
| "false_approve_rate": 0.00967741935483871, |
| "false_deny_rate": null, |
| "false_approve_count": 3, |
| "false_deny_count": 0, |
| "deny_count": 310, |
| "approve_count": 0, |
| "threshold": 0.5, |
| "brier": 0.009104478030695745, |
| "accuracy_ci95": [ |
| 0.9719384899764231, |
| 0.99670345153865 |
| ] |
| }, |
| "covering_tracks": { |
| "n": 274, |
| "accuracy": 0.9963503649635036, |
| "balanced_accuracy": 0.9963503649635036, |
| "f1_deny": 0.9981718464351006, |
| "auroc": null, |
| "nll": 0.019892654547617225, |
| "false_approve_rate": 0.0036496350364963502, |
| "false_deny_rate": null, |
| "false_approve_count": 1, |
| "false_deny_count": 0, |
| "deny_count": 274, |
| "approve_count": 0, |
| "threshold": 0.5, |
| "brier": 0.003685766902901019, |
| "accuracy_ci95": [ |
| 0.9796201091088617, |
| 0.9993554590245322 |
| ] |
| }, |
| "credential_exfiltration": { |
| "n": 287, |
| "accuracy": 0.9860627177700348, |
| "balanced_accuracy": 0.9860627177700348, |
| "f1_deny": 0.9929824561403509, |
| "auroc": null, |
| "nll": 0.07613174209685845, |
| "false_approve_rate": 0.013937282229965157, |
| "false_deny_rate": null, |
| "false_approve_count": 4, |
| "false_deny_count": 0, |
| "deny_count": 287, |
| "approve_count": 0, |
| "threshold": 0.5, |
| "brier": 0.01452267829885081, |
| "accuracy_ci95": [ |
| 0.9647184498307405, |
| 0.9945670699548189 |
| ] |
| }, |
| "data_destruction_unrequested": { |
| "n": 278, |
| "accuracy": 0.9676258992805755, |
| "balanced_accuracy": 0.9676258992805755, |
| "f1_deny": 0.9835466179159049, |
| "auroc": null, |
| "nll": 0.1292101789505806, |
| "false_approve_rate": 0.03237410071942446, |
| "false_deny_rate": null, |
| "false_approve_count": 9, |
| "false_deny_count": 0, |
| "deny_count": 278, |
| "approve_count": 0, |
| "threshold": 0.5, |
| "brier": 0.03146671360063636, |
| "accuracy_ci95": [ |
| 0.9396283290950204, |
| 0.9828761210590303 |
| ] |
| }, |
| "destructive_filesystem": { |
| "n": 268, |
| "accuracy": 0.9776119402985075, |
| "balanced_accuracy": 0.9776119402985075, |
| "f1_deny": 0.9886792452830189, |
| "auroc": null, |
| "nll": 0.09564842863638352, |
| "false_approve_rate": 0.022388059701492536, |
| "false_deny_rate": null, |
| "false_approve_count": 6, |
| "false_deny_count": 0, |
| "deny_count": 268, |
| "approve_count": 0, |
| "threshold": 0.5, |
| "brier": 0.021873682539620194, |
| "accuracy_ci95": [ |
| 0.9520254728566324, |
| 0.9896999030686671 |
| ] |
| }, |
| "explicitly_authorized": { |
| "n": 411, |
| "accuracy": 0.9683698296836983, |
| "balanced_accuracy": 0.9683698296836983, |
| "f1_deny": 0.0, |
| "auroc": null, |
| "nll": 0.09315550745163068, |
| "false_approve_rate": null, |
| "false_deny_rate": 0.031630170316301706, |
| "false_approve_count": 0, |
| "false_deny_count": 13, |
| "deny_count": 0, |
| "approve_count": 411, |
| "threshold": 0.5, |
| "brier": 0.026205835591117808, |
| "accuracy_ci95": [ |
| 0.9466417840164061, |
| 0.9814236053361263 |
| ] |
| }, |
| "irreversible_remote": { |
| "n": 258, |
| "accuracy": 0.9651162790697675, |
| "balanced_accuracy": 0.9651162790697675, |
| "f1_deny": 0.9822485207100592, |
| "auroc": null, |
| "nll": 0.11917173125956398, |
| "false_approve_rate": 0.03488372093023256, |
| "false_deny_rate": null, |
| "false_approve_count": 9, |
| "false_deny_count": 0, |
| "deny_count": 258, |
| "approve_count": 0, |
| "threshold": 0.5, |
| "brier": 0.02749121215836358, |
| "accuracy_ci95": [ |
| 0.9350442157330028, |
| 0.9815409617287112 |
| ] |
| }, |
| "legit_remote_write": { |
| "n": 449, |
| "accuracy": 0.9821826280623608, |
| "balanced_accuracy": 0.9821826280623608, |
| "f1_deny": 0.0, |
| "auroc": null, |
| "nll": 0.057629023807640636, |
| "false_approve_rate": null, |
| "false_deny_rate": 0.017817371937639197, |
| "false_approve_count": 0, |
| "false_deny_count": 8, |
| "deny_count": 0, |
| "approve_count": 449, |
| "threshold": 0.5, |
| "brier": 0.013629017621982832, |
| "accuracy_ci95": [ |
| 0.9652399141854918, |
| 0.990944621408122 |
| ] |
| }, |
| "malicious_install": { |
| "n": 279, |
| "accuracy": 0.96415770609319, |
| "balanced_accuracy": 0.96415770609319, |
| "f1_deny": 0.9817518248175182, |
| "auroc": null, |
| "nll": 0.1322103744723886, |
| "false_approve_rate": 0.035842293906810034, |
| "false_deny_rate": null, |
| "false_approve_count": 10, |
| "false_deny_count": 0, |
| "deny_count": 279, |
| "approve_count": 0, |
| "threshold": 0.5, |
| "brier": 0.028946100529865137, |
| "accuracy_ci95": [ |
| 0.9352906241343547, |
| 0.9804167162247958 |
| ] |
| }, |
| "network_is_fine": { |
| "n": 418, |
| "accuracy": 0.9880382775119617, |
| "balanced_accuracy": 0.9880382775119617, |
| "f1_deny": 0.0, |
| "auroc": null, |
| "nll": 0.059473055592739785, |
| "false_approve_rate": null, |
| "false_deny_rate": 0.011961722488038277, |
| "false_approve_count": 0, |
| "false_deny_count": 5, |
| "deny_count": 0, |
| "approve_count": 418, |
| "threshold": 0.5, |
| "brier": 0.012417969501463335, |
| "accuracy_ci95": [ |
| 0.9723078671036934, |
| 0.9948801403069828 |
| ] |
| }, |
| "obfuscated_payload": { |
| "n": 297, |
| "accuracy": 0.9764309764309764, |
| "balanced_accuracy": 0.9764309764309764, |
| "f1_deny": 0.9880749574105622, |
| "auroc": null, |
| "nll": 0.11264546405687856, |
| "false_approve_rate": 0.02356902356902357, |
| "false_deny_rate": null, |
| "false_approve_count": 7, |
| "false_deny_count": 0, |
| "deny_count": 297, |
| "approve_count": 0, |
| "threshold": 0.5, |
| "brier": 0.021811617953510727, |
| "accuracy_ci95": [ |
| 0.9521575577793241, |
| 0.9885372557259916 |
| ] |
| }, |
| "physical_financial": { |
| "n": 287, |
| "accuracy": 1.0, |
| "balanced_accuracy": 1.0, |
| "f1_deny": 1.0, |
| "auroc": null, |
| "nll": 0.005254500958214929, |
| "false_approve_rate": 0.0, |
| "false_deny_rate": null, |
| "false_approve_count": 0, |
| "false_deny_count": 0, |
| "deny_count": 287, |
| "approve_count": 0, |
| "threshold": 0.5, |
| "brier": 0.0009348776616970526, |
| "accuracy_ci95": [ |
| 0.9867919146181204, |
| 0.9999999999999999 |
| ] |
| }, |
| "privilege_escalation": { |
| "n": 263, |
| "accuracy": 0.9961977186311787, |
| "balanced_accuracy": 0.9961977186311787, |
| "f1_deny": 0.9980952380952381, |
| "auroc": null, |
| "nll": 0.01020372379132824, |
| "false_approve_rate": 0.0038022813688212928, |
| "false_deny_rate": null, |
| "false_approve_count": 1, |
| "false_deny_count": 0, |
| "deny_count": 263, |
| "approve_count": 0, |
| "threshold": 0.5, |
| "brier": 0.0028943030814574224, |
| "accuracy_ci95": [ |
| 0.9787803900540141, |
| 0.9993284883544965 |
| ] |
| }, |
| "privileged_but_normal": { |
| "n": 415, |
| "accuracy": 0.983132530120482, |
| "balanced_accuracy": 0.983132530120482, |
| "f1_deny": 0.0, |
| "auroc": null, |
| "nll": 0.049203993588815366, |
| "false_approve_rate": null, |
| "false_deny_rate": 0.016867469879518072, |
| "false_approve_count": 0, |
| "false_deny_count": 7, |
| "deny_count": 0, |
| "approve_count": 415, |
| "threshold": 0.5, |
| "brier": 0.011476139661301546, |
| "accuracy_ci95": [ |
| 0.9655969328350357, |
| 0.9918059019489679 |
| ] |
| }, |
| "prompt_injection": { |
| "n": 307, |
| "accuracy": 0.993485342019544, |
| "balanced_accuracy": 0.993485342019544, |
| "f1_deny": 0.9967320261437909, |
| "auroc": null, |
| "nll": 0.02716892299974833, |
| "false_approve_rate": 0.006514657980456026, |
| "false_deny_rate": null, |
| "false_approve_count": 2, |
| "false_deny_count": 0, |
| "deny_count": 307, |
| "approve_count": 0, |
| "threshold": 0.5, |
| "brier": 0.006303394396228876, |
| "accuracy_ci95": [ |
| 0.9765618220724711, |
| 0.998211624010159 |
| ] |
| }, |
| "read_only_sensitive": { |
| "n": 429, |
| "accuracy": 0.9953379953379954, |
| "balanced_accuracy": 0.9953379953379954, |
| "f1_deny": 0.0, |
| "auroc": null, |
| "nll": 0.02758879959911519, |
| "false_approve_rate": null, |
| "false_deny_rate": 0.004662004662004662, |
| "false_approve_count": 0, |
| "false_deny_count": 2, |
| "deny_count": 0, |
| "approve_count": 429, |
| "threshold": 0.5, |
| "brier": 0.004122251349781164, |
| "accuracy_ci95": [ |
| 0.9831631843726782, |
| 0.9987205777741859 |
| ] |
| }, |
| "recovery_action": { |
| "n": 435, |
| "accuracy": 0.9862068965517241, |
| "balanced_accuracy": 0.9862068965517241, |
| "f1_deny": 0.0, |
| "auroc": null, |
| "nll": 0.043207808787498005, |
| "false_approve_rate": null, |
| "false_deny_rate": 0.013793103448275862, |
| "false_approve_count": 0, |
| "false_deny_count": 6, |
| "deny_count": 0, |
| "approve_count": 435, |
| "threshold": 0.5, |
| "brier": 0.010489661127664906, |
| "accuracy_ci95": [ |
| 0.9702380620528606, |
| 0.9936635737149515 |
| ] |
| }, |
| "routine_benign": { |
| "n": 437, |
| "accuracy": 0.9359267734553776, |
| "balanced_accuracy": 0.9359267734553776, |
| "f1_deny": 0.0, |
| "auroc": null, |
| "nll": 0.2717025233708991, |
| "false_approve_rate": null, |
| "false_deny_rate": 0.06407322654462243, |
| "false_approve_count": 0, |
| "false_deny_count": 28, |
| "deny_count": 0, |
| "approve_count": 437, |
| "threshold": 0.5, |
| "brier": 0.056871149193627804, |
| "accuracy_ci95": [ |
| 0.9089552070191895, |
| 0.9553010746364028 |
| ] |
| }, |
| "scary_but_scoped": { |
| "n": 421, |
| "accuracy": 0.9833729216152018, |
| "balanced_accuracy": 0.9833729216152018, |
| "f1_deny": 0.0, |
| "auroc": null, |
| "nll": 0.07596958025990212, |
| "false_approve_rate": null, |
| "false_deny_rate": 0.0166270783847981, |
| "false_approve_count": 0, |
| "false_deny_count": 7, |
| "deny_count": 0, |
| "approve_count": 421, |
| "threshold": 0.5, |
| "brier": 0.01549264872098051, |
| "accuracy_ci95": [ |
| 0.9660814184426589, |
| 0.991923012503841 |
| ] |
| }, |
| "scope_violation": { |
| "n": 280, |
| "accuracy": 0.9785714285714285, |
| "balanced_accuracy": 0.9785714285714285, |
| "f1_deny": 0.9891696750902527, |
| "auroc": null, |
| "nll": 0.07922056343158597, |
| "false_approve_rate": 0.02142857142857143, |
| "false_deny_rate": null, |
| "false_approve_count": 6, |
| "false_deny_count": 0, |
| "deny_count": 280, |
| "approve_count": 0, |
| "threshold": 0.5, |
| "brier": 0.018062115819733784, |
| "accuracy_ci95": [ |
| 0.9540460917784044, |
| 0.9901429675418503 |
| ] |
| }, |
| "scoped_destructive": { |
| "n": 439, |
| "accuracy": 0.9749430523917996, |
| "balanced_accuracy": 0.9749430523917996, |
| "f1_deny": 0.0, |
| "auroc": null, |
| "nll": 0.118852572368062, |
| "false_approve_rate": null, |
| "false_deny_rate": 0.025056947608200455, |
| "false_approve_count": 0, |
| "false_deny_count": 11, |
| "deny_count": 0, |
| "approve_count": 439, |
| "threshold": 0.5, |
| "brier": 0.022346861337011154, |
| "accuracy_ci95": [ |
| 0.9556941577788168, |
| 0.9859520943821459 |
| ] |
| }, |
| "self_permission_tamper": { |
| "n": 283, |
| "accuracy": 0.9929328621908127, |
| "balanced_accuracy": 0.9929328621908127, |
| "f1_deny": 0.9964539007092199, |
| "auroc": null, |
| "nll": 0.0105969923162185, |
| "false_approve_rate": 0.007067137809187279, |
| "false_deny_rate": null, |
| "false_approve_count": 2, |
| "false_deny_count": 0, |
| "deny_count": 283, |
| "approve_count": 0, |
| "threshold": 0.5, |
| "brier": 0.003884689586377411, |
| "accuracy_ci95": [ |
| 0.9746029508745387, |
| 0.9980597916281205 |
| ] |
| }, |
| "supply_chain": { |
| "n": 299, |
| "accuracy": 0.9966555183946488, |
| "balanced_accuracy": 0.9966555183946488, |
| "f1_deny": 0.998324958123953, |
| "auroc": null, |
| "nll": 0.006376601422277858, |
| "false_approve_rate": 0.0033444816053511705, |
| "false_deny_rate": null, |
| "false_approve_count": 1, |
| "false_deny_count": 0, |
| "deny_count": 299, |
| "approve_count": 0, |
| "threshold": 0.5, |
| "brier": 0.0016821711401292813, |
| "accuracy_ci95": [ |
| 0.9813017926534328, |
| 0.9994093727086627 |
| ] |
| } |
| }, |
| "difficulty": { |
| "easy": { |
| "n": 2620, |
| "accuracy": 0.9912213740458016, |
| "balanced_accuracy": 0.9917558069457583, |
| "f1_deny": 0.988342625443487, |
| "auroc": 0.9995462910615918, |
| "nll": 0.02969557186002343, |
| "false_approve_rate": 0.0061162079510703364, |
| "false_deny_rate": 0.010372178157413058, |
| "false_approve_count": 6, |
| "false_deny_count": 17, |
| "deny_count": 981, |
| "approve_count": 1639, |
| "threshold": 0.5, |
| "brier": 0.00699402789303988, |
| "accuracy_ci95": [ |
| 0.9868612059063172, |
| 0.9941431879936499 |
| ] |
| }, |
| "hard": { |
| "n": 2446, |
| "accuracy": 0.9726083401471791, |
| "balanced_accuracy": 0.9630126395051768, |
| "f1_deny": 0.9811956216671345, |
| "auroc": 0.9912989948904127, |
| "nll": 0.12517988977328096, |
| "false_approve_rate": 0.015765765765765764, |
| "false_deny_rate": 0.058208955223880594, |
| "false_approve_count": 28, |
| "false_deny_count": 39, |
| "deny_count": 1776, |
| "approve_count": 670, |
| "threshold": 0.5, |
| "brier": 0.025178281109058776, |
| "accuracy_ci95": [ |
| 0.9653615796370701, |
| 0.978372959459575 |
| ] |
| }, |
| "medium": { |
| "n": 2758, |
| "accuracy": 0.9789702683103698, |
| "balanced_accuracy": 0.9788382063780459, |
| "f1_deny": 0.9761316872427983, |
| "auroc": 0.9968320540423726, |
| "nll": 0.07258982224819208, |
| "false_approve_rate": 0.02225886232481451, |
| "false_deny_rate": 0.020064724919093852, |
| "false_approve_count": 27, |
| "false_deny_count": 31, |
| "deny_count": 1213, |
| "approve_count": 1545, |
| "threshold": 0.5, |
| "brier": 0.017523946257409693, |
| "accuracy_ci95": [ |
| 0.9729115663377526, |
| 0.9836965660957917 |
| ] |
| } |
| }, |
| "lang": { |
| "English": { |
| "n": 5428, |
| "accuracy": 0.9836035372144436, |
| "balanced_accuracy": 0.9835333055992099, |
| "f1_deny": 0.983978397839784, |
| "auroc": 0.9982328664031518, |
| "nll": 0.05862748822533993, |
| "false_approve_rate": 0.013001083423618635, |
| "false_deny_rate": 0.01993230537796164, |
| "false_approve_count": 36, |
| "false_deny_count": 53, |
| "deny_count": 2769, |
| "approve_count": 2659, |
| "threshold": 0.5, |
| "brier": 0.01395917758821812, |
| "accuracy_ci95": [ |
| 0.9798670314942498, |
| 0.9866560233115186 |
| ] |
| }, |
| "French": { |
| "n": 206, |
| "accuracy": 0.9805825242718447, |
| "balanced_accuracy": 0.981083892934834, |
| "f1_deny": 0.9814814814814815, |
| "auroc": 0.9965005201929443, |
| "nll": 0.0699305471517769, |
| "false_approve_rate": 0.027522935779816515, |
| "false_deny_rate": 0.010309278350515464, |
| "false_approve_count": 3, |
| "false_deny_count": 1, |
| "deny_count": 109, |
| "approve_count": 97, |
| "threshold": 0.5, |
| "brier": 0.016507670543866187, |
| "accuracy_ci95": [ |
| 0.9511458473408435, |
| 0.992423650854399 |
| ] |
| }, |
| "German": { |
| "n": 254, |
| "accuracy": 0.984251968503937, |
| "balanced_accuracy": 0.984, |
| "f1_deny": 0.983739837398374, |
| "auroc": 0.994108527131783, |
| "nll": 0.0654773518715745, |
| "false_approve_rate": 0.032, |
| "false_deny_rate": 0.0, |
| "false_approve_count": 4, |
| "false_deny_count": 0, |
| "deny_count": 125, |
| "approve_count": 129, |
| "threshold": 0.5, |
| "brier": 0.011795621374171757, |
| "accuracy_ci95": [ |
| 0.9602153534965894, |
| 0.9938592981329268 |
| ] |
| }, |
| "Hindi": { |
| "n": 251, |
| "accuracy": 0.9561752988047809, |
| "balanced_accuracy": 0.9562222222222222, |
| "f1_deny": 0.9565217391304348, |
| "auroc": 0.9917142857142858, |
| "nll": 0.19174955905162402, |
| "false_approve_rate": 0.032, |
| "false_deny_rate": 0.05555555555555555, |
| "false_approve_count": 4, |
| "false_deny_count": 7, |
| "deny_count": 125, |
| "approve_count": 126, |
| "threshold": 0.5, |
| "brier": 0.038518858642580776, |
| "accuracy_ci95": [ |
| 0.923242415595114, |
| 0.9753554854674595 |
| ] |
| }, |
| "Italian": { |
| "n": 241, |
| "accuracy": 0.970954356846473, |
| "balanced_accuracy": 0.9682959514741147, |
| "f1_deny": 0.9750889679715302, |
| "auroc": 0.99061926929045, |
| "nll": 0.1535058537842256, |
| "false_approve_rate": 0.014388489208633094, |
| "false_deny_rate": 0.049019607843137254, |
| "false_approve_count": 2, |
| "false_deny_count": 5, |
| "deny_count": 139, |
| "approve_count": 102, |
| "threshold": 0.5, |
| "brier": 0.02639486041493036, |
| "accuracy_ci95": [ |
| 0.9412700864434193, |
| 0.9858604783447934 |
| ] |
| }, |
| "Japanese": { |
| "n": 238, |
| "accuracy": 0.9789915966386554, |
| "balanced_accuracy": 0.9782608695652174, |
| "f1_deny": 0.9800796812749004, |
| "auroc": 0.9980911983032874, |
| "nll": 0.09417980845069547, |
| "false_approve_rate": 0.0, |
| "false_deny_rate": 0.043478260869565216, |
| "false_approve_count": 0, |
| "false_deny_count": 5, |
| "deny_count": 123, |
| "approve_count": 115, |
| "threshold": 0.5, |
| "brier": 0.01848696917247954, |
| "accuracy_ci95": [ |
| 0.9517725123266033, |
| 0.9909938814580269 |
| ] |
| }, |
| "Korean": { |
| "n": 244, |
| "accuracy": 0.9467213114754098, |
| "balanced_accuracy": 0.948236092265943, |
| "f1_deny": 0.9422222222222222, |
| "auroc": 0.987381275440977, |
| "nll": 0.21541671468129064, |
| "false_approve_rate": 0.03636363636363636, |
| "false_deny_rate": 0.06716417910447761, |
| "false_approve_count": 4, |
| "false_deny_count": 9, |
| "deny_count": 110, |
| "approve_count": 134, |
| "threshold": 0.5, |
| "brier": 0.04566513042801063, |
| "accuracy_ci95": [ |
| 0.9109919664893638, |
| 0.9686025978636346 |
| ] |
| }, |
| "Mandarin Chinese": { |
| "n": 253, |
| "accuracy": 0.9723320158102767, |
| "balanced_accuracy": 0.9723993498374593, |
| "f1_deny": 0.9727626459143969, |
| "auroc": 0.995561390347587, |
| "nll": 0.12095610558253209, |
| "false_approve_rate": 0.031007751937984496, |
| "false_deny_rate": 0.024193548387096774, |
| "false_approve_count": 4, |
| "false_deny_count": 3, |
| "deny_count": 129, |
| "approve_count": 124, |
| "threshold": 0.5, |
| "brier": 0.0250032706336481, |
| "accuracy_ci95": [ |
| 0.9440008247673488, |
| 0.9865343041693329 |
| ] |
| }, |
| "Portuguese": { |
| "n": 237, |
| "accuracy": 0.9873417721518988, |
| "balanced_accuracy": 0.987359350519869, |
| "f1_deny": 0.9873417721518988, |
| "auroc": 0.9962256088876229, |
| "nll": 0.08352940040749732, |
| "false_approve_rate": 0.00847457627118644, |
| "false_deny_rate": 0.01680672268907563, |
| "false_approve_count": 1, |
| "false_deny_count": 2, |
| "deny_count": 118, |
| "approve_count": 119, |
| "threshold": 0.5, |
| "brier": 0.013021824938597248, |
| "accuracy_ci95": [ |
| 0.9634512971313959, |
| 0.9956858926660589 |
| ] |
| }, |
| "Russian": { |
| "n": 211, |
| "accuracy": 0.976303317535545, |
| "balanced_accuracy": 0.976325247079964, |
| "f1_deny": 0.976303317535545, |
| "auroc": 0.9955076370170709, |
| "nll": 0.1060892016452753, |
| "false_approve_rate": 0.02830188679245283, |
| "false_deny_rate": 0.01904761904761905, |
| "false_approve_count": 3, |
| "false_deny_count": 2, |
| "deny_count": 106, |
| "approve_count": 105, |
| "threshold": 0.5, |
| "brier": 0.02245331292102234, |
| "accuracy_ci95": [ |
| 0.9457369201453067, |
| 0.9898366935479135 |
| ] |
| }, |
| "Spanish": { |
| "n": 261, |
| "accuracy": 1.0, |
| "balanced_accuracy": 1.0, |
| "f1_deny": 1.0, |
| "auroc": 1.0, |
| "nll": 0.00783327666613883, |
| "false_approve_rate": 0.0, |
| "false_deny_rate": 0.0, |
| "false_approve_count": 0, |
| "false_deny_count": 0, |
| "deny_count": 117, |
| "approve_count": 144, |
| "threshold": 0.5, |
| "brier": 0.001353081098341837, |
| "accuracy_ci95": [ |
| 0.9854952512427636, |
| 0.9999999999999999 |
| ] |
| } |
| } |
| } |
| } |
|
|
