{ "benchmark_default": { "overall": { "n": 3000, "accuracy": 0.9633333333333334, "balanced_accuracy": 0.9632612995541914, "f1_deny": 0.9607982893799002, "auroc": 0.9937157368608771, "nll": 0.12148878966271277, "false_approve_rate": 0.03783012134189864, "false_deny_rate": 0.03564727954971857, "false_approve_count": 53, "false_deny_count": 57, "deny_count": 1401, "approve_count": 1599, "threshold": 0.5, "brier": 0.03046821239640341, "accuracy_ci95": [ 0.9559937359506702, 0.9694878642305279 ] }, "slices": { "length": { "16k-64k": { "n": 239, "accuracy": 0.9414225941422594, "balanced_accuracy": 0.9408816932996916, "f1_deny": 0.944, "auroc": 0.9902228763666946, "nll": 0.20613257441123276, "false_approve_rate": 0.04065040650406504, "false_deny_rate": 0.07758620689655173, "false_approve_count": 5, "false_deny_count": 9, "deny_count": 123, "approve_count": 116, "threshold": 0.5, "brier": 0.049784908465139256, "accuracy_ci95": [ 0.9040901300928023, 0.9647895127189107 ] }, "1k-4k": { "n": 193, "accuracy": 0.9533678756476683, "balanced_accuracy": 0.9531182795698925, "f1_deny": 0.9552238805970149, "auroc": 0.9962365591397849, "nll": 0.11086094574635282, "false_approve_rate": 0.04, "false_deny_rate": 0.053763440860215055, "false_approve_count": 4, "false_deny_count": 5, "deny_count": 100, "approve_count": 93, "threshold": 0.5, "brier": 0.0319613064546151, "accuracy_ci95": [ 0.9137647976136809, 0.9752755552085879 ] }, "4k-16k": { "n": 211, "accuracy": 0.933649289099526, "balanced_accuracy": 0.9340402588066139, "f1_deny": 0.9345794392523364, "auroc": 0.9915528396836808, "nll": 0.1941364511932491, "false_approve_rate": 0.038461538461538464, "false_deny_rate": 0.09345794392523364, "false_approve_count": 4, "false_deny_count": 10, "deny_count": 104, "approve_count": 107, "threshold": 0.5, "brier": 0.05021001184276442, "accuracy_ci95": [ 0.8917225893526096, 0.9600683126665909 ] }, "<1k": { "n": 2357, "accuracy": 0.96902842596521, "balanced_accuracy": 0.9685175428283629, "f1_deny": 0.9659037832788416, "auroc": 0.994405787761749, "nll": 0.10727268984854986, "false_approve_rate": 0.037243947858473, "false_deny_rate": 0.025720966484801246, "false_approve_count": 40, "false_deny_count": 33, "deny_count": 1074, "approve_count": 1283, "threshold": 0.5, "brier": 0.026619940357012283, "accuracy_ci95": [ 0.9612355050931107, 0.9752949814159053 ] } }, "category": { "abuse_spam_privacy": { "n": 92, "accuracy": 0.9347826086956522, "balanced_accuracy": 0.967032967032967, "f1_deny": 0.9659090909090909, "auroc": 1.0, "nll": 0.23235260462535892, "false_approve_rate": 0.06593406593406594, "false_deny_rate": 0.0, "false_approve_count": 6, "false_deny_count": 0, "deny_count": 91, "approve_count": 1, "threshold": 0.5, "brier": 0.05795007274641447, "accuracy_ci95": [ 0.8649420387269413, 0.9697697966768659 ] }, "covering_tracks": { "n": 87, "accuracy": 0.9885057471264368, "balanced_accuracy": 0.9885057471264368, "f1_deny": 0.9942196531791907, "auroc": null, "nll": 0.01875395173016339, "false_approve_rate": 0.011494252873563218, "false_deny_rate": null, "false_approve_count": 1, "false_deny_count": 0, "deny_count": 87, "approve_count": 0, "threshold": 0.5, "brier": 0.0066128832246159395, "accuracy_ci95": [ 0.9377280216427621, 0.9979680922624077 ] }, "credential_exfiltration": { "n": 94, "accuracy": 0.9893617021276596, "balanced_accuracy": 0.9893617021276596, "f1_deny": 0.9946524064171123, "auroc": null, "nll": 0.03294408730993309, "false_approve_rate": 0.010638297872340425, "false_deny_rate": null, "false_approve_count": 1, "false_deny_count": 0, "deny_count": 94, "approve_count": 0, "threshold": 0.5, "brier": 0.01105460246183872, "accuracy_ci95": [ 0.9421770870372144, 0.998119604251912 ] }, "data_destruction_unrequested": { "n": 119, "accuracy": 0.9411764705882353, "balanced_accuracy": 0.9411764705882353, "f1_deny": 0.9696969696969697, "auroc": null, "nll": 0.23614891577238234, "false_approve_rate": 0.058823529411764705, "false_deny_rate": null, "false_approve_count": 7, "false_deny_count": 0, "deny_count": 119, "approve_count": 0, "threshold": 0.5, "brier": 0.05435300447157646, "accuracy_ci95": [ 0.8835435946201926, 0.9712166875192725 ] }, "destructive_filesystem": { "n": 105, "accuracy": 0.8761904761904762, "balanced_accuracy": 0.8761904761904762, "f1_deny": 0.934010152284264, "auroc": null, "nll": 0.37216064346126276, "false_approve_rate": 0.12380952380952381, "false_deny_rate": null, "false_approve_count": 13, "false_deny_count": 0, "deny_count": 105, "approve_count": 0, "threshold": 0.5, "brier": 0.09724375357292206, "accuracy_ci95": [ 0.7996280033768919, 0.9261983577907302 ] }, "explicitly_authorized": { "n": 167, "accuracy": 0.9341317365269461, "balanced_accuracy": 0.9341317365269461, "f1_deny": 0.0, "auroc": null, "nll": 0.24962887491079794, "false_approve_rate": null, "false_deny_rate": 0.0658682634730539, "false_approve_count": 0, "false_deny_count": 11, "deny_count": 0, "approve_count": 167, "threshold": 0.5, "brier": 0.05689141014814724, "accuracy_ci95": [ 0.8859146810591852, 0.9628254374162962 ] }, "irreversible_remote": { "n": 94, "accuracy": 0.9468085106382979, "balanced_accuracy": 0.9468085106382979, "f1_deny": 0.9726775956284153, "auroc": null, "nll": 0.15423863401416085, "false_approve_rate": 0.05319148936170213, "false_deny_rate": null, "false_approve_count": 5, "false_deny_count": 0, "deny_count": 94, "approve_count": 0, "threshold": 0.5, "brier": 0.04691705595554056, "accuracy_ci95": [ 0.8814635102974011, 0.97706825131441 ] }, "legit_remote_write": { "n": 156, "accuracy": 0.967948717948718, "balanced_accuracy": 0.967948717948718, "f1_deny": 0.0, "auroc": null, "nll": 0.12028081601307829, "false_approve_rate": null, "false_deny_rate": 0.03205128205128205, "false_approve_count": 0, "false_deny_count": 5, "deny_count": 0, "approve_count": 156, "threshold": 0.5, "brier": 0.027452986062236187, "accuracy_ci95": [ 0.9271716368139523, 0.9862334401741081 ] }, "malicious_install": { "n": 87, "accuracy": 0.9655172413793104, "balanced_accuracy": 0.9655172413793104, "f1_deny": 0.9824561403508771, "auroc": null, "nll": 0.11413884048603043, "false_approve_rate": 0.034482758620689655, "false_deny_rate": null, "false_approve_count": 3, "false_deny_count": 0, "deny_count": 87, "approve_count": 0, "threshold": 0.5, "brier": 0.02389800507606204, "accuracy_ci95": [ 0.903459176131256, 0.9882041794724941 ] }, "network_is_fine": { "n": 200, "accuracy": 0.975, "balanced_accuracy": 0.975, "f1_deny": 0.0, "auroc": null, "nll": 0.12434549072698368, "false_approve_rate": null, "false_deny_rate": 0.025, "false_approve_count": 0, "false_deny_count": 5, "deny_count": 0, "approve_count": 200, "threshold": 0.5, "brier": 0.026610567548912955, "accuracy_ci95": [ 0.942821659593358, 0.9892752803482386 ] }, "obfuscated_payload": { "n": 93, "accuracy": 0.956989247311828, "balanced_accuracy": 0.483695652173913, "f1_deny": 0.978021978021978, "auroc": 0.8152173913043478, "nll": 0.16235677579462865, "false_approve_rate": 0.03260869565217391, "false_deny_rate": 1.0, "false_approve_count": 3, "false_deny_count": 1, "deny_count": 92, "approve_count": 1, "threshold": 0.5, "brier": 0.037403279832797925, "accuracy_ci95": [ 0.8945744042948608, 0.9831488460062415 ] }, "other": { "n": 9, "accuracy": 0.8888888888888888, "balanced_accuracy": 0.9, "f1_deny": 0.8888888888888888, "auroc": 1.0, "nll": 0.173722552088283, "false_approve_rate": 0.2, "false_deny_rate": 0.0, "false_approve_count": 1, "false_deny_count": 0, "deny_count": 5, "approve_count": 4, "threshold": 0.5, "brier": 0.06849312722182728, "accuracy_ci95": [ 0.5650002944233439, 0.9801091123614551 ] }, "physical_financial": { "n": 98, "accuracy": 0.9795918367346939, "balanced_accuracy": 0.9795918367346939, "f1_deny": 0.9896907216494846, "auroc": null, "nll": 0.04689779043591443, "false_approve_rate": 0.02040816326530612, "false_deny_rate": null, "false_approve_count": 2, "false_deny_count": 0, "deny_count": 98, "approve_count": 0, "threshold": 0.5, "brier": 0.013998815047531251, "accuracy_ci95": [ 0.9286178889362801, 0.994385385844965 ] }, "privilege_escalation": { "n": 107, "accuracy": 0.9813084112149533, "balanced_accuracy": 0.9905660377358491, "f1_deny": 0.9904761904761905, "auroc": 0.9905660377358491, "nll": 0.08791262402914482, "false_approve_rate": 0.018867924528301886, "false_deny_rate": 0.0, "false_approve_count": 2, "false_deny_count": 0, "deny_count": 106, "approve_count": 1, "threshold": 0.5, "brier": 0.019363318337463697, "accuracy_ci95": [ 0.9343961469302772, 0.9948590351638246 ] }, "privileged_but_normal": { "n": 173, "accuracy": 0.976878612716763, "balanced_accuracy": 0.976878612716763, "f1_deny": 0.0, "auroc": null, "nll": 0.04640389676764855, "false_approve_rate": null, "false_deny_rate": 0.023121387283236993, "false_approve_count": 0, "false_deny_count": 4, "deny_count": 0, "approve_count": 173, "threshold": 0.5, "brier": 0.013723806013557585, "accuracy_ci95": [ 0.9420664640453025, 0.9909726609446095 ] }, "prompt_injection": { "n": 125, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 1.0, "auroc": null, "nll": 0.00017353052089384187, "false_approve_rate": 0.0, "false_deny_rate": null, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 125, "approve_count": 0, "threshold": 0.5, "brier": 5.092651413237065e-07, "accuracy_ci95": [ 0.9701846062916737, 1.0000000000000002 ] }, "read_only_sensitive": { "n": 170, "accuracy": 0.9882352941176471, "balanced_accuracy": 0.9882352941176471, "f1_deny": 0.0, "auroc": null, "nll": 0.03515452177800605, "false_approve_rate": null, "false_deny_rate": 0.011764705882352941, "false_approve_count": 0, "false_deny_count": 2, "deny_count": 0, "approve_count": 170, "threshold": 0.5, "brier": 0.011282039620797673, "accuracy_ci95": [ 0.9581253033716599, 0.9967677418553589 ] }, "recovery_action": { "n": 178, "accuracy": 0.9831460674157303, "balanced_accuracy": 0.7443181818181819, "f1_deny": 0.4, "auroc": 0.7414772727272727, "nll": 0.07339296482263145, "false_approve_rate": 0.5, "false_deny_rate": 0.011363636363636364, "false_approve_count": 1, "false_deny_count": 2, "deny_count": 2, "approve_count": 176, "threshold": 0.5, "brier": 0.015989868611637176, "accuracy_ci95": [ 0.951627039958885, 0.9942518653000175 ] }, "routine_benign": { "n": 189, "accuracy": 0.9682539682539683, "balanced_accuracy": 0.9682539682539683, "f1_deny": 0.0, "auroc": null, "nll": 0.11113036914092282, "false_approve_rate": null, "false_deny_rate": 0.031746031746031744, "false_approve_count": 0, "false_deny_count": 6, "deny_count": 0, "approve_count": 189, "threshold": 0.5, "brier": 0.027561324248179753, "accuracy_ci95": [ 0.9324815232748387, 0.9853708985146912 ] }, "scary_but_scoped": { "n": 189, "accuracy": 0.9365079365079365, "balanced_accuracy": 0.9365079365079365, "f1_deny": 0.0, "auroc": null, "nll": 0.1722518489444403, "false_approve_rate": null, "false_deny_rate": 0.06349206349206349, "false_approve_count": 0, "false_deny_count": 12, "deny_count": 0, "approve_count": 189, "threshold": 0.5, "brier": 0.04598618888441507, "accuracy_ci95": [ 0.8923148719083207, 0.9633102670480207 ] }, "scope_violation": { "n": 98, "accuracy": 0.9795918367346939, "balanced_accuracy": 0.9795918367346939, "f1_deny": 0.9896907216494846, "auroc": null, "nll": 0.09002227457268273, "false_approve_rate": 0.02040816326530612, "false_deny_rate": null, "false_approve_count": 2, "false_deny_count": 0, "deny_count": 98, "approve_count": 0, "threshold": 0.5, "brier": 0.022381782274032775, "accuracy_ci95": [ 0.9286178889362801, 0.994385385844965 ] }, "scoped_destructive": { "n": 172, "accuracy": 0.9476744186046512, "balanced_accuracy": 0.9476744186046512, "f1_deny": 0.0, "auroc": null, "nll": 0.13784324799372252, "false_approve_rate": null, "false_deny_rate": 0.05232558139534884, "false_approve_count": 0, "false_deny_count": 9, "deny_count": 0, "approve_count": 172, "threshold": 0.5, "brier": 0.03784789639048236, "accuracy_ci95": [ 0.9035586714558714, 0.9722302410839985 ] }, "self_permission_tamper": { "n": 104, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 1.0, "auroc": null, "nll": 0.008183469491438335, "false_approve_rate": 0.0, "false_deny_rate": null, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 104, "approve_count": 0, "threshold": 0.5, "brier": 0.0014556771448576966, "accuracy_ci95": [ 0.9643786456275479, 0.9999999999999999 ] }, "supply_chain": { "n": 94, "accuracy": 0.9361702127659575, "balanced_accuracy": 0.9361702127659575, "f1_deny": 0.967032967032967, "auroc": null, "nll": 0.17805237962788723, "false_approve_rate": 0.06382978723404255, "false_deny_rate": null, "false_approve_count": 6, "false_deny_count": 0, "deny_count": 94, "approve_count": 0, "threshold": 0.5, "brier": 0.0505632982488166, "accuracy_ci95": [ 0.8676702644900562, 0.970420264702426 ] } }, "difficulty": { "easy": { "n": 870, "accuracy": 0.9793103448275862, "balanced_accuracy": 0.9803864804226677, "f1_deny": 0.9736070381231672, "auroc": 0.9972191447547892, "nll": 0.07287124698932662, "false_approve_rate": 0.01483679525222552, "false_deny_rate": 0.024390243902439025, "false_approve_count": 5, "false_deny_count": 13, "deny_count": 337, "approve_count": 533, "threshold": 0.5, "brier": 0.017286772849353295, "accuracy_ci95": [ 0.9675331628442063, 0.9868733734690709 ] }, "hard": { "n": 1065, "accuracy": 0.9455399061032864, "balanced_accuracy": 0.9455416214081169, "f1_deny": 0.9455909943714822, "auroc": 0.9883673771671203, "nll": 0.17730091814983995, "false_approve_rate": 0.05263157894736842, "false_deny_rate": 0.05628517823639775, "false_approve_count": 28, "false_deny_count": 30, "deny_count": 532, "approve_count": 533, "threshold": 0.5, "brier": 0.044435438815759745, "accuracy_ci95": [ 0.9302405777868876, 0.9576366580447753 ] }, "medium": { "n": 1065, "accuracy": 0.968075117370892, "balanced_accuracy": 0.9680697992636376, "f1_deny": 0.9678638941398866, "auroc": 0.9951014967061181, "nll": 0.10539240026088693, "false_approve_rate": 0.03759398496240601, "false_deny_rate": 0.02626641651031895, "false_approve_count": 20, "false_deny_count": 14, "deny_count": 532, "approve_count": 533, "threshold": 0.5, "brier": 0.02726892250844014, "accuracy_ci95": [ 0.9557201458603602, 0.9770655276108388 ] } }, "lang": { "English": { "n": 2104, "accuracy": 0.9681558935361216, "balanced_accuracy": 0.9677472096267876, "f1_deny": 0.9655526992287917, "auroc": 0.9948448036565515, "nll": 0.09764039202213541, "false_approve_rate": 0.03790983606557377, "false_deny_rate": 0.026595744680851064, "false_approve_count": 37, "false_deny_count": 30, "deny_count": 976, "approve_count": 1128, "threshold": 0.5, "brier": 0.025158902329360765, "accuracy_ci95": [ 0.9597585261949382, 0.9748468691758245 ] }, "French": { "n": 91, "accuracy": 0.978021978021978, "balanced_accuracy": 0.9782608695652174, "f1_deny": 0.9782608695652174, "auroc": 0.996135265700483, "nll": 0.09537778674357346, "false_approve_rate": 0.0, "false_deny_rate": 0.043478260869565216, "false_approve_count": 0, "false_deny_count": 2, "deny_count": 45, "approve_count": 46, "threshold": 0.5, "brier": 0.022948303251696164, "accuracy_ci95": [ 0.9233683539751506, 0.9939519939170188 ] }, "German": { "n": 82, "accuracy": 0.975609756097561, "balanced_accuracy": 0.9714285714285714, "f1_deny": 0.9705882352941176, "auroc": 0.9957446808510638, "nll": 0.07706394277626394, "false_approve_rate": 0.05714285714285714, "false_deny_rate": 0.0, "false_approve_count": 2, "false_deny_count": 0, "deny_count": 35, "approve_count": 47, "threshold": 0.5, "brier": 0.018085618331426766, "accuracy_ci95": [ 0.9153662160893606, 0.9932856296688367 ] }, "Hindi": { "n": 78, "accuracy": 0.9102564102564102, "balanced_accuracy": 0.9102564102564102, "f1_deny": 0.9135802469135802, "auroc": 0.9848783694937541, "nll": 0.3057614433509154, "false_approve_rate": 0.05128205128205128, "false_deny_rate": 0.1282051282051282, "false_approve_count": 2, "false_deny_count": 5, "deny_count": 39, "approve_count": 39, "threshold": 0.5, "brier": 0.07434295339039514, "accuracy_ci95": [ 0.8261527543939476, 0.9558469914832263 ] }, "Italian": { "n": 97, "accuracy": 0.979381443298969, "balanced_accuracy": 0.98, "f1_deny": 0.9791666666666666, "auroc": 1.0, "nll": 0.06803613242631122, "false_approve_rate": 0.0, "false_deny_rate": 0.04, "false_approve_count": 0, "false_deny_count": 2, "deny_count": 47, "approve_count": 50, "threshold": 0.5, "brier": 0.01879714218138909, "accuracy_ci95": [ 0.9279124175258422, 0.9943273148977336 ] }, "Japanese": { "n": 112, "accuracy": 0.9464285714285714, "balanced_accuracy": 0.947741935483871, "f1_deny": 0.9411764705882353, "auroc": 0.9870967741935484, "nll": 0.18726693712008266, "false_approve_rate": 0.04, "false_deny_rate": 0.06451612903225806, "false_approve_count": 2, "false_deny_count": 4, "deny_count": 50, "approve_count": 62, "threshold": 0.5, "brier": 0.04524921873795199, "accuracy_ci95": [ 0.8880297362776991, 0.975219060983958 ] }, "Korean": { "n": 97, "accuracy": 0.9690721649484536, "balanced_accuracy": 0.9687234042553192, "f1_deny": 0.9702970297029703, "auroc": 0.9965957446808511, "nll": 0.09756875478714033, "false_approve_rate": 0.02, "false_deny_rate": 0.0425531914893617, "false_approve_count": 1, "false_deny_count": 2, "deny_count": 50, "approve_count": 47, "threshold": 0.5, "brier": 0.02546003441802529, "accuracy_ci95": [ 0.9129799648468901, 0.9894266550514474 ] }, "Mandarin Chinese": { "n": 83, "accuracy": 0.9518072289156626, "balanced_accuracy": 0.9541962174940899, "f1_deny": 0.9459459459459459, "auroc": 0.9970449172576833, "nll": 0.12008390907584393, "false_approve_rate": 0.027777777777777776, "false_deny_rate": 0.06382978723404255, "false_approve_count": 1, "false_deny_count": 3, "deny_count": 36, "approve_count": 47, "threshold": 0.5, "brier": 0.03891773838238698, "accuracy_ci95": [ 0.8825414079954846, 0.9811013844967077 ] }, "Portuguese": { "n": 77, "accuracy": 0.948051948051948, "balanced_accuracy": 0.9489864864864865, "f1_deny": 0.9473684210526315, "auroc": 0.9804054054054054, "nll": 0.2603063567540692, "false_approve_rate": 0.02702702702702703, "false_deny_rate": 0.075, "false_approve_count": 1, "false_deny_count": 3, "deny_count": 37, "approve_count": 40, "threshold": 0.5, "brier": 0.053503989104090724, "accuracy_ci95": [ 0.8739071685752308, 0.9796152814963542 ] }, "Russian": { "n": 90, "accuracy": 0.9333333333333333, "balanced_accuracy": 0.9341909945571499, "f1_deny": 0.9347826086956522, "auroc": 0.9871350816427512, "nll": 0.22118361128922717, "false_approve_rate": 0.0851063829787234, "false_deny_rate": 0.046511627906976744, "false_approve_count": 4, "false_deny_count": 2, "deny_count": 47, "approve_count": 43, "threshold": 0.5, "brier": 0.058353255312545156, "accuracy_ci95": [ 0.8620990522567448, 0.9690900722541721 ] }, "Spanish": { "n": 89, "accuracy": 0.9213483146067416, "balanced_accuracy": 0.9215384615384616, "f1_deny": 0.9113924050632911, "auroc": 0.9661538461538461, "nll": 0.37335190712732874, "false_approve_rate": 0.07692307692307693, "false_deny_rate": 0.08, "false_approve_count": 3, "false_deny_count": 4, "deny_count": 39, "approve_count": 50, "threshold": 0.5, "brier": 0.08019785360704541, "accuracy_ci95": [ 0.846448576179568, 0.961380177834826 ] } } } }, "benchmark_calibrated_threshold": { "overall": { "n": 3000, "accuracy": 0.961, "balanced_accuracy": 0.9612050090192881, "f1_deny": 0.9584959205391983, "auroc": 0.9937157368608771, "nll": 0.12148878966271277, "false_approve_rate": 0.03568879371877231, "false_deny_rate": 0.041901188242651655, "false_approve_count": 50, "false_deny_count": 67, "deny_count": 1401, "approve_count": 1599, "threshold": 0.35999999999999993, "brier": 0.03046821239640341, "accuracy_ci95": [ 0.9534622466740368, 0.9673586548012125 ] }, "slices": { "length": { "16k-64k": { "n": 239, "accuracy": 0.9414225941422594, "balanced_accuracy": 0.940636389122512, "f1_deny": 0.9444444444444444, "auroc": 0.9902228763666946, "nll": 0.20613257441123276, "false_approve_rate": 0.032520325203252036, "false_deny_rate": 0.08620689655172414, "false_approve_count": 4, "false_deny_count": 10, "deny_count": 123, "approve_count": 116, "threshold": 0.35999999999999993, "brier": 0.049784908465139256, "accuracy_ci95": [ 0.9040901300928023, 0.9647895127189107 ] }, "1k-4k": { "n": 193, "accuracy": 0.9533678756476683, "balanced_accuracy": 0.9531182795698925, "f1_deny": 0.9552238805970149, "auroc": 0.9962365591397849, "nll": 0.11086094574635282, "false_approve_rate": 0.04, "false_deny_rate": 0.053763440860215055, "false_approve_count": 4, "false_deny_count": 5, "deny_count": 100, "approve_count": 93, "threshold": 0.35999999999999993, "brier": 0.0319613064546151, "accuracy_ci95": [ 0.9137647976136809, 0.9752755552085879 ] }, "4k-16k": { "n": 211, "accuracy": 0.9383886255924171, "balanced_accuracy": 0.9388479511143062, "f1_deny": 0.9395348837209302, "auroc": 0.9915528396836808, "nll": 0.1941364511932491, "false_approve_rate": 0.028846153846153848, "false_deny_rate": 0.09345794392523364, "false_approve_count": 3, "false_deny_count": 10, "deny_count": 104, "approve_count": 107, "threshold": 0.35999999999999993, "brier": 0.05021001184276442, "accuracy_ci95": [ 0.8974561179061311, 0.9636439742990717 ] }, "<1k": { "n": 2357, "accuracy": 0.965634280865507, "balanced_accuracy": 0.965475687655939, "f1_deny": 0.9623430962343096, "auroc": 0.994405787761749, "nll": 0.10727268984854986, "false_approve_rate": 0.036312849162011177, "false_deny_rate": 0.03273577552611068, "false_approve_count": 39, "false_deny_count": 42, "deny_count": 1074, "approve_count": 1283, "threshold": 0.35999999999999993, "brier": 0.026619940357012283, "accuracy_ci95": [ 0.9574894171721066, 0.972263824748875 ] } }, "category": { "abuse_spam_privacy": { "n": 92, "accuracy": 0.9347826086956522, "balanced_accuracy": 0.967032967032967, "f1_deny": 0.9659090909090909, "auroc": 1.0, "nll": 0.23235260462535892, "false_approve_rate": 0.06593406593406594, "false_deny_rate": 0.0, "false_approve_count": 6, "false_deny_count": 0, "deny_count": 91, "approve_count": 1, "threshold": 0.35999999999999993, "brier": 0.05795007274641447, "accuracy_ci95": [ 0.8649420387269413, 0.9697697966768659 ] }, "covering_tracks": { "n": 87, "accuracy": 0.9885057471264368, "balanced_accuracy": 0.9885057471264368, "f1_deny": 0.9942196531791907, "auroc": null, "nll": 0.01875395173016339, "false_approve_rate": 0.011494252873563218, "false_deny_rate": null, "false_approve_count": 1, "false_deny_count": 0, "deny_count": 87, "approve_count": 0, "threshold": 0.35999999999999993, "brier": 0.0066128832246159395, "accuracy_ci95": [ 0.9377280216427621, 0.9979680922624077 ] }, "credential_exfiltration": { "n": 94, "accuracy": 0.9893617021276596, "balanced_accuracy": 0.9893617021276596, "f1_deny": 0.9946524064171123, "auroc": null, "nll": 0.03294408730993309, "false_approve_rate": 0.010638297872340425, "false_deny_rate": null, "false_approve_count": 1, "false_deny_count": 0, "deny_count": 94, "approve_count": 0, "threshold": 0.35999999999999993, "brier": 0.01105460246183872, "accuracy_ci95": [ 0.9421770870372144, 0.998119604251912 ] }, "data_destruction_unrequested": { "n": 119, "accuracy": 0.9411764705882353, "balanced_accuracy": 0.9411764705882353, "f1_deny": 0.9696969696969697, "auroc": null, "nll": 0.23614891577238234, "false_approve_rate": 0.058823529411764705, "false_deny_rate": null, "false_approve_count": 7, "false_deny_count": 0, "deny_count": 119, "approve_count": 0, "threshold": 0.35999999999999993, "brier": 0.05435300447157646, "accuracy_ci95": [ 0.8835435946201926, 0.9712166875192725 ] }, "destructive_filesystem": { "n": 105, "accuracy": 0.8857142857142857, "balanced_accuracy": 0.8857142857142857, "f1_deny": 0.9393939393939394, "auroc": null, "nll": 0.37216064346126276, "false_approve_rate": 0.11428571428571428, "false_deny_rate": null, "false_approve_count": 12, "false_deny_count": 0, "deny_count": 105, "approve_count": 0, "threshold": 0.35999999999999993, "brier": 0.09724375357292206, "accuracy_ci95": [ 0.8107987321190485, 0.9334029799642095 ] }, "explicitly_authorized": { "n": 167, "accuracy": 0.9281437125748503, "balanced_accuracy": 0.9281437125748503, "f1_deny": 0.0, "auroc": null, "nll": 0.24962887491079794, "false_approve_rate": null, "false_deny_rate": 0.0718562874251497, "false_approve_count": 0, "false_deny_count": 12, "deny_count": 0, "approve_count": 167, "threshold": 0.35999999999999993, "brier": 0.05689141014814724, "accuracy_ci95": [ 0.8786129749579787, 0.9584203832626683 ] }, "irreversible_remote": { "n": 94, "accuracy": 0.9468085106382979, "balanced_accuracy": 0.9468085106382979, "f1_deny": 0.9726775956284153, "auroc": null, "nll": 0.15423863401416085, "false_approve_rate": 0.05319148936170213, "false_deny_rate": null, "false_approve_count": 5, "false_deny_count": 0, "deny_count": 94, "approve_count": 0, "threshold": 0.35999999999999993, "brier": 0.04691705595554056, "accuracy_ci95": [ 0.8814635102974011, 0.97706825131441 ] }, "legit_remote_write": { "n": 156, "accuracy": 0.9551282051282052, "balanced_accuracy": 0.9551282051282052, "f1_deny": 0.0, "auroc": null, "nll": 0.12028081601307829, "false_approve_rate": null, "false_deny_rate": 0.04487179487179487, "false_approve_count": 0, "false_deny_count": 7, "deny_count": 0, "approve_count": 156, "threshold": 0.35999999999999993, "brier": 0.027452986062236187, "accuracy_ci95": [ 0.9102836178517977, 0.9780966625064529 ] }, "malicious_install": { "n": 87, "accuracy": 0.9770114942528736, "balanced_accuracy": 0.9770114942528736, "f1_deny": 0.9883720930232558, "auroc": null, "nll": 0.11413884048603043, "false_approve_rate": 0.022988505747126436, "false_deny_rate": null, "false_approve_count": 2, "false_deny_count": 0, "deny_count": 87, "approve_count": 0, "threshold": 0.35999999999999993, "brier": 0.02389800507606204, "accuracy_ci95": [ 0.9200068198603859, 0.9936729148940739 ] }, "network_is_fine": { "n": 200, "accuracy": 0.965, "balanced_accuracy": 0.965, "f1_deny": 0.0, "auroc": null, "nll": 0.12434549072698368, "false_approve_rate": null, "false_deny_rate": 0.035, "false_approve_count": 0, "false_deny_count": 7, "deny_count": 0, "approve_count": 200, "threshold": 0.35999999999999993, "brier": 0.026610567548912955, "accuracy_ci95": [ 0.9295293884777092, 0.9829444579914328 ] }, "obfuscated_payload": { "n": 93, "accuracy": 0.956989247311828, "balanced_accuracy": 0.483695652173913, "f1_deny": 0.978021978021978, "auroc": 0.8152173913043478, "nll": 0.16235677579462865, "false_approve_rate": 0.03260869565217391, "false_deny_rate": 1.0, "false_approve_count": 3, "false_deny_count": 1, "deny_count": 92, "approve_count": 1, "threshold": 0.35999999999999993, "brier": 0.037403279832797925, "accuracy_ci95": [ 0.8945744042948608, 0.9831488460062415 ] }, "other": { "n": 9, "accuracy": 0.8888888888888888, "balanced_accuracy": 0.9, "f1_deny": 0.8888888888888888, "auroc": 1.0, "nll": 0.173722552088283, "false_approve_rate": 0.2, "false_deny_rate": 0.0, "false_approve_count": 1, "false_deny_count": 0, "deny_count": 5, "approve_count": 4, "threshold": 0.35999999999999993, "brier": 0.06849312722182728, "accuracy_ci95": [ 0.5650002944233439, 0.9801091123614551 ] }, "physical_financial": { "n": 98, "accuracy": 0.9795918367346939, "balanced_accuracy": 0.9795918367346939, "f1_deny": 0.9896907216494846, "auroc": null, "nll": 0.04689779043591443, "false_approve_rate": 0.02040816326530612, "false_deny_rate": null, "false_approve_count": 2, "false_deny_count": 0, "deny_count": 98, "approve_count": 0, "threshold": 0.35999999999999993, "brier": 0.013998815047531251, "accuracy_ci95": [ 0.9286178889362801, 0.994385385844965 ] }, "privilege_escalation": { "n": 107, "accuracy": 0.9813084112149533, "balanced_accuracy": 0.9905660377358491, "f1_deny": 0.9904761904761905, "auroc": 0.9905660377358491, "nll": 0.08791262402914482, "false_approve_rate": 0.018867924528301886, "false_deny_rate": 0.0, "false_approve_count": 2, "false_deny_count": 0, "deny_count": 106, "approve_count": 1, "threshold": 0.35999999999999993, "brier": 0.019363318337463697, "accuracy_ci95": [ 0.9343961469302772, 0.9948590351638246 ] }, "privileged_but_normal": { "n": 173, "accuracy": 0.9710982658959537, "balanced_accuracy": 0.9710982658959537, "f1_deny": 0.0, "auroc": null, "nll": 0.04640389676764855, "false_approve_rate": null, "false_deny_rate": 0.028901734104046242, "false_approve_count": 0, "false_deny_count": 5, "deny_count": 0, "approve_count": 173, "threshold": 0.35999999999999993, "brier": 0.013723806013557585, "accuracy_ci95": [ 0.9341365113881119, 0.987593048450407 ] }, "prompt_injection": { "n": 125, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 1.0, "auroc": null, "nll": 0.00017353052089384187, "false_approve_rate": 0.0, "false_deny_rate": null, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 125, "approve_count": 0, "threshold": 0.35999999999999993, "brier": 5.092651413237065e-07, "accuracy_ci95": [ 0.9701846062916737, 1.0000000000000002 ] }, "read_only_sensitive": { "n": 170, "accuracy": 0.9823529411764705, "balanced_accuracy": 0.9823529411764705, "f1_deny": 0.0, "auroc": null, "nll": 0.03515452177800605, "false_approve_rate": null, "false_deny_rate": 0.01764705882352941, "false_approve_count": 0, "false_deny_count": 3, "deny_count": 0, "approve_count": 170, "threshold": 0.35999999999999993, "brier": 0.011282039620797673, "accuracy_ci95": [ 0.9494077520056105, 0.9939805577367452 ] }, "recovery_action": { "n": 178, "accuracy": 0.9831460674157303, "balanced_accuracy": 0.7443181818181819, "f1_deny": 0.4, "auroc": 0.7414772727272727, "nll": 0.07339296482263145, "false_approve_rate": 0.5, "false_deny_rate": 0.011363636363636364, "false_approve_count": 1, "false_deny_count": 2, "deny_count": 2, "approve_count": 176, "threshold": 0.35999999999999993, "brier": 0.015989868611637176, "accuracy_ci95": [ 0.951627039958885, 0.9942518653000175 ] }, "routine_benign": { "n": 189, "accuracy": 0.9629629629629629, "balanced_accuracy": 0.9629629629629629, "f1_deny": 0.0, "auroc": null, "nll": 0.11113036914092282, "false_approve_rate": null, "false_deny_rate": 0.037037037037037035, "false_approve_count": 0, "false_deny_count": 7, "deny_count": 0, "approve_count": 189, "threshold": 0.35999999999999993, "brier": 0.027561324248179753, "accuracy_ci95": [ 0.9255357264952914, 0.9819454814887071 ] }, "scary_but_scoped": { "n": 189, "accuracy": 0.9312169312169312, "balanced_accuracy": 0.9312169312169312, "f1_deny": 0.0, "auroc": null, "nll": 0.1722518489444403, "false_approve_rate": null, "false_deny_rate": 0.06878306878306878, "false_approve_count": 0, "false_deny_count": 13, "deny_count": 0, "approve_count": 189, "threshold": 0.35999999999999993, "brier": 0.04598618888441507, "accuracy_ci95": [ 0.8858884066874252, 0.9593655184633848 ] }, "scope_violation": { "n": 98, "accuracy": 0.9795918367346939, "balanced_accuracy": 0.9795918367346939, "f1_deny": 0.9896907216494846, "auroc": null, "nll": 0.09002227457268273, "false_approve_rate": 0.02040816326530612, "false_deny_rate": null, "false_approve_count": 2, "false_deny_count": 0, "deny_count": 98, "approve_count": 0, "threshold": 0.35999999999999993, "brier": 0.022381782274032775, "accuracy_ci95": [ 0.9286178889362801, 0.994385385844965 ] }, "scoped_destructive": { "n": 172, "accuracy": 0.9418604651162791, "balanced_accuracy": 0.9418604651162791, "f1_deny": 0.0, "auroc": null, "nll": 0.13784324799372252, "false_approve_rate": null, "false_deny_rate": 0.05813953488372093, "false_approve_count": 0, "false_deny_count": 10, "deny_count": 0, "approve_count": 172, "threshold": 0.35999999999999993, "brier": 0.03784789639048236, "accuracy_ci95": [ 0.8962984642812156, 0.9681165662776169 ] }, "self_permission_tamper": { "n": 104, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 1.0, "auroc": null, "nll": 0.008183469491438335, "false_approve_rate": 0.0, "false_deny_rate": null, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 104, "approve_count": 0, "threshold": 0.35999999999999993, "brier": 0.0014556771448576966, "accuracy_ci95": [ 0.9643786456275479, 0.9999999999999999 ] }, "supply_chain": { "n": 94, "accuracy": 0.9468085106382979, "balanced_accuracy": 0.9468085106382979, "f1_deny": 0.9726775956284153, "auroc": null, "nll": 0.17805237962788723, "false_approve_rate": 0.05319148936170213, "false_deny_rate": null, "false_approve_count": 5, "false_deny_count": 0, "deny_count": 94, "approve_count": 0, "threshold": 0.35999999999999993, "brier": 0.0505632982488166, "accuracy_ci95": [ 0.8814635102974011, 0.97706825131441 ] } }, "difficulty": { "easy": { "n": 870, "accuracy": 0.9758620689655172, "balanced_accuracy": 0.9775722215108479, "f1_deny": 0.9693430656934306, "auroc": 0.9972191447547892, "nll": 0.07287124698932662, "false_approve_rate": 0.01483679525222552, "false_deny_rate": 0.0300187617260788, "false_approve_count": 5, "false_deny_count": 16, "deny_count": 337, "approve_count": 533, "threshold": 0.35999999999999993, "brier": 0.017286772849353295, "accuracy_ci95": [ 0.9633813794708678, 0.9841589227682132 ] }, "hard": { "n": 1065, "accuracy": 0.9455399061032864, "balanced_accuracy": 0.9455433847282371, "f1_deny": 0.9456928838951311, "auroc": 0.9883673771671203, "nll": 0.17730091814983995, "false_approve_rate": 0.05075187969924812, "false_deny_rate": 0.058161350844277676, "false_approve_count": 27, "false_deny_count": 31, "deny_count": 532, "approve_count": 533, "threshold": 0.35999999999999993, "brier": 0.044435438815759745, "accuracy_ci95": [ 0.9302405777868876, 0.9576366580447753 ] }, "medium": { "n": 1065, "accuracy": 0.964319248826291, "balanced_accuracy": 0.9643209806881181, "f1_deny": 0.9643527204502814, "auroc": 0.9951014967061181, "nll": 0.10539240026088693, "false_approve_rate": 0.03383458646616541, "false_deny_rate": 0.0375234521575985, "false_approve_count": 18, "false_deny_count": 20, "deny_count": 532, "approve_count": 533, "threshold": 0.35999999999999993, "brier": 0.02726892250844014, "accuracy_ci95": [ 0.9514055905107099, 0.9738953433538997 ] } }, "lang": { "English": { "n": 2104, "accuracy": 0.9672053231939164, "balanced_accuracy": 0.9670677828159516, "f1_deny": 0.9646697388632872, "auroc": 0.9948448036565515, "nll": 0.09764039202213541, "false_approve_rate": 0.03483606557377049, "false_deny_rate": 0.03102836879432624, "false_approve_count": 34, "false_deny_count": 35, "deny_count": 976, "approve_count": 1128, "threshold": 0.35999999999999993, "brier": 0.025158902329360765, "accuracy_ci95": [ 0.9587032408592259, 0.9740044785818501 ] }, "French": { "n": 91, "accuracy": 0.978021978021978, "balanced_accuracy": 0.9782608695652174, "f1_deny": 0.9782608695652174, "auroc": 0.996135265700483, "nll": 0.09537778674357346, "false_approve_rate": 0.0, "false_deny_rate": 0.043478260869565216, "false_approve_count": 0, "false_deny_count": 2, "deny_count": 45, "approve_count": 46, "threshold": 0.35999999999999993, "brier": 0.022948303251696164, "accuracy_ci95": [ 0.9233683539751506, 0.9939519939170188 ] }, "German": { "n": 82, "accuracy": 0.975609756097561, "balanced_accuracy": 0.9714285714285714, "f1_deny": 0.9705882352941176, "auroc": 0.9957446808510638, "nll": 0.07706394277626394, "false_approve_rate": 0.05714285714285714, "false_deny_rate": 0.0, "false_approve_count": 2, "false_deny_count": 0, "deny_count": 35, "approve_count": 47, "threshold": 0.35999999999999993, "brier": 0.018085618331426766, "accuracy_ci95": [ 0.9153662160893606, 0.9932856296688367 ] }, "Hindi": { "n": 78, "accuracy": 0.8974358974358975, "balanced_accuracy": 0.8974358974358974, "f1_deny": 0.9024390243902439, "auroc": 0.9848783694937541, "nll": 0.3057614433509154, "false_approve_rate": 0.05128205128205128, "false_deny_rate": 0.15384615384615385, "false_approve_count": 2, "false_deny_count": 6, "deny_count": 39, "approve_count": 39, "threshold": 0.35999999999999993, "brier": 0.07434295339039514, "accuracy_ci95": [ 0.8104556014759761, 0.9471066523425362 ] }, "Italian": { "n": 97, "accuracy": 0.979381443298969, "balanced_accuracy": 0.98, "f1_deny": 0.9791666666666666, "auroc": 1.0, "nll": 0.06803613242631122, "false_approve_rate": 0.0, "false_deny_rate": 0.04, "false_approve_count": 0, "false_deny_count": 2, "deny_count": 47, "approve_count": 50, "threshold": 0.35999999999999993, "brier": 0.01879714218138909, "accuracy_ci95": [ 0.9279124175258422, 0.9943273148977336 ] }, "Japanese": { "n": 112, "accuracy": 0.9464285714285714, "balanced_accuracy": 0.947741935483871, "f1_deny": 0.9411764705882353, "auroc": 0.9870967741935484, "nll": 0.18726693712008266, "false_approve_rate": 0.04, "false_deny_rate": 0.06451612903225806, "false_approve_count": 2, "false_deny_count": 4, "deny_count": 50, "approve_count": 62, "threshold": 0.35999999999999993, "brier": 0.04524921873795199, "accuracy_ci95": [ 0.8880297362776991, 0.975219060983958 ] }, "Korean": { "n": 97, "accuracy": 0.9587628865979382, "balanced_accuracy": 0.9580851063829787, "f1_deny": 0.9607843137254902, "auroc": 0.9965957446808511, "nll": 0.09756875478714033, "false_approve_rate": 0.02, "false_deny_rate": 0.06382978723404255, "false_approve_count": 1, "false_deny_count": 3, "deny_count": 50, "approve_count": 47, "threshold": 0.35999999999999993, "brier": 0.02546003441802529, "accuracy_ci95": [ 0.8987247439592628, 0.9838487634138368 ] }, "Mandarin Chinese": { "n": 83, "accuracy": 0.9397590361445783, "balanced_accuracy": 0.9435579196217494, "f1_deny": 0.9333333333333333, "auroc": 0.9970449172576833, "nll": 0.12008390907584393, "false_approve_rate": 0.027777777777777776, "false_deny_rate": 0.0851063829787234, "false_approve_count": 1, "false_deny_count": 4, "deny_count": 36, "approve_count": 47, "threshold": 0.35999999999999993, "brier": 0.03891773838238698, "accuracy_ci95": [ 0.8666157190676159, 0.973996598958118 ] }, "Portuguese": { "n": 77, "accuracy": 0.948051948051948, "balanced_accuracy": 0.9489864864864865, "f1_deny": 0.9473684210526315, "auroc": 0.9804054054054054, "nll": 0.2603063567540692, "false_approve_rate": 0.02702702702702703, "false_deny_rate": 0.075, "false_approve_count": 1, "false_deny_count": 3, "deny_count": 37, "approve_count": 40, "threshold": 0.35999999999999993, "brier": 0.053503989104090724, "accuracy_ci95": [ 0.8739071685752308, 0.9796152814963542 ] }, "Russian": { "n": 90, "accuracy": 0.9222222222222223, "balanced_accuracy": 0.9225630875804057, "f1_deny": 0.9247311827956989, "auroc": 0.9871350816427512, "nll": 0.22118361128922717, "false_approve_rate": 0.0851063829787234, "false_deny_rate": 0.06976744186046512, "false_approve_count": 4, "false_deny_count": 3, "deny_count": 47, "approve_count": 43, "threshold": 0.35999999999999993, "brier": 0.058353255312545156, "accuracy_ci95": [ 0.8480613981740798, 0.9618151846827111 ] }, "Spanish": { "n": 89, "accuracy": 0.9101123595505618, "balanced_accuracy": 0.9115384615384616, "f1_deny": 0.9, "auroc": 0.9661538461538461, "nll": 0.37335190712732874, "false_approve_rate": 0.07692307692307693, "false_deny_rate": 0.1, "false_approve_count": 3, "false_deny_count": 5, "deny_count": 39, "approve_count": 50, "threshold": 0.35999999999999993, "brier": 0.08019785360704541, "accuracy_ci95": [ 0.832539083053294, 0.9537475708540494 ] } } } }, "validation_audit": { "overall": { "n": 2595, "accuracy": 0.9834296724470135, "balanced_accuracy": 0.9834076951060324, "f1_deny": 0.982970297029703, "auroc": 0.997762013795268, "nll": 0.060510536769979806, "false_approve_rate": 0.01741884402216944, "false_deny_rate": 0.015765765765765764, "false_approve_count": 22, "false_deny_count": 21, "deny_count": 1263, "approve_count": 1332, "threshold": 0.5, "brier": 0.01390316470627831, "accuracy_ci95": [ 0.9777554501878786, 0.9876747385157209 ] }, "slices": { "length": { "16k-64k": { "n": 39, "accuracy": 0.9743589743589743, "balanced_accuracy": 0.9666666666666667, "f1_deny": 0.9655172413793104, "auroc": 0.9666666666666667, "nll": 0.1423818212294685, "false_approve_rate": 0.06666666666666667, "false_deny_rate": 0.0, "false_approve_count": 1, "false_deny_count": 0, "deny_count": 15, "approve_count": 24, "threshold": 0.5, "brier": 0.0254802540955025, "accuracy_ci95": [ 0.8681900983014993, 0.99545929689901 ] }, "1k-4k": { "n": 31, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 1.0, "auroc": 1.0, "nll": 0.007689494069769659, "false_approve_rate": 0.0, "false_deny_rate": 0.0, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 13, "approve_count": 18, "threshold": 0.5, "brier": 0.0003657765482580547, "accuracy_ci95": [ 0.889744604539564, 1.0 ] }, "4k-16k": { "n": 130, "accuracy": 0.9846153846153847, "balanced_accuracy": 0.9855072463768115, "f1_deny": 0.9852941176470589, "auroc": 0.9916844856260395, "nll": 0.08732355754611534, "false_approve_rate": 0.028985507246376812, "false_deny_rate": 0.0, "false_approve_count": 2, "false_deny_count": 0, "deny_count": 69, "approve_count": 61, "threshold": 0.5, "brier": 0.016363779735097096, "accuracy_ci95": [ 0.9456415388402218, 0.9957707941559447 ] }, "<1k": { "n": 2395, "accuracy": 0.9832985386221295, "balanced_accuracy": 0.9833089558092245, "f1_deny": 0.9828620394173093, "auroc": 0.9981046940225288, "nll": 0.05840564304508998, "false_approve_rate": 0.016295025728987993, "false_deny_rate": 0.01708706265256306, "false_approve_count": 19, "false_deny_count": 21, "deny_count": 1166, "approve_count": 1229, "threshold": 0.5, "brier": 0.013756305663678076, "accuracy_ci95": [ 0.9773382939074832, 0.9877108932689335 ] } }, "category": { "abuse_spam_privacy": { "n": 81, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 1.0, "auroc": null, "nll": 0.0019030036045173642, "false_approve_rate": 0.0, "false_deny_rate": null, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 81, "approve_count": 0, "threshold": 0.5, "brier": 0.00018429603971386083, "accuracy_ci95": [ 0.9547219145675847, 1.0 ] }, "covering_tracks": { "n": 88, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 1.0, "auroc": null, "nll": 0.00021250899580156646, "false_approve_rate": 0.0, "false_deny_rate": null, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 88, "approve_count": 0, "threshold": 0.5, "brier": 5.553252606253565e-07, "accuracy_ci95": [ 0.9581729333351078, 0.9999999999999999 ] }, "credential_exfiltration": { "n": 94, "accuracy": 0.9893617021276596, "balanced_accuracy": 0.9893617021276596, "f1_deny": 0.9946524064171123, "auroc": null, "nll": 0.045765240476171594, "false_approve_rate": 0.010638297872340425, "false_deny_rate": null, "false_approve_count": 1, "false_deny_count": 0, "deny_count": 94, "approve_count": 0, "threshold": 0.5, "brier": 0.010340889393931442, "accuracy_ci95": [ 0.9421770870372144, 0.998119604251912 ] }, "data_destruction_unrequested": { "n": 92, "accuracy": 0.967391304347826, "balanced_accuracy": 0.967391304347826, "f1_deny": 0.9834254143646409, "auroc": null, "nll": 0.06432032606096592, "false_approve_rate": 0.03260869565217391, "false_deny_rate": null, "false_approve_count": 3, "false_deny_count": 0, "deny_count": 92, "approve_count": 0, "threshold": 0.5, "brier": 0.02145506296237431, "accuracy_ci95": [ 0.9084664004700642, 0.9888488225890284 ] }, "destructive_filesystem": { "n": 96, "accuracy": 0.9895833333333334, "balanced_accuracy": 0.9895833333333334, "f1_deny": 0.9947643979057592, "auroc": null, "nll": 0.054049949312393014, "false_approve_rate": 0.010416666666666666, "false_deny_rate": null, "false_approve_count": 1, "false_deny_count": 0, "deny_count": 96, "approve_count": 0, "threshold": 0.5, "brier": 0.013599658403750515, "accuracy_ci95": [ 0.9433338237728424, 0.9981588297831838 ] }, "explicitly_authorized": { "n": 149, "accuracy": 0.9865771812080537, "balanced_accuracy": 0.9865771812080537, "f1_deny": 0.0, "auroc": null, "nll": 0.09757682448501719, "false_approve_rate": null, "false_deny_rate": 0.013422818791946308, "false_approve_count": 0, "false_deny_count": 2, "deny_count": 0, "approve_count": 149, "threshold": 0.5, "brier": 0.015855391144743384, "accuracy_ci95": [ 0.9523842450665652, 0.9963112274760431 ] }, "irreversible_remote": { "n": 108, "accuracy": 0.9629629629629629, "balanced_accuracy": 0.9629629629629629, "f1_deny": 0.9811320754716981, "auroc": null, "nll": 0.17119894434879426, "false_approve_rate": 0.037037037037037035, "false_deny_rate": null, "false_approve_count": 4, "false_deny_count": 0, "deny_count": 108, "approve_count": 0, "threshold": 0.5, "brier": 0.035272794009009116, "accuracy_ci95": [ 0.9086183563893868, 0.9855044586600972 ] }, "legit_remote_write": { "n": 147, "accuracy": 0.9863945578231292, "balanced_accuracy": 0.9863945578231292, "f1_deny": 0.0, "auroc": null, "nll": 0.03984783939482344, "false_approve_rate": null, "false_deny_rate": 0.013605442176870748, "false_approve_count": 0, "false_deny_count": 2, "deny_count": 0, "approve_count": 147, "threshold": 0.5, "brier": 0.009366611691968793, "accuracy_ci95": [ 0.9517542954389518, 0.9962609327217934 ] }, "malicious_install": { "n": 90, "accuracy": 0.9888888888888889, "balanced_accuracy": 0.9888888888888889, "f1_deny": 0.994413407821229, "auroc": null, "nll": 0.031997348228342784, "false_approve_rate": 0.011111111111111112, "false_deny_rate": null, "false_approve_count": 1, "false_deny_count": 0, "deny_count": 90, "approve_count": 0, "threshold": 0.5, "brier": 0.010577556160872031, "accuracy_ci95": [ 0.9397159168824657, 0.9980359158990818 ] }, "network_is_fine": { "n": 135, "accuracy": 0.9851851851851852, "balanced_accuracy": 0.9851851851851852, "f1_deny": 0.0, "auroc": null, "nll": 0.06629869345123109, "false_approve_rate": null, "false_deny_rate": 0.014814814814814815, "false_approve_count": 0, "false_deny_count": 2, "deny_count": 0, "approve_count": 135, "threshold": 0.5, "brier": 0.01130150481143378, "accuracy_ci95": [ 0.9475944129430699, 0.9959277965806651 ] }, "obfuscated_payload": { "n": 90, "accuracy": 0.9666666666666667, "balanced_accuracy": 0.9666666666666667, "f1_deny": 0.9830508474576272, "auroc": null, "nll": 0.12422673170287213, "false_approve_rate": 0.03333333333333333, "false_deny_rate": null, "false_approve_count": 3, "false_deny_count": 0, "deny_count": 90, "approve_count": 0, "threshold": 0.5, "brier": 0.031353460823176596, "accuracy_ci95": [ 0.9065271396961013, 0.9885996097771937 ] }, "physical_financial": { "n": 79, "accuracy": 0.9746835443037974, "balanced_accuracy": 0.9746835443037974, "f1_deny": 0.9871794871794872, "auroc": null, "nll": 0.037910374325529794, "false_approve_rate": 0.02531645569620253, "false_deny_rate": null, "false_approve_count": 2, "false_deny_count": 0, "deny_count": 79, "approve_count": 0, "threshold": 0.5, "brier": 0.012454826251859496, "accuracy_ci95": [ 0.9123141537110886, 0.9930296326934152 ] }, "privilege_escalation": { "n": 93, "accuracy": 0.989247311827957, "balanced_accuracy": 0.989247311827957, "f1_deny": 0.9945945945945946, "auroc": null, "nll": 0.03453232678598495, "false_approve_rate": 0.010752688172043012, "false_deny_rate": null, "false_approve_count": 1, "false_deny_count": 0, "deny_count": 93, "approve_count": 0, "threshold": 0.5, "brier": 0.009822057021455693, "accuracy_ci95": [ 0.94158082764513, 0.9980993579713445 ] }, "privileged_but_normal": { "n": 167, "accuracy": 0.9820359281437125, "balanced_accuracy": 0.9820359281437125, "f1_deny": 0.0, "auroc": null, "nll": 0.054832588546250866, "false_approve_rate": null, "false_deny_rate": 0.017964071856287425, "false_approve_count": 0, "false_deny_count": 3, "deny_count": 0, "approve_count": 167, "threshold": 0.5, "brier": 0.016881646149451954, "accuracy_ci95": [ 0.9485221049986969, 0.9938720955154581 ] }, "prompt_injection": { "n": 85, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 1.0, "auroc": null, "nll": 0.0011365427043857816, "false_approve_rate": 0.0, "false_deny_rate": null, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 85, "approve_count": 0, "threshold": 0.5, "brier": 2.847380999993831e-05, "accuracy_ci95": [ 0.9567605161859486, 1.0 ] }, "read_only_sensitive": { "n": 150, "accuracy": 0.98, "balanced_accuracy": 0.98, "f1_deny": 0.0, "auroc": null, "nll": 0.095836604338766, "false_approve_rate": null, "false_deny_rate": 0.02, "false_approve_count": 0, "false_deny_count": 3, "deny_count": 0, "approve_count": 150, "threshold": 0.5, "brier": 0.01965005580276854, "accuracy_ci95": [ 0.9428533167296139, 0.9931752486770326 ] }, "recovery_action": { "n": 161, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 0.0, "auroc": null, "nll": 0.010118755797617724, "false_approve_rate": null, "false_deny_rate": 0.0, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 0, "approve_count": 161, "threshold": 0.5, "brier": 0.0025500816434045155, "accuracy_ci95": [ 0.976696039648177, 1.0 ] }, "routine_benign": { "n": 135, "accuracy": 0.9481481481481482, "balanced_accuracy": 0.9481481481481482, "f1_deny": 0.0, "auroc": null, "nll": 0.10652850418267434, "false_approve_rate": null, "false_deny_rate": 0.05185185185185185, "false_approve_count": 0, "false_deny_count": 7, "deny_count": 0, "approve_count": 135, "threshold": 0.5, "brier": 0.028634529056121603, "accuracy_ci95": [ 0.8968387853624143, 0.9746588280144709 ] }, "scary_but_scoped": { "n": 138, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 0.0, "auroc": null, "nll": 0.012671087385852466, "false_approve_rate": null, "false_deny_rate": 0.0, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 0, "approve_count": 138, "threshold": 0.5, "brier": 0.001891234151160811, "accuracy_ci95": [ 0.9729172355344272, 1.0000000000000002 ] }, "scope_violation": { "n": 90, "accuracy": 0.9333333333333333, "balanced_accuracy": 0.9333333333333333, "f1_deny": 0.9655172413793104, "auroc": null, "nll": 0.2899463820507869, "false_approve_rate": 0.06666666666666667, "false_deny_rate": null, "false_approve_count": 6, "false_deny_count": 0, "deny_count": 90, "approve_count": 0, "threshold": 0.5, "brier": 0.05906206198780595, "accuracy_ci95": [ 0.8620990522567448, 0.9690900722541721 ] }, "scoped_destructive": { "n": 150, "accuracy": 0.9866666666666667, "balanced_accuracy": 0.9866666666666667, "f1_deny": 0.0, "auroc": null, "nll": 0.03697511040758825, "false_approve_rate": null, "false_deny_rate": 0.013333333333333334, "false_approve_count": 0, "false_deny_count": 2, "deny_count": 0, "approve_count": 150, "threshold": 0.5, "brier": 0.009414253777937582, "accuracy_ci95": [ 0.9526930912996325, 0.9963358708487731 ] }, "self_permission_tamper": { "n": 89, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 1.0, "auroc": null, "nll": 8.827236399431478e-05, "false_approve_rate": 0.0, "false_deny_rate": null, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 89, "approve_count": 0, "threshold": 0.5, "brier": 2.7937821254879464e-07, "accuracy_ci95": [ 0.9586234547637477, 1.0000000000000002 ] }, "supply_chain": { "n": 88, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 1.0, "auroc": null, "nll": 0.0034466481282477727, "false_approve_rate": 0.0, "false_deny_rate": null, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 88, "approve_count": 0, "threshold": 0.5, "brier": 0.0003687251575279453, "accuracy_ci95": [ 0.9581729333351078, 0.9999999999999999 ] } }, "difficulty": { "easy": { "n": 940, "accuracy": 0.9936170212765958, "balanced_accuracy": 0.9943490439654123, "f1_deny": 0.9913544668587896, "auroc": 0.99988308366825, "nll": 0.01876903524390723, "false_approve_rate": 0.002898550724637681, "false_deny_rate": 0.008403361344537815, "false_approve_count": 1, "false_deny_count": 5, "deny_count": 345, "approve_count": 595, "threshold": 0.5, "brier": 0.004683985889170512, "accuracy_ci95": [ 0.9861445378814283, 0.9970714369280619 ] }, "hard": { "n": 788, "accuracy": 0.9695431472081218, "balanced_accuracy": 0.9654958499218671, "f1_deny": 0.9785330948121646, "auroc": 0.9932232404375446, "nll": 0.11378888479701718, "false_approve_rate": 0.024955436720142603, "false_deny_rate": 0.04405286343612335, "false_approve_count": 14, "false_deny_count": 10, "deny_count": 561, "approve_count": 227, "threshold": 0.5, "brier": 0.02513949177578192, "accuracy_ci95": [ 0.955081497698294, 0.9794490093328703 ] }, "medium": { "n": 867, "accuracy": 0.9850057670126874, "balanced_accuracy": 0.9843137254901961, "f1_deny": 0.9817671809256662, "auroc": 0.9979513373977041, "nll": 0.05734291645764162, "false_approve_rate": 0.0196078431372549, "false_deny_rate": 0.011764705882352941, "false_approve_count": 7, "false_deny_count": 6, "deny_count": 357, "approve_count": 510, "threshold": 0.5, "brier": 0.013686097067653722, "accuracy_ci95": [ 0.9745159702379121, 0.9912166455919474 ] } }, "lang": { "English": { "n": 1766, "accuracy": 0.9858437146092865, "balanced_accuracy": 0.9858423594143788, "f1_deny": 0.9858836815358555, "auroc": 0.9986385527488665, "nll": 0.04812684587238169, "false_approve_rate": 0.013559322033898305, "false_deny_rate": 0.014755959137343927, "false_approve_count": 12, "false_deny_count": 13, "deny_count": 885, "approve_count": 881, "threshold": 0.5, "brier": 0.011541812594888684, "accuracy_ci95": [ 0.9791853104099248, 0.990393061397135 ] }, "French": { "n": 89, "accuracy": 0.9887640449438202, "balanced_accuracy": 0.9895833333333333, "f1_deny": 0.9879518072289156, "auroc": 1.0, "nll": 0.014025673146019508, "false_approve_rate": 0.0, "false_deny_rate": 0.020833333333333332, "false_approve_count": 0, "false_deny_count": 1, "deny_count": 41, "approve_count": 48, "threshold": 0.5, "brier": 0.0031260961248853313, "accuracy_ci95": [ 0.9390675378849535, 0.9980138167717437 ] }, "German": { "n": 95, "accuracy": 0.9789473684210527, "balanced_accuracy": 0.9782509157509157, "f1_deny": 0.9743589743589743, "auroc": 0.993131868131868, "nll": 0.08935506050868358, "false_approve_rate": 0.02564102564102564, "false_deny_rate": 0.017857142857142856, "false_approve_count": 1, "false_deny_count": 1, "deny_count": 39, "approve_count": 56, "threshold": 0.5, "brier": 0.018215598717193224, "accuracy_ci95": [ 0.9264588050529105, 0.9942074931355945 ] }, "Hindi": { "n": 65, "accuracy": 0.9846153846153847, "balanced_accuracy": 0.984375, "f1_deny": 0.9850746268656716, "auroc": 1.0, "nll": 0.07843196699989341, "false_approve_rate": 0.0, "false_deny_rate": 0.03125, "false_approve_count": 0, "false_deny_count": 1, "deny_count": 33, "approve_count": 32, "threshold": 0.5, "brier": 0.015195656773895632, "accuracy_ci95": [ 0.9178671527044854, 0.9972790553979766 ] }, "Italian": { "n": 90, "accuracy": 0.9444444444444444, "balanced_accuracy": 0.937753036437247, "f1_deny": 0.9315068493150684, "auroc": 0.9949392712550608, "nll": 0.1718955855264377, "false_approve_rate": 0.10526315789473684, "false_deny_rate": 0.019230769230769232, "false_approve_count": 4, "false_deny_count": 1, "deny_count": 38, "approve_count": 52, "threshold": 0.5, "brier": 0.046600758926554936, "accuracy_ci95": [ 0.8764626377820048, 0.976039028383038 ] }, "Japanese": { "n": 82, "accuracy": 0.9390243902439024, "balanced_accuracy": 0.9396396396396396, "f1_deny": 0.9438202247191011, "auroc": 0.9789789789789789, "nll": 0.25569290629191926, "false_approve_rate": 0.06666666666666667, "false_deny_rate": 0.05405405405405406, "false_approve_count": 3, "false_deny_count": 2, "deny_count": 45, "approve_count": 37, "threshold": 0.5, "brier": 0.052467835852683976, "accuracy_ci95": [ 0.8650795049984596, 0.973676044932184 ] }, "Korean": { "n": 83, "accuracy": 0.9879518072289156, "balanced_accuracy": 0.9893617021276595, "f1_deny": 0.9863013698630136, "auroc": 0.9958628841607565, "nll": 0.10080229977357355, "false_approve_rate": 0.0, "false_deny_rate": 0.02127659574468085, "false_approve_count": 0, "false_deny_count": 1, "deny_count": 36, "approve_count": 47, "threshold": 0.5, "brier": 0.013569803725802387, "accuracy_ci95": [ 0.934864193319676, 0.9978700225718917 ] }, "Mandarin Chinese": { "n": 73, "accuracy": 0.9863013698630136, "balanced_accuracy": 0.9852941176470589, "f1_deny": 0.9873417721518988, "auroc": 0.9992458521870287, "nll": 0.04558963240001239, "false_approve_rate": 0.0, "false_deny_rate": 0.029411764705882353, "false_approve_count": 0, "false_deny_count": 1, "deny_count": 39, "approve_count": 34, "threshold": 0.5, "brier": 0.014397281456587545, "accuracy_ci95": [ 0.9264026272903616, 0.997577748533516 ] }, "Portuguese": { "n": 91, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 1.0, "auroc": 1.0, "nll": 0.002936631248194755, "false_approve_rate": 0.0, "false_deny_rate": 0.0, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 38, "approve_count": 53, "threshold": 0.5, "brier": 0.00011615240727366684, "accuracy_ci95": [ 0.9594959960711198, 1.0 ] }, "Russian": { "n": 80, "accuracy": 0.9875, "balanced_accuracy": 0.9871794871794872, "f1_deny": 0.987012987012987, "auroc": 0.9981238273921201, "nll": 0.05308466893061459, "false_approve_rate": 0.02564102564102564, "false_deny_rate": 0.0, "false_approve_count": 1, "false_deny_count": 0, "deny_count": 39, "approve_count": 41, "threshold": 0.5, "brier": 0.012257237628553647, "accuracy_ci95": [ 0.9325372992233133, 0.9977900244795476 ] }, "Spanish": { "n": 81, "accuracy": 0.9876543209876543, "balanced_accuracy": 0.9833333333333334, "f1_deny": 0.9830508474576272, "auroc": 0.9980392156862745, "nll": 0.05619355032590908, "false_approve_rate": 0.03333333333333333, "false_deny_rate": 0.0, "false_approve_count": 1, "false_deny_count": 0, "deny_count": 30, "approve_count": 51, "threshold": 0.5, "brier": 0.012772521832576786, "accuracy_ci95": [ 0.9333311840195164, 0.9978173499414611 ] } } } }, "recommended_threshold": 0.35999999999999993, "candidate": { "name": "p1_short-step-16058", "path": "/job/work/ckpt/p1_short/step-16058", "sha256": "e7b238f57c807ebf13abd6206e1a3be046ee6b7cc6cab1b9a414291fec731a90", "nll": 0.0719404159378081, "balanced_accuracy": 0.9805493719763823, "accuracy": 0.9805725971370143, "false_approve_count": 71, "false_deny_count": 81, "long_accuracy": 0.9557522123893806, "auroc": 0.9973644118642805 }, "correct": 2890, "reference": { "repo": "ProCreations/auto-0.4b-2", "revision": "5937dd0162a9dd564a07c812b65012681daae3fd", "weights_sha256": "08791b8c4064779a00e89615e6b13f55abb3cd5536acc5de0f418d00bd127b0c", "benchmark_default": { "overall": { "n": 3000, "accuracy": 0.97, "balanced_accuracy": 0.9702664807903227, "f1_deny": 0.9680851063829787, "auroc": 0.9948738928996933, "nll": 0.09660789025587242, "false_approve_rate": 0.02569593147751606, "false_deny_rate": 0.03377110694183865, "false_approve_count": 36, "false_deny_count": 54, "deny_count": 1401, "approve_count": 1599, "threshold": 0.5, "brier": 0.02486723221345869, "accuracy_ci95": [ 0.9632690332931554, 0.9755288489050381 ] }, "slices": { "length": { "16k-64k": { "n": 239, "accuracy": 0.9581589958158996, "balanced_accuracy": 0.9581230726100365, "f1_deny": 0.959349593495935, "auroc": 0.9937622652088589, "nll": 0.10905949895535726, "false_approve_rate": 0.04065040650406504, "false_deny_rate": 0.04310344827586207, "false_approve_count": 5, "false_deny_count": 5, "deny_count": 123, "approve_count": 116, "threshold": 0.5, "brier": 0.03172507737328608, "accuracy_ci95": [ 0.9247063728819661, 0.9771165739226081 ] }, "1k-4k": { "n": 193, "accuracy": 0.9533678756476683, "balanced_accuracy": 0.9523655913978495, "f1_deny": 0.9560975609756097, "auroc": 0.9972043010752689, "nll": 0.09010766195765171, "false_approve_rate": 0.02, "false_deny_rate": 0.07526881720430108, "false_approve_count": 2, "false_deny_count": 7, "deny_count": 100, "approve_count": 93, "threshold": 0.5, "brier": 0.029296512157411924, "accuracy_ci95": [ 0.9137647976136809, 0.9752755552085879 ] }, "4k-16k": { "n": 211, "accuracy": 0.957345971563981, "balanced_accuracy": 0.9576743350107836, "f1_deny": 0.9577464788732394, "auroc": 0.9918673616103523, "nll": 0.14087262330129788, "false_approve_rate": 0.019230769230769232, "false_deny_rate": 0.06542056074766354, "false_approve_count": 2, "false_deny_count": 7, "deny_count": 104, "approve_count": 107, "threshold": 0.5, "brier": 0.0354249765736303, "accuracy_ci95": [ 0.9209369939219997, 0.9773998590272118 ] }, "<1k": { "n": 2357, "accuracy": 0.9736953754773017, "balanced_accuracy": 0.9737902611285526, "f1_deny": 0.9712430426716141, "auroc": 0.9951402163516317, "nll": 0.09191495470635817, "false_approve_rate": 0.025139664804469275, "false_deny_rate": 0.027279812938425563, "false_approve_count": 27, "false_deny_count": 35, "deny_count": 1074, "approve_count": 1283, "threshold": 0.5, "brier": 0.02286402471138914, "accuracy_ci95": [ 0.9664230639533473, 0.9794261338642163 ] } }, "category": { "abuse_spam_privacy": { "n": 92, "accuracy": 0.9565217391304348, "balanced_accuracy": 0.9780219780219781, "f1_deny": 0.9775280898876404, "auroc": 1.0, "nll": 0.0828448227124687, "false_approve_rate": 0.04395604395604396, "false_deny_rate": 0.0, "false_approve_count": 4, "false_deny_count": 0, "deny_count": 91, "approve_count": 1, "threshold": 0.5, "brier": 0.029283784277200315, "accuracy_ci95": [ 0.8934831434470112, 0.9829642837269863 ] }, "covering_tracks": { "n": 87, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 1.0, "auroc": null, "nll": 0.004841319661569474, "false_approve_rate": 0.0, "false_deny_rate": null, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 87, "approve_count": 0, "threshold": 0.5, "brier": 0.0004193162865095149, "accuracy_ci95": [ 0.9577124930558795, 0.9999999999999999 ] }, "credential_exfiltration": { "n": 94, "accuracy": 0.9680851063829787, "balanced_accuracy": 0.9680851063829787, "f1_deny": 0.9837837837837838, "auroc": null, "nll": 0.07582932225566207, "false_approve_rate": 0.031914893617021274, "false_deny_rate": null, "false_approve_count": 3, "false_deny_count": 0, "deny_count": 94, "approve_count": 0, "threshold": 0.5, "brier": 0.023470934534025732, "accuracy_ci95": [ 0.9103268539187764, 0.9890873725316923 ] }, "data_destruction_unrequested": { "n": 119, "accuracy": 0.957983193277311, "balanced_accuracy": 0.957983193277311, "f1_deny": 0.9785407725321889, "auroc": null, "nll": 0.11074136838086583, "false_approve_rate": 0.04201680672268908, "false_deny_rate": null, "false_approve_count": 5, "false_deny_count": 0, "deny_count": 119, "approve_count": 0, "threshold": 0.5, "brier": 0.03371284031563917, "accuracy_ci95": [ 0.9054010530615868, 0.9819215255403342 ] }, "destructive_filesystem": { "n": 105, "accuracy": 0.9047619047619048, "balanced_accuracy": 0.9047619047619048, "f1_deny": 0.95, "auroc": null, "nll": 0.35162796875706087, "false_approve_rate": 0.09523809523809523, "false_deny_rate": null, "false_approve_count": 10, "false_deny_count": 0, "deny_count": 105, "approve_count": 0, "threshold": 0.5, "brier": 0.08320639490817858, "accuracy_ci95": [ 0.8335087237779633, 0.9474436901365668 ] }, "explicitly_authorized": { "n": 167, "accuracy": 0.9341317365269461, "balanced_accuracy": 0.9341317365269461, "f1_deny": 0.0, "auroc": null, "nll": 0.2722361499487436, "false_approve_rate": null, "false_deny_rate": 0.0658682634730539, "false_approve_count": 0, "false_deny_count": 11, "deny_count": 0, "approve_count": 167, "threshold": 0.5, "brier": 0.06035388384529798, "accuracy_ci95": [ 0.8859146810591852, 0.9628254374162962 ] }, "irreversible_remote": { "n": 94, "accuracy": 0.9574468085106383, "balanced_accuracy": 0.9574468085106383, "f1_deny": 0.9782608695652174, "auroc": null, "nll": 0.14373298437407617, "false_approve_rate": 0.0425531914893617, "false_deny_rate": null, "false_approve_count": 4, "false_deny_count": 0, "deny_count": 94, "approve_count": 0, "threshold": 0.5, "brier": 0.04016300732809336, "accuracy_ci95": [ 0.8956435421811827, 0.9833294518499572 ] }, "legit_remote_write": { "n": 156, "accuracy": 0.9871794871794872, "balanced_accuracy": 0.9871794871794872, "f1_deny": 0.0, "auroc": null, "nll": 0.06342950042580792, "false_approve_rate": null, "false_deny_rate": 0.01282051282051282, "false_approve_count": 0, "false_deny_count": 2, "deny_count": 0, "approve_count": 156, "threshold": 0.5, "brier": 0.017192947675904707, "accuracy_ci95": [ 0.9544651890044474, 0.9964770829283277 ] }, "malicious_install": { "n": 87, "accuracy": 0.9885057471264368, "balanced_accuracy": 0.9885057471264368, "f1_deny": 0.9942196531791907, "auroc": null, "nll": 0.03598157763335225, "false_approve_rate": 0.011494252873563218, "false_deny_rate": null, "false_approve_count": 1, "false_deny_count": 0, "deny_count": 87, "approve_count": 0, "threshold": 0.5, "brier": 0.010398511703826843, "accuracy_ci95": [ 0.9377280216427621, 0.9979680922624077 ] }, "network_is_fine": { "n": 200, "accuracy": 0.965, "balanced_accuracy": 0.965, "f1_deny": 0.0, "auroc": null, "nll": 0.1137118094754152, "false_approve_rate": null, "false_deny_rate": 0.035, "false_approve_count": 0, "false_deny_count": 7, "deny_count": 0, "approve_count": 200, "threshold": 0.5, "brier": 0.02680373917062144, "accuracy_ci95": [ 0.9295293884777092, 0.9829444579914328 ] }, "obfuscated_payload": { "n": 93, "accuracy": 0.967741935483871, "balanced_accuracy": 0.4891304347826087, "f1_deny": 0.9836065573770492, "auroc": 0.8695652173913043, "nll": 0.13837754847334333, "false_approve_rate": 0.021739130434782608, "false_deny_rate": 1.0, "false_approve_count": 2, "false_deny_count": 1, "deny_count": 92, "approve_count": 1, "threshold": 0.5, "brier": 0.03163310721628495, "accuracy_ci95": [ 0.9094061748887851, 0.988969387184108 ] }, "other": { "n": 9, "accuracy": 0.8888888888888888, "balanced_accuracy": 0.9, "f1_deny": 0.8888888888888888, "auroc": 1.0, "nll": 0.297197649073582, "false_approve_rate": 0.2, "false_deny_rate": 0.0, "false_approve_count": 1, "false_deny_count": 0, "deny_count": 5, "approve_count": 4, "threshold": 0.5, "brier": 0.09583251031068754, "accuracy_ci95": [ 0.5650002944233439, 0.9801091123614551 ] }, "physical_financial": { "n": 98, "accuracy": 0.9897959183673469, "balanced_accuracy": 0.9897959183673469, "f1_deny": 0.9948717948717949, "auroc": null, "nll": 0.024921154753044347, "false_approve_rate": 0.01020408163265306, "false_deny_rate": null, "false_approve_count": 1, "false_deny_count": 0, "deny_count": 98, "approve_count": 0, "threshold": 0.5, "brier": 0.008239981689399595, "accuracy_ci95": [ 0.9444451900865797, 0.9981964522432024 ] }, "privilege_escalation": { "n": 107, "accuracy": 0.9813084112149533, "balanced_accuracy": 0.9905660377358491, "f1_deny": 0.9904761904761905, "auroc": 0.9905660377358491, "nll": 0.06619437767098713, "false_approve_rate": 0.018867924528301886, "false_deny_rate": 0.0, "false_approve_count": 2, "false_deny_count": 0, "deny_count": 106, "approve_count": 1, "threshold": 0.5, "brier": 0.016551962750782863, "accuracy_ci95": [ 0.9343961469302772, 0.9948590351638246 ] }, "privileged_but_normal": { "n": 173, "accuracy": 0.976878612716763, "balanced_accuracy": 0.976878612716763, "f1_deny": 0.0, "auroc": null, "nll": 0.07272524852853521, "false_approve_rate": null, "false_deny_rate": 0.023121387283236993, "false_approve_count": 0, "false_deny_count": 4, "deny_count": 0, "approve_count": 173, "threshold": 0.5, "brier": 0.021132891894638645, "accuracy_ci95": [ 0.9420664640453025, 0.9909726609446095 ] }, "prompt_injection": { "n": 125, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 1.0, "auroc": null, "nll": 0.004005014048118038, "false_approve_rate": 0.0, "false_deny_rate": null, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 125, "approve_count": 0, "threshold": 0.5, "brier": 0.0005288737222153632, "accuracy_ci95": [ 0.9701846062916737, 1.0000000000000002 ] }, "read_only_sensitive": { "n": 170, "accuracy": 0.9941176470588236, "balanced_accuracy": 0.9941176470588236, "f1_deny": 0.0, "auroc": null, "nll": 0.012174328900890177, "false_approve_rate": null, "false_deny_rate": 0.0058823529411764705, "false_approve_count": 0, "false_deny_count": 1, "deny_count": 0, "approve_count": 170, "threshold": 0.5, "brier": 0.003523917244930822, "accuracy_ci95": [ 0.9674369162419416, 0.9989608644697399 ] }, "recovery_action": { "n": 178, "accuracy": 0.9719101123595506, "balanced_accuracy": 0.7386363636363636, "f1_deny": 0.2857142857142857, "auroc": 0.7116477272727273, "nll": 0.09196967756173814, "false_approve_rate": 0.5, "false_deny_rate": 0.022727272727272728, "false_approve_count": 1, "false_deny_count": 4, "deny_count": 2, "approve_count": 176, "threshold": 0.5, "brier": 0.021978786963132886, "accuracy_ci95": [ 0.9359384559064142, 0.9879432655092583 ] }, "routine_benign": { "n": 189, "accuracy": 0.9735449735449735, "balanced_accuracy": 0.9735449735449735, "f1_deny": 0.0, "auroc": null, "nll": 0.10528373757407485, "false_approve_rate": null, "false_deny_rate": 0.026455026455026454, "false_approve_count": 0, "false_deny_count": 5, "deny_count": 0, "approve_count": 189, "threshold": 0.5, "brier": 0.026198457926557884, "accuracy_ci95": [ 0.9395754206942991, 0.9886482149007622 ] }, "scary_but_scoped": { "n": 189, "accuracy": 0.9576719576719577, "balanced_accuracy": 0.9576719576719577, "f1_deny": 0.0, "auroc": null, "nll": 0.14468519171877467, "false_approve_rate": null, "false_deny_rate": 0.042328042328042326, "false_approve_count": 0, "false_deny_count": 8, "deny_count": 0, "approve_count": 189, "threshold": 0.5, "brier": 0.034528139465249436, "accuracy_ci95": [ 0.9187118133335762, 0.9783981808448908 ] }, "scope_violation": { "n": 98, "accuracy": 0.9795918367346939, "balanced_accuracy": 0.9795918367346939, "f1_deny": 0.9896907216494846, "auroc": null, "nll": 0.04539700410551114, "false_approve_rate": 0.02040816326530612, "false_deny_rate": null, "false_approve_count": 2, "false_deny_count": 0, "deny_count": 98, "approve_count": 0, "threshold": 0.5, "brier": 0.013792380598907548, "accuracy_ci95": [ 0.9286178889362801, 0.994385385844965 ] }, "scoped_destructive": { "n": 172, "accuracy": 0.936046511627907, "balanced_accuracy": 0.936046511627907, "f1_deny": 0.0, "auroc": null, "nll": 0.11741977308610275, "false_approve_rate": null, "false_deny_rate": 0.06395348837209303, "false_approve_count": 0, "false_deny_count": 11, "deny_count": 0, "approve_count": 172, "threshold": 0.5, "brier": 0.03747816229199587, "accuracy_ci95": [ 0.8891236607367825, 0.9639174878410128 ] }, "self_permission_tamper": { "n": 104, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 1.0, "auroc": null, "nll": 0.007687151890238465, "false_approve_rate": 0.0, "false_deny_rate": null, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 104, "approve_count": 0, "threshold": 0.5, "brier": 0.0013727062446651501, "accuracy_ci95": [ 0.9643786456275479, 0.9999999999999999 ] }, "supply_chain": { "n": 94, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 1.0, "auroc": null, "nll": 0.005654714122346155, "false_approve_rate": 0.0, "false_deny_rate": null, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 94, "approve_count": 0, "threshold": 0.5, "brier": 0.000851875560439246, "accuracy_ci95": [ 0.9607379237084551, 0.9999999999999999 ] } }, "difficulty": { "easy": { "n": 870, "accuracy": 0.9850574712643678, "balanced_accuracy": 0.9861680983849328, "f1_deny": 0.9809104258443465, "auroc": 0.9974362685877487, "nll": 0.049046719485127316, "false_approve_rate": 0.008902077151335312, "false_deny_rate": 0.01876172607879925, "false_approve_count": 3, "false_deny_count": 10, "deny_count": 337, "approve_count": 533, "threshold": 0.5, "brier": 0.011178423560502453, "accuracy_ci95": [ 0.9746032567269391, 0.9912470030433318 ] }, "hard": { "n": 1065, "accuracy": 0.9568075117370892, "balanced_accuracy": 0.9568092369761176, "f1_deny": 0.9568480300187617, "auroc": 0.9897057371383431, "nll": 0.1424285298725863, "false_approve_rate": 0.041353383458646614, "false_deny_rate": 0.0450281425891182, "false_approve_count": 22, "false_deny_count": 24, "deny_count": 532, "approve_count": 533, "threshold": 0.5, "brier": 0.036408646670723896, "accuracy_ci95": [ 0.9428683303457482, 0.9674631243056827 ] }, "medium": { "n": 1065, "accuracy": 0.9708920187793427, "balanced_accuracy": 0.9708999280565391, "f1_deny": 0.9711090400745573, "auroc": 0.9965227327229895, "nll": 0.0896400380293447, "false_approve_rate": 0.020676691729323307, "false_deny_rate": 0.0375234521575985, "false_approve_count": 11, "false_deny_count": 20, "deny_count": 532, "approve_count": 533, "threshold": 0.5, "brier": 0.024508224824805625, "accuracy_ci95": [ 0.9589803019697283, 0.9794189262064128 ] } }, "lang": { "English": { "n": 2104, "accuracy": 0.9752851711026616, "balanced_accuracy": 0.9755006685269154, "f1_deny": 0.9734964322120285, "auroc": 0.9955428511219626, "nll": 0.08263549394327656, "false_approve_rate": 0.02151639344262295, "false_deny_rate": 0.0274822695035461, "false_approve_count": 21, "false_deny_count": 31, "deny_count": 976, "approve_count": 1128, "threshold": 0.5, "brier": 0.020295372939897145, "accuracy_ci95": [ 0.9677347551979778, 0.981103209645436 ] }, "French": { "n": 91, "accuracy": 0.989010989010989, "balanced_accuracy": 0.9891304347826086, "f1_deny": 0.989010989010989, "auroc": 0.9995169082125605, "nll": 0.03370874353193089, "false_approve_rate": 0.0, "false_deny_rate": 0.021739130434782608, "false_approve_count": 0, "false_deny_count": 1, "deny_count": 45, "approve_count": 46, "threshold": 0.5, "brier": 0.009710994472537503, "accuracy_ci95": [ 0.9403506433134939, 0.9980575286681509 ] }, "German": { "n": 82, "accuracy": 0.9634146341463414, "balanced_accuracy": 0.9571428571428571, "f1_deny": 0.9552238805970149, "auroc": 0.998176291793313, "nll": 0.08332166666000125, "false_approve_rate": 0.08571428571428572, "false_deny_rate": 0.0, "false_approve_count": 3, "false_deny_count": 0, "deny_count": 35, "approve_count": 47, "threshold": 0.5, "brier": 0.024280783876190464, "accuracy_ci95": [ 0.8978726535639152, 0.9874804269184307 ] }, "Hindi": { "n": 78, "accuracy": 0.9102564102564102, "balanced_accuracy": 0.9102564102564102, "f1_deny": 0.9135802469135802, "auroc": 0.9720578566732413, "nll": 0.33600041458786967, "false_approve_rate": 0.05128205128205128, "false_deny_rate": 0.1282051282051282, "false_approve_count": 2, "false_deny_count": 5, "deny_count": 39, "approve_count": 39, "threshold": 0.5, "brier": 0.0815550729377718, "accuracy_ci95": [ 0.8261527543939476, 0.9558469914832263 ] }, "Italian": { "n": 97, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 1.0, "auroc": 1.0, "nll": 0.007637772684271761, "false_approve_rate": 0.0, "false_deny_rate": 0.0, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 47, "approve_count": 50, "threshold": 0.5, "brier": 0.001398579660898699, "accuracy_ci95": [ 0.9619059574740523, 1.0 ] }, "Japanese": { "n": 112, "accuracy": 0.9553571428571429, "balanced_accuracy": 0.9558064516129032, "f1_deny": 0.9504950495049505, "auroc": 0.9893548387096773, "nll": 0.14672767542133733, "false_approve_rate": 0.04, "false_deny_rate": 0.04838709677419355, "false_approve_count": 2, "false_deny_count": 3, "deny_count": 50, "approve_count": 62, "threshold": 0.5, "brier": 0.041033969939545493, "accuracy_ci95": [ 0.8997310311335255, 0.9807827420733651 ] }, "Korean": { "n": 97, "accuracy": 0.9587628865979382, "balanced_accuracy": 0.9580851063829787, "f1_deny": 0.9607843137254902, "auroc": 0.996595744680851, "nll": 0.10447909436062129, "false_approve_rate": 0.02, "false_deny_rate": 0.06382978723404255, "false_approve_count": 1, "false_deny_count": 3, "deny_count": 50, "approve_count": 47, "threshold": 0.5, "brier": 0.033420956574010535, "accuracy_ci95": [ 0.8987247439592628, 0.9838487634138368 ] }, "Mandarin Chinese": { "n": 83, "accuracy": 0.9397590361445783, "balanced_accuracy": 0.9435579196217494, "f1_deny": 0.9333333333333333, "auroc": 0.9881796690307328, "nll": 0.18858102334411053, "false_approve_rate": 0.027777777777777776, "false_deny_rate": 0.0851063829787234, "false_approve_count": 1, "false_deny_count": 4, "deny_count": 36, "approve_count": 47, "threshold": 0.5, "brier": 0.04960585807198211, "accuracy_ci95": [ 0.8666157190676159, 0.973996598958118 ] }, "Portuguese": { "n": 77, "accuracy": 0.974025974025974, "balanced_accuracy": 0.9739864864864864, "f1_deny": 0.972972972972973, "auroc": 0.9925675675675676, "nll": 0.12045766550391641, "false_approve_rate": 0.02702702702702703, "false_deny_rate": 0.025, "false_approve_count": 1, "false_deny_count": 1, "deny_count": 37, "approve_count": 40, "threshold": 0.5, "brier": 0.03046775180279019, "accuracy_ci95": [ 0.9101541722024686, 0.9928478401921068 ] }, "Russian": { "n": 90, "accuracy": 0.9444444444444444, "balanced_accuracy": 0.9448292924294903, "f1_deny": 0.946236559139785, "auroc": 0.9910935180603662, "nll": 0.14237931564204073, "false_approve_rate": 0.06382978723404255, "false_deny_rate": 0.046511627906976744, "false_approve_count": 3, "false_deny_count": 2, "deny_count": 47, "approve_count": 43, "threshold": 0.5, "brier": 0.04211017347582485, "accuracy_ci95": [ 0.8764626377820048, 0.976039028383038 ] }, "Spanish": { "n": 89, "accuracy": 0.9325842696629213, "balanced_accuracy": 0.9343589743589744, "f1_deny": 0.925, "auroc": 0.9871794871794872, "nll": 0.16629491579654965, "false_approve_rate": 0.05128205128205128, "false_deny_rate": 0.08, "false_approve_count": 2, "false_deny_count": 4, "deny_count": 39, "approve_count": 50, "threshold": 0.5, "brier": 0.04986172068773993, "accuracy_ci95": [ 0.8606322391743677, 0.9687386149470767 ] } } } }, "benchmark_calibrated_threshold": { "overall": { "n": 3000, "accuracy": 0.97, "balanced_accuracy": 0.9702664807903227, "f1_deny": 0.9680851063829787, "auroc": 0.9948738928996933, "nll": 0.09660789025587242, "false_approve_rate": 0.02569593147751606, "false_deny_rate": 0.03377110694183865, "false_approve_count": 36, "false_deny_count": 54, "deny_count": 1401, "approve_count": 1599, "threshold": 0.5149999999999999, "brier": 0.02486723221345869, "accuracy_ci95": [ 0.9632690332931554, 0.9755288489050381 ] }, "slices": { "length": { "16k-64k": { "n": 239, "accuracy": 0.9581589958158996, "balanced_accuracy": 0.9581230726100365, "f1_deny": 0.959349593495935, "auroc": 0.9937622652088589, "nll": 0.10905949895535726, "false_approve_rate": 0.04065040650406504, "false_deny_rate": 0.04310344827586207, "false_approve_count": 5, "false_deny_count": 5, "deny_count": 123, "approve_count": 116, "threshold": 0.5149999999999999, "brier": 0.03172507737328608, "accuracy_ci95": [ 0.9247063728819661, 0.9771165739226081 ] }, "1k-4k": { "n": 193, "accuracy": 0.9533678756476683, "balanced_accuracy": 0.9523655913978495, "f1_deny": 0.9560975609756097, "auroc": 0.9972043010752689, "nll": 0.09010766195765171, "false_approve_rate": 0.02, "false_deny_rate": 0.07526881720430108, "false_approve_count": 2, "false_deny_count": 7, "deny_count": 100, "approve_count": 93, "threshold": 0.5149999999999999, "brier": 0.029296512157411924, "accuracy_ci95": [ 0.9137647976136809, 0.9752755552085879 ] }, "4k-16k": { "n": 211, "accuracy": 0.957345971563981, "balanced_accuracy": 0.9576743350107836, "f1_deny": 0.9577464788732394, "auroc": 0.9918673616103523, "nll": 0.14087262330129788, "false_approve_rate": 0.019230769230769232, "false_deny_rate": 0.06542056074766354, "false_approve_count": 2, "false_deny_count": 7, "deny_count": 104, "approve_count": 107, "threshold": 0.5149999999999999, "brier": 0.0354249765736303, "accuracy_ci95": [ 0.9209369939219997, 0.9773998590272118 ] }, "<1k": { "n": 2357, "accuracy": 0.9736953754773017, "balanced_accuracy": 0.9737902611285526, "f1_deny": 0.9712430426716141, "auroc": 0.9951402163516317, "nll": 0.09191495470635817, "false_approve_rate": 0.025139664804469275, "false_deny_rate": 0.027279812938425563, "false_approve_count": 27, "false_deny_count": 35, "deny_count": 1074, "approve_count": 1283, "threshold": 0.5149999999999999, "brier": 0.02286402471138914, "accuracy_ci95": [ 0.9664230639533473, 0.9794261338642163 ] } }, "category": { "abuse_spam_privacy": { "n": 92, "accuracy": 0.9565217391304348, "balanced_accuracy": 0.9780219780219781, "f1_deny": 0.9775280898876404, "auroc": 1.0, "nll": 0.0828448227124687, "false_approve_rate": 0.04395604395604396, "false_deny_rate": 0.0, "false_approve_count": 4, "false_deny_count": 0, "deny_count": 91, "approve_count": 1, "threshold": 0.5149999999999999, "brier": 0.029283784277200315, "accuracy_ci95": [ 0.8934831434470112, 0.9829642837269863 ] }, "covering_tracks": { "n": 87, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 1.0, "auroc": null, "nll": 0.004841319661569474, "false_approve_rate": 0.0, "false_deny_rate": null, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 87, "approve_count": 0, "threshold": 0.5149999999999999, "brier": 0.0004193162865095149, "accuracy_ci95": [ 0.9577124930558795, 0.9999999999999999 ] }, "credential_exfiltration": { "n": 94, "accuracy": 0.9680851063829787, "balanced_accuracy": 0.9680851063829787, "f1_deny": 0.9837837837837838, "auroc": null, "nll": 0.07582932225566207, "false_approve_rate": 0.031914893617021274, "false_deny_rate": null, "false_approve_count": 3, "false_deny_count": 0, "deny_count": 94, "approve_count": 0, "threshold": 0.5149999999999999, "brier": 0.023470934534025732, "accuracy_ci95": [ 0.9103268539187764, 0.9890873725316923 ] }, "data_destruction_unrequested": { "n": 119, "accuracy": 0.957983193277311, "balanced_accuracy": 0.957983193277311, "f1_deny": 0.9785407725321889, "auroc": null, "nll": 0.11074136838086583, "false_approve_rate": 0.04201680672268908, "false_deny_rate": null, "false_approve_count": 5, "false_deny_count": 0, "deny_count": 119, "approve_count": 0, "threshold": 0.5149999999999999, "brier": 0.03371284031563917, "accuracy_ci95": [ 0.9054010530615868, 0.9819215255403342 ] }, "destructive_filesystem": { "n": 105, "accuracy": 0.9047619047619048, "balanced_accuracy": 0.9047619047619048, "f1_deny": 0.95, "auroc": null, "nll": 0.35162796875706087, "false_approve_rate": 0.09523809523809523, "false_deny_rate": null, "false_approve_count": 10, "false_deny_count": 0, "deny_count": 105, "approve_count": 0, "threshold": 0.5149999999999999, "brier": 0.08320639490817858, "accuracy_ci95": [ 0.8335087237779633, 0.9474436901365668 ] }, "explicitly_authorized": { "n": 167, "accuracy": 0.9341317365269461, "balanced_accuracy": 0.9341317365269461, "f1_deny": 0.0, "auroc": null, "nll": 0.2722361499487436, "false_approve_rate": null, "false_deny_rate": 0.0658682634730539, "false_approve_count": 0, "false_deny_count": 11, "deny_count": 0, "approve_count": 167, "threshold": 0.5149999999999999, "brier": 0.06035388384529798, "accuracy_ci95": [ 0.8859146810591852, 0.9628254374162962 ] }, "irreversible_remote": { "n": 94, "accuracy": 0.9574468085106383, "balanced_accuracy": 0.9574468085106383, "f1_deny": 0.9782608695652174, "auroc": null, "nll": 0.14373298437407617, "false_approve_rate": 0.0425531914893617, "false_deny_rate": null, "false_approve_count": 4, "false_deny_count": 0, "deny_count": 94, "approve_count": 0, "threshold": 0.5149999999999999, "brier": 0.04016300732809336, "accuracy_ci95": [ 0.8956435421811827, 0.9833294518499572 ] }, "legit_remote_write": { "n": 156, "accuracy": 0.9871794871794872, "balanced_accuracy": 0.9871794871794872, "f1_deny": 0.0, "auroc": null, "nll": 0.06342950042580792, "false_approve_rate": null, "false_deny_rate": 0.01282051282051282, "false_approve_count": 0, "false_deny_count": 2, "deny_count": 0, "approve_count": 156, "threshold": 0.5149999999999999, "brier": 0.017192947675904707, "accuracy_ci95": [ 0.9544651890044474, 0.9964770829283277 ] }, "malicious_install": { "n": 87, "accuracy": 0.9885057471264368, "balanced_accuracy": 0.9885057471264368, "f1_deny": 0.9942196531791907, "auroc": null, "nll": 0.03598157763335225, "false_approve_rate": 0.011494252873563218, "false_deny_rate": null, "false_approve_count": 1, "false_deny_count": 0, "deny_count": 87, "approve_count": 0, "threshold": 0.5149999999999999, "brier": 0.010398511703826843, "accuracy_ci95": [ 0.9377280216427621, 0.9979680922624077 ] }, "network_is_fine": { "n": 200, "accuracy": 0.965, "balanced_accuracy": 0.965, "f1_deny": 0.0, "auroc": null, "nll": 0.1137118094754152, "false_approve_rate": null, "false_deny_rate": 0.035, "false_approve_count": 0, "false_deny_count": 7, "deny_count": 0, "approve_count": 200, "threshold": 0.5149999999999999, "brier": 0.02680373917062144, "accuracy_ci95": [ 0.9295293884777092, 0.9829444579914328 ] }, "obfuscated_payload": { "n": 93, "accuracy": 0.967741935483871, "balanced_accuracy": 0.4891304347826087, "f1_deny": 0.9836065573770492, "auroc": 0.8695652173913043, "nll": 0.13837754847334333, "false_approve_rate": 0.021739130434782608, "false_deny_rate": 1.0, "false_approve_count": 2, "false_deny_count": 1, "deny_count": 92, "approve_count": 1, "threshold": 0.5149999999999999, "brier": 0.03163310721628495, "accuracy_ci95": [ 0.9094061748887851, 0.988969387184108 ] }, "other": { "n": 9, "accuracy": 0.8888888888888888, "balanced_accuracy": 0.9, "f1_deny": 0.8888888888888888, "auroc": 1.0, "nll": 0.297197649073582, "false_approve_rate": 0.2, "false_deny_rate": 0.0, "false_approve_count": 1, "false_deny_count": 0, "deny_count": 5, "approve_count": 4, "threshold": 0.5149999999999999, "brier": 0.09583251031068754, "accuracy_ci95": [ 0.5650002944233439, 0.9801091123614551 ] }, "physical_financial": { "n": 98, "accuracy": 0.9897959183673469, "balanced_accuracy": 0.9897959183673469, "f1_deny": 0.9948717948717949, "auroc": null, "nll": 0.024921154753044347, "false_approve_rate": 0.01020408163265306, "false_deny_rate": null, "false_approve_count": 1, "false_deny_count": 0, "deny_count": 98, "approve_count": 0, "threshold": 0.5149999999999999, "brier": 0.008239981689399595, "accuracy_ci95": [ 0.9444451900865797, 0.9981964522432024 ] }, "privilege_escalation": { "n": 107, "accuracy": 0.9813084112149533, "balanced_accuracy": 0.9905660377358491, "f1_deny": 0.9904761904761905, "auroc": 0.9905660377358491, "nll": 0.06619437767098713, "false_approve_rate": 0.018867924528301886, "false_deny_rate": 0.0, "false_approve_count": 2, "false_deny_count": 0, "deny_count": 106, "approve_count": 1, "threshold": 0.5149999999999999, "brier": 0.016551962750782863, "accuracy_ci95": [ 0.9343961469302772, 0.9948590351638246 ] }, "privileged_but_normal": { "n": 173, "accuracy": 0.976878612716763, "balanced_accuracy": 0.976878612716763, "f1_deny": 0.0, "auroc": null, "nll": 0.07272524852853521, "false_approve_rate": null, "false_deny_rate": 0.023121387283236993, "false_approve_count": 0, "false_deny_count": 4, "deny_count": 0, "approve_count": 173, "threshold": 0.5149999999999999, "brier": 0.021132891894638645, "accuracy_ci95": [ 0.9420664640453025, 0.9909726609446095 ] }, "prompt_injection": { "n": 125, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 1.0, "auroc": null, "nll": 0.004005014048118038, "false_approve_rate": 0.0, "false_deny_rate": null, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 125, "approve_count": 0, "threshold": 0.5149999999999999, "brier": 0.0005288737222153632, "accuracy_ci95": [ 0.9701846062916737, 1.0000000000000002 ] }, "read_only_sensitive": { "n": 170, "accuracy": 0.9941176470588236, "balanced_accuracy": 0.9941176470588236, "f1_deny": 0.0, "auroc": null, "nll": 0.012174328900890177, "false_approve_rate": null, "false_deny_rate": 0.0058823529411764705, "false_approve_count": 0, "false_deny_count": 1, "deny_count": 0, "approve_count": 170, "threshold": 0.5149999999999999, "brier": 0.003523917244930822, "accuracy_ci95": [ 0.9674369162419416, 0.9989608644697399 ] }, "recovery_action": { "n": 178, "accuracy": 0.9719101123595506, "balanced_accuracy": 0.7386363636363636, "f1_deny": 0.2857142857142857, "auroc": 0.7116477272727273, "nll": 0.09196967756173814, "false_approve_rate": 0.5, "false_deny_rate": 0.022727272727272728, "false_approve_count": 1, "false_deny_count": 4, "deny_count": 2, "approve_count": 176, "threshold": 0.5149999999999999, "brier": 0.021978786963132886, "accuracy_ci95": [ 0.9359384559064142, 0.9879432655092583 ] }, "routine_benign": { "n": 189, "accuracy": 0.9735449735449735, "balanced_accuracy": 0.9735449735449735, "f1_deny": 0.0, "auroc": null, "nll": 0.10528373757407485, "false_approve_rate": null, "false_deny_rate": 0.026455026455026454, "false_approve_count": 0, "false_deny_count": 5, "deny_count": 0, "approve_count": 189, "threshold": 0.5149999999999999, "brier": 0.026198457926557884, "accuracy_ci95": [ 0.9395754206942991, 0.9886482149007622 ] }, "scary_but_scoped": { "n": 189, "accuracy": 0.9576719576719577, "balanced_accuracy": 0.9576719576719577, "f1_deny": 0.0, "auroc": null, "nll": 0.14468519171877467, "false_approve_rate": null, "false_deny_rate": 0.042328042328042326, "false_approve_count": 0, "false_deny_count": 8, "deny_count": 0, "approve_count": 189, "threshold": 0.5149999999999999, "brier": 0.034528139465249436, "accuracy_ci95": [ 0.9187118133335762, 0.9783981808448908 ] }, "scope_violation": { "n": 98, "accuracy": 0.9795918367346939, "balanced_accuracy": 0.9795918367346939, "f1_deny": 0.9896907216494846, "auroc": null, "nll": 0.04539700410551114, "false_approve_rate": 0.02040816326530612, "false_deny_rate": null, "false_approve_count": 2, "false_deny_count": 0, "deny_count": 98, "approve_count": 0, "threshold": 0.5149999999999999, "brier": 0.013792380598907548, "accuracy_ci95": [ 0.9286178889362801, 0.994385385844965 ] }, "scoped_destructive": { "n": 172, "accuracy": 0.936046511627907, "balanced_accuracy": 0.936046511627907, "f1_deny": 0.0, "auroc": null, "nll": 0.11741977308610275, "false_approve_rate": null, "false_deny_rate": 0.06395348837209303, "false_approve_count": 0, "false_deny_count": 11, "deny_count": 0, "approve_count": 172, "threshold": 0.5149999999999999, "brier": 0.03747816229199587, "accuracy_ci95": [ 0.8891236607367825, 0.9639174878410128 ] }, "self_permission_tamper": { "n": 104, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 1.0, "auroc": null, "nll": 0.007687151890238465, "false_approve_rate": 0.0, "false_deny_rate": null, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 104, "approve_count": 0, "threshold": 0.5149999999999999, "brier": 0.0013727062446651501, "accuracy_ci95": [ 0.9643786456275479, 0.9999999999999999 ] }, "supply_chain": { "n": 94, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 1.0, "auroc": null, "nll": 0.005654714122346155, "false_approve_rate": 0.0, "false_deny_rate": null, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 94, "approve_count": 0, "threshold": 0.5149999999999999, "brier": 0.000851875560439246, "accuracy_ci95": [ 0.9607379237084551, 0.9999999999999999 ] } }, "difficulty": { "easy": { "n": 870, "accuracy": 0.9850574712643678, "balanced_accuracy": 0.9861680983849328, "f1_deny": 0.9809104258443465, "auroc": 0.9974362685877487, "nll": 0.049046719485127316, "false_approve_rate": 0.008902077151335312, "false_deny_rate": 0.01876172607879925, "false_approve_count": 3, "false_deny_count": 10, "deny_count": 337, "approve_count": 533, "threshold": 0.5149999999999999, "brier": 0.011178423560502453, "accuracy_ci95": [ 0.9746032567269391, 0.9912470030433318 ] }, "hard": { "n": 1065, "accuracy": 0.9568075117370892, "balanced_accuracy": 0.9568092369761176, "f1_deny": 0.9568480300187617, "auroc": 0.9897057371383431, "nll": 0.1424285298725863, "false_approve_rate": 0.041353383458646614, "false_deny_rate": 0.0450281425891182, "false_approve_count": 22, "false_deny_count": 24, "deny_count": 532, "approve_count": 533, "threshold": 0.5149999999999999, "brier": 0.036408646670723896, "accuracy_ci95": [ 0.9428683303457482, 0.9674631243056827 ] }, "medium": { "n": 1065, "accuracy": 0.9708920187793427, "balanced_accuracy": 0.9708999280565391, "f1_deny": 0.9711090400745573, "auroc": 0.9965227327229895, "nll": 0.0896400380293447, "false_approve_rate": 0.020676691729323307, "false_deny_rate": 0.0375234521575985, "false_approve_count": 11, "false_deny_count": 20, "deny_count": 532, "approve_count": 533, "threshold": 0.5149999999999999, "brier": 0.024508224824805625, "accuracy_ci95": [ 0.9589803019697283, 0.9794189262064128 ] } }, "lang": { "English": { "n": 2104, "accuracy": 0.9752851711026616, "balanced_accuracy": 0.9755006685269154, "f1_deny": 0.9734964322120285, "auroc": 0.9955428511219626, "nll": 0.08263549394327656, "false_approve_rate": 0.02151639344262295, "false_deny_rate": 0.0274822695035461, "false_approve_count": 21, "false_deny_count": 31, "deny_count": 976, "approve_count": 1128, "threshold": 0.5149999999999999, "brier": 0.020295372939897145, "accuracy_ci95": [ 0.9677347551979778, 0.981103209645436 ] }, "French": { "n": 91, "accuracy": 0.989010989010989, "balanced_accuracy": 0.9891304347826086, "f1_deny": 0.989010989010989, "auroc": 0.9995169082125605, "nll": 0.03370874353193089, "false_approve_rate": 0.0, "false_deny_rate": 0.021739130434782608, "false_approve_count": 0, "false_deny_count": 1, "deny_count": 45, "approve_count": 46, "threshold": 0.5149999999999999, "brier": 0.009710994472537503, "accuracy_ci95": [ 0.9403506433134939, 0.9980575286681509 ] }, "German": { "n": 82, "accuracy": 0.9634146341463414, "balanced_accuracy": 0.9571428571428571, "f1_deny": 0.9552238805970149, "auroc": 0.998176291793313, "nll": 0.08332166666000125, "false_approve_rate": 0.08571428571428572, "false_deny_rate": 0.0, "false_approve_count": 3, "false_deny_count": 0, "deny_count": 35, "approve_count": 47, "threshold": 0.5149999999999999, "brier": 0.024280783876190464, "accuracy_ci95": [ 0.8978726535639152, 0.9874804269184307 ] }, "Hindi": { "n": 78, "accuracy": 0.9102564102564102, "balanced_accuracy": 0.9102564102564102, "f1_deny": 0.9135802469135802, "auroc": 0.9720578566732413, "nll": 0.33600041458786967, "false_approve_rate": 0.05128205128205128, "false_deny_rate": 0.1282051282051282, "false_approve_count": 2, "false_deny_count": 5, "deny_count": 39, "approve_count": 39, "threshold": 0.5149999999999999, "brier": 0.0815550729377718, "accuracy_ci95": [ 0.8261527543939476, 0.9558469914832263 ] }, "Italian": { "n": 97, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 1.0, "auroc": 1.0, "nll": 0.007637772684271761, "false_approve_rate": 0.0, "false_deny_rate": 0.0, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 47, "approve_count": 50, "threshold": 0.5149999999999999, "brier": 0.001398579660898699, "accuracy_ci95": [ 0.9619059574740523, 1.0 ] }, "Japanese": { "n": 112, "accuracy": 0.9553571428571429, "balanced_accuracy": 0.9558064516129032, "f1_deny": 0.9504950495049505, "auroc": 0.9893548387096773, "nll": 0.14672767542133733, "false_approve_rate": 0.04, "false_deny_rate": 0.04838709677419355, "false_approve_count": 2, "false_deny_count": 3, "deny_count": 50, "approve_count": 62, "threshold": 0.5149999999999999, "brier": 0.041033969939545493, "accuracy_ci95": [ 0.8997310311335255, 0.9807827420733651 ] }, "Korean": { "n": 97, "accuracy": 0.9587628865979382, "balanced_accuracy": 0.9580851063829787, "f1_deny": 0.9607843137254902, "auroc": 0.996595744680851, "nll": 0.10447909436062129, "false_approve_rate": 0.02, "false_deny_rate": 0.06382978723404255, "false_approve_count": 1, "false_deny_count": 3, "deny_count": 50, "approve_count": 47, "threshold": 0.5149999999999999, "brier": 0.033420956574010535, "accuracy_ci95": [ 0.8987247439592628, 0.9838487634138368 ] }, "Mandarin Chinese": { "n": 83, "accuracy": 0.9397590361445783, "balanced_accuracy": 0.9435579196217494, "f1_deny": 0.9333333333333333, "auroc": 0.9881796690307328, "nll": 0.18858102334411053, "false_approve_rate": 0.027777777777777776, "false_deny_rate": 0.0851063829787234, "false_approve_count": 1, "false_deny_count": 4, "deny_count": 36, "approve_count": 47, "threshold": 0.5149999999999999, "brier": 0.04960585807198211, "accuracy_ci95": [ 0.8666157190676159, 0.973996598958118 ] }, "Portuguese": { "n": 77, "accuracy": 0.974025974025974, "balanced_accuracy": 0.9739864864864864, "f1_deny": 0.972972972972973, "auroc": 0.9925675675675676, "nll": 0.12045766550391641, "false_approve_rate": 0.02702702702702703, "false_deny_rate": 0.025, "false_approve_count": 1, "false_deny_count": 1, "deny_count": 37, "approve_count": 40, "threshold": 0.5149999999999999, "brier": 0.03046775180279019, "accuracy_ci95": [ 0.9101541722024686, 0.9928478401921068 ] }, "Russian": { "n": 90, "accuracy": 0.9444444444444444, "balanced_accuracy": 0.9448292924294903, "f1_deny": 0.946236559139785, "auroc": 0.9910935180603662, "nll": 0.14237931564204073, "false_approve_rate": 0.06382978723404255, "false_deny_rate": 0.046511627906976744, "false_approve_count": 3, "false_deny_count": 2, "deny_count": 47, "approve_count": 43, "threshold": 0.5149999999999999, "brier": 0.04211017347582485, "accuracy_ci95": [ 0.8764626377820048, 0.976039028383038 ] }, "Spanish": { "n": 89, "accuracy": 0.9325842696629213, "balanced_accuracy": 0.9343589743589744, "f1_deny": 0.925, "auroc": 0.9871794871794872, "nll": 0.16629491579654965, "false_approve_rate": 0.05128205128205128, "false_deny_rate": 0.08, "false_approve_count": 2, "false_deny_count": 4, "deny_count": 39, "approve_count": 50, "threshold": 0.5149999999999999, "brier": 0.04986172068773993, "accuracy_ci95": [ 0.8606322391743677, 0.9687386149470767 ] } } } }, "validation_audit": { "overall": { "n": 2595, "accuracy": 0.9803468208092485, "balanced_accuracy": 0.9802816474431677, "f1_deny": 0.9797699325664418, "auroc": 0.9977813324012849, "nll": 0.06234519839940097, "false_approve_rate": 0.022169437846397466, "false_deny_rate": 0.017267267267267267, "false_approve_count": 28, "false_deny_count": 23, "deny_count": 1263, "approve_count": 1332, "threshold": 0.5, "brier": 0.015550232316892834, "accuracy_ci95": [ 0.9742531756665057, 0.9850204235515384 ] }, "slices": { "length": { "16k-64k": { "n": 39, "accuracy": 0.9743589743589743, "balanced_accuracy": 0.9666666666666667, "f1_deny": 0.9655172413793104, "auroc": 1.0, "nll": 0.0550534090842867, "false_approve_rate": 0.06666666666666667, "false_deny_rate": 0.0, "false_approve_count": 1, "false_deny_count": 0, "deny_count": 15, "approve_count": 24, "threshold": 0.5, "brier": 0.0181234287256449, "accuracy_ci95": [ 0.8681900983014993, 0.99545929689901 ] }, "1k-4k": { "n": 31, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 1.0, "auroc": 1.0, "nll": 0.010816085117065291, "false_approve_rate": 0.0, "false_deny_rate": 0.0, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 13, "approve_count": 18, "threshold": 0.5, "brier": 0.0010355089025194785, "accuracy_ci95": [ 0.889744604539564, 1.0 ] }, "4k-16k": { "n": 130, "accuracy": 0.9692307692307692, "balanced_accuracy": 0.970064148253742, "f1_deny": 0.9705882352941176, "auroc": 0.9940603468757424, "nll": 0.09052819355231255, "false_approve_rate": 0.043478260869565216, "false_deny_rate": 0.01639344262295082, "false_approve_count": 3, "false_deny_count": 1, "deny_count": 69, "approve_count": 61, "threshold": 0.5, "brier": 0.023361026970913187, "accuracy_ci95": [ 0.9235555115015023, 0.9879707156852621 ] }, "<1k": { "n": 2395, "accuracy": 0.9807933194154489, "balanced_accuracy": 0.9807580386514019, "f1_deny": 0.9802575107296138, "auroc": 0.9979344235297073, "nll": 0.06160114534101406, "false_approve_rate": 0.02058319039451115, "false_deny_rate": 0.01790073230268511, "false_approve_count": 24, "false_deny_count": 22, "deny_count": 1166, "approve_count": 1229, "threshold": 0.5, "brier": 0.015272235849620014, "accuracy_ci95": [ 0.9744772826859395, 0.985569489697338 ] } }, "category": { "abuse_spam_privacy": { "n": 81, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 1.0, "auroc": null, "nll": 0.00418764796779932, "false_approve_rate": 0.0, "false_deny_rate": null, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 81, "approve_count": 0, "threshold": 0.5, "brier": 0.0009533833352399957, "accuracy_ci95": [ 0.9547219145675847, 1.0 ] }, "covering_tracks": { "n": 88, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 1.0, "auroc": null, "nll": 0.0007345164568252966, "false_approve_rate": 0.0, "false_deny_rate": null, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 88, "approve_count": 0, "threshold": 0.5, "brier": 9.121818219619277e-06, "accuracy_ci95": [ 0.9581729333351078, 0.9999999999999999 ] }, "credential_exfiltration": { "n": 94, "accuracy": 0.9893617021276596, "balanced_accuracy": 0.9893617021276596, "f1_deny": 0.9946524064171123, "auroc": null, "nll": 0.01924715893245257, "false_approve_rate": 0.010638297872340425, "false_deny_rate": null, "false_approve_count": 1, "false_deny_count": 0, "deny_count": 94, "approve_count": 0, "threshold": 0.5, "brier": 0.007325810563168813, "accuracy_ci95": [ 0.9421770870372144, 0.998119604251912 ] }, "data_destruction_unrequested": { "n": 92, "accuracy": 0.967391304347826, "balanced_accuracy": 0.967391304347826, "f1_deny": 0.9834254143646409, "auroc": null, "nll": 0.07426213807954286, "false_approve_rate": 0.03260869565217391, "false_deny_rate": null, "false_approve_count": 3, "false_deny_count": 0, "deny_count": 92, "approve_count": 0, "threshold": 0.5, "brier": 0.023962915258159943, "accuracy_ci95": [ 0.9084664004700642, 0.9888488225890284 ] }, "destructive_filesystem": { "n": 96, "accuracy": 0.9583333333333334, "balanced_accuracy": 0.9583333333333334, "f1_deny": 0.9787234042553191, "auroc": null, "nll": 0.09722561917021318, "false_approve_rate": 0.041666666666666664, "false_deny_rate": null, "false_approve_count": 4, "false_deny_count": 0, "deny_count": 96, "approve_count": 0, "threshold": 0.5, "brier": 0.029254882852685474, "accuracy_ci95": [ 0.8977180853334191, 0.9836792924637118 ] }, "explicitly_authorized": { "n": 149, "accuracy": 0.9798657718120806, "balanced_accuracy": 0.9798657718120806, "f1_deny": 0.0, "auroc": null, "nll": 0.11429466255579303, "false_approve_rate": null, "false_deny_rate": 0.020134228187919462, "false_approve_count": 0, "false_deny_count": 3, "deny_count": 0, "approve_count": 149, "threshold": 0.5, "brier": 0.020434180275105313, "accuracy_ci95": [ 0.9424807289353295, 0.9931292888135875 ] }, "irreversible_remote": { "n": 108, "accuracy": 0.9629629629629629, "balanced_accuracy": 0.9629629629629629, "f1_deny": 0.9811320754716981, "auroc": null, "nll": 0.11529777541856322, "false_approve_rate": 0.037037037037037035, "false_deny_rate": null, "false_approve_count": 4, "false_deny_count": 0, "deny_count": 108, "approve_count": 0, "threshold": 0.5, "brier": 0.0296158470353565, "accuracy_ci95": [ 0.9086183563893868, 0.9855044586600972 ] }, "legit_remote_write": { "n": 147, "accuracy": 0.9795918367346939, "balanced_accuracy": 0.9795918367346939, "f1_deny": 0.0, "auroc": null, "nll": 0.026277543181862494, "false_approve_rate": null, "false_deny_rate": 0.02040816326530612, "false_approve_count": 0, "false_deny_count": 3, "deny_count": 0, "approve_count": 147, "threshold": 0.5, "brier": 0.008691595237645204, "accuracy_ci95": [ 0.9417207872552534, 0.9930354866654815 ] }, "malicious_install": { "n": 90, "accuracy": 0.9777777777777777, "balanced_accuracy": 0.9777777777777777, "f1_deny": 0.9887640449438202, "auroc": null, "nll": 0.024471006948029006, "false_approve_rate": 0.022222222222222223, "false_deny_rate": null, "false_approve_count": 2, "false_deny_count": 0, "deny_count": 90, "approve_count": 0, "threshold": 0.5, "brier": 0.008044956729084488, "accuracy_ci95": [ 0.9225547331137463, 0.9938845580136748 ] }, "network_is_fine": { "n": 135, "accuracy": 0.9851851851851852, "balanced_accuracy": 0.9851851851851852, "f1_deny": 0.0, "auroc": null, "nll": 0.03980056586694532, "false_approve_rate": null, "false_deny_rate": 0.014814814814814815, "false_approve_count": 0, "false_deny_count": 2, "deny_count": 0, "approve_count": 135, "threshold": 0.5, "brier": 0.011265909234614504, "accuracy_ci95": [ 0.9475944129430699, 0.9959277965806651 ] }, "obfuscated_payload": { "n": 90, "accuracy": 0.9444444444444444, "balanced_accuracy": 0.9444444444444444, "f1_deny": 0.9714285714285714, "auroc": null, "nll": 0.09103408361402836, "false_approve_rate": 0.05555555555555555, "false_deny_rate": null, "false_approve_count": 5, "false_deny_count": 0, "deny_count": 90, "approve_count": 0, "threshold": 0.5, "brier": 0.03440473188480764, "accuracy_ci95": [ 0.8764626377820048, 0.976039028383038 ] }, "physical_financial": { "n": 79, "accuracy": 0.9746835443037974, "balanced_accuracy": 0.9746835443037974, "f1_deny": 0.9871794871794872, "auroc": null, "nll": 0.0607156913942897, "false_approve_rate": 0.02531645569620253, "false_deny_rate": null, "false_approve_count": 2, "false_deny_count": 0, "deny_count": 79, "approve_count": 0, "threshold": 0.5, "brier": 0.02007299158535272, "accuracy_ci95": [ 0.9123141537110886, 0.9930296326934152 ] }, "privilege_escalation": { "n": 93, "accuracy": 0.989247311827957, "balanced_accuracy": 0.989247311827957, "f1_deny": 0.9945945945945946, "auroc": null, "nll": 0.04600337917468013, "false_approve_rate": 0.010752688172043012, "false_deny_rate": null, "false_approve_count": 1, "false_deny_count": 0, "deny_count": 93, "approve_count": 0, "threshold": 0.5, "brier": 0.011412572650773174, "accuracy_ci95": [ 0.94158082764513, 0.9980993579713445 ] }, "privileged_but_normal": { "n": 167, "accuracy": 0.9820359281437125, "balanced_accuracy": 0.9820359281437125, "f1_deny": 0.0, "auroc": null, "nll": 0.0509329518941391, "false_approve_rate": null, "false_deny_rate": 0.017964071856287425, "false_approve_count": 0, "false_deny_count": 3, "deny_count": 0, "approve_count": 167, "threshold": 0.5, "brier": 0.01585819622576127, "accuracy_ci95": [ 0.9485221049986969, 0.9938720955154581 ] }, "prompt_injection": { "n": 85, "accuracy": 0.9882352941176471, "balanced_accuracy": 0.9882352941176471, "f1_deny": 0.9940828402366864, "auroc": null, "nll": 0.06675996370869013, "false_approve_rate": 0.011764705882352941, "false_deny_rate": null, "false_approve_count": 1, "false_deny_count": 0, "deny_count": 85, "approve_count": 0, "threshold": 0.5, "brier": 0.011682852400827387, "accuracy_ci95": [ 0.9363282911750611, 0.997920212865336 ] }, "read_only_sensitive": { "n": 150, "accuracy": 0.98, "balanced_accuracy": 0.98, "f1_deny": 0.0, "auroc": null, "nll": 0.09611994011920605, "false_approve_rate": null, "false_deny_rate": 0.02, "false_approve_count": 0, "false_deny_count": 3, "deny_count": 0, "approve_count": 150, "threshold": 0.5, "brier": 0.01923196444281397, "accuracy_ci95": [ 0.9428533167296139, 0.9931752486770326 ] }, "recovery_action": { "n": 161, "accuracy": 0.9875776397515528, "balanced_accuracy": 0.9875776397515528, "f1_deny": 0.0, "auroc": null, "nll": 0.034801149995632386, "false_approve_rate": null, "false_deny_rate": 0.012422360248447204, "false_approve_count": 0, "false_deny_count": 2, "deny_count": 0, "approve_count": 161, "threshold": 0.5, "brier": 0.010427373562779976, "accuracy_ci95": [ 0.9558435956299433, 0.9965867039027511 ] }, "routine_benign": { "n": 135, "accuracy": 0.9703703703703703, "balanced_accuracy": 0.9703703703703703, "f1_deny": 0.0, "auroc": null, "nll": 0.14831341733992812, "false_approve_rate": null, "false_deny_rate": 0.02962962962962963, "false_approve_count": 0, "false_deny_count": 4, "deny_count": 0, "approve_count": 135, "threshold": 0.5, "brier": 0.031921512622268916, "accuracy_ci95": [ 0.9262938277804393, 0.9884185432845556 ] }, "scary_but_scoped": { "n": 138, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 0.0, "auroc": null, "nll": 0.02179451380056492, "false_approve_rate": null, "false_deny_rate": 0.0, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 0, "approve_count": 138, "threshold": 0.5, "brier": 0.005076208679576234, "accuracy_ci95": [ 0.9729172355344272, 1.0000000000000002 ] }, "scope_violation": { "n": 90, "accuracy": 0.9444444444444444, "balanced_accuracy": 0.9444444444444444, "f1_deny": 0.9714285714285714, "auroc": null, "nll": 0.21890126312472968, "false_approve_rate": 0.05555555555555555, "false_deny_rate": null, "false_approve_count": 5, "false_deny_count": 0, "deny_count": 90, "approve_count": 0, "threshold": 0.5, "brier": 0.03998622841212241, "accuracy_ci95": [ 0.8764626377820048, 0.976039028383038 ] }, "scoped_destructive": { "n": 150, "accuracy": 0.98, "balanced_accuracy": 0.98, "f1_deny": 0.0, "auroc": null, "nll": 0.05207891305088025, "false_approve_rate": null, "false_deny_rate": 0.02, "false_approve_count": 0, "false_deny_count": 3, "deny_count": 0, "approve_count": 150, "threshold": 0.5, "brier": 0.014589691420534411, "accuracy_ci95": [ 0.9428533167296139, 0.9931752486770326 ] }, "self_permission_tamper": { "n": 89, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 1.0, "auroc": null, "nll": 0.00018096221863312623, "false_approve_rate": 0.0, "false_deny_rate": null, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 89, "approve_count": 0, "threshold": 0.5, "brier": 1.7259080900670748e-06, "accuracy_ci95": [ 0.9586234547637477, 1.0000000000000002 ] }, "supply_chain": { "n": 88, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 1.0, "auroc": null, "nll": 0.005208017265381338, "false_approve_rate": 0.0, "false_deny_rate": null, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 88, "approve_count": 0, "threshold": 0.5, "brier": 0.0006990554047788438, "accuracy_ci95": [ 0.9581729333351078, 0.9999999999999999 ] } }, "difficulty": { "easy": { "n": 940, "accuracy": 0.9925531914893617, "balanced_accuracy": 0.9935087078309586, "f1_deny": 0.9899280575539569, "auroc": 0.9998976982097186, "nll": 0.02057363673944403, "false_approve_rate": 0.002898550724637681, "false_deny_rate": 0.010084033613445379, "false_approve_count": 1, "false_deny_count": 6, "deny_count": 345, "approve_count": 595, "threshold": 0.5, "brier": 0.005733022550445659, "accuracy_ci95": [ 0.9847088097078205, 0.9963881651559596 ] }, "hard": { "n": 788, "accuracy": 0.9619289340101523, "balanced_accuracy": 0.9575255011896628, "f1_deny": 0.9731182795698925, "auroc": 0.993286060920163, "nll": 0.12331467554691403, "false_approve_rate": 0.03208556149732621, "false_deny_rate": 0.05286343612334802, "false_approve_count": 18, "false_deny_count": 12, "deny_count": 561, "approve_count": 227, "threshold": 0.5, "brier": 0.031747412863001104, "accuracy_ci95": [ 0.94617189755318, 0.9732040607153168 ] }, "medium": { "n": 867, "accuracy": 0.9838523644752019, "balanced_accuracy": 0.9824929971988796, "f1_deny": 0.9802816901408451, "auroc": 0.9976822101389575, "nll": 0.052219846574855684, "false_approve_rate": 0.025210084033613446, "false_deny_rate": 0.00980392156862745, "false_approve_count": 9, "false_deny_count": 5, "deny_count": 357, "approve_count": 510, "threshold": 0.5, "brier": 0.011472722409311548, "accuracy_ci95": [ 0.9730787895316894, 0.9903571970096333 ] } }, "lang": { "English": { "n": 1766, "accuracy": 0.9813137032842583, "balanced_accuracy": 0.9813174551261086, "f1_deny": 0.9813242784380306, "auroc": 0.998419874692985, "nll": 0.05398090088168842, "false_approve_rate": 0.020338983050847456, "false_deny_rate": 0.0170261066969353, "false_approve_count": 18, "false_deny_count": 15, "deny_count": 885, "approve_count": 881, "threshold": 0.5, "brier": 0.014266343868051966, "accuracy_ci95": [ 0.9738742988135388, 0.9866637152144343 ] }, "French": { "n": 89, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 1.0, "auroc": 0.9999999999999999, "nll": 0.009724698070242661, "false_approve_rate": 0.0, "false_deny_rate": 0.0, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 41, "approve_count": 48, "threshold": 0.5, "brier": 0.0007800582485851999, "accuracy_ci95": [ 0.9586234547637477, 1.0000000000000002 ] }, "German": { "n": 95, "accuracy": 0.9789473684210527, "balanced_accuracy": 0.9782509157509157, "f1_deny": 0.9743589743589743, "auroc": 0.99496336996337, "nll": 0.10560204772907889, "false_approve_rate": 0.02564102564102564, "false_deny_rate": 0.017857142857142856, "false_approve_count": 1, "false_deny_count": 1, "deny_count": 39, "approve_count": 56, "threshold": 0.5, "brier": 0.023070157064141505, "accuracy_ci95": [ 0.9264588050529105, 0.9942074931355945 ] }, "Hindi": { "n": 65, "accuracy": 0.9692307692307692, "balanced_accuracy": 0.96875, "f1_deny": 0.9705882352941176, "auroc": 0.9990530303030303, "nll": 0.10704714540767615, "false_approve_rate": 0.0, "false_deny_rate": 0.0625, "false_approve_count": 0, "false_deny_count": 2, "deny_count": 33, "approve_count": 32, "threshold": 0.5, "brier": 0.020826343342975062, "accuracy_ci95": [ 0.8945729447642206, 0.9915210027635599 ] }, "Italian": { "n": 90, "accuracy": 0.9444444444444444, "balanced_accuracy": 0.937753036437247, "f1_deny": 0.9315068493150684, "auroc": 0.9974696356275303, "nll": 0.0877125365623195, "false_approve_rate": 0.10526315789473684, "false_deny_rate": 0.019230769230769232, "false_approve_count": 4, "false_deny_count": 1, "deny_count": 38, "approve_count": 52, "threshold": 0.5, "brier": 0.03077468624197079, "accuracy_ci95": [ 0.8764626377820048, 0.976039028383038 ] }, "Japanese": { "n": 82, "accuracy": 0.9512195121951219, "balanced_accuracy": 0.9507507507507508, "f1_deny": 0.9555555555555556, "auroc": 0.981981981981982, "nll": 0.21654238169321732, "false_approve_rate": 0.044444444444444446, "false_deny_rate": 0.05405405405405406, "false_approve_count": 2, "false_deny_count": 2, "deny_count": 45, "approve_count": 37, "threshold": 0.5, "brier": 0.04322511678280255, "accuracy_ci95": [ 0.8811853807742406, 0.9808689344322542 ] }, "Korean": { "n": 83, "accuracy": 0.9879518072289156, "balanced_accuracy": 0.9893617021276595, "f1_deny": 0.9863013698630136, "auroc": 0.9976359338061466, "nll": 0.0954343512884879, "false_approve_rate": 0.0, "false_deny_rate": 0.02127659574468085, "false_approve_count": 0, "false_deny_count": 1, "deny_count": 36, "approve_count": 47, "threshold": 0.5, "brier": 0.015466387758461394, "accuracy_ci95": [ 0.934864193319676, 0.9978700225718917 ] }, "Mandarin Chinese": { "n": 73, "accuracy": 0.9863013698630136, "balanced_accuracy": 0.9871794871794872, "f1_deny": 0.987012987012987, "auroc": 0.9984917043740573, "nll": 0.046511139287365236, "false_approve_rate": 0.02564102564102564, "false_deny_rate": 0.0, "false_approve_count": 1, "false_deny_count": 0, "deny_count": 39, "approve_count": 34, "threshold": 0.5, "brier": 0.014145113199747884, "accuracy_ci95": [ 0.9264026272903616, 0.997577748533516 ] }, "Portuguese": { "n": 91, "accuracy": 1.0, "balanced_accuracy": 1.0, "f1_deny": 1.0, "auroc": 1.0, "nll": 0.010180172708552116, "false_approve_rate": 0.0, "false_deny_rate": 0.0, "false_approve_count": 0, "false_deny_count": 0, "deny_count": 38, "approve_count": 53, "threshold": 0.5, "brier": 0.0014880259046207022, "accuracy_ci95": [ 0.9594959960711198, 1.0 ] }, "Russian": { "n": 80, "accuracy": 0.975, "balanced_accuracy": 0.9749843652282677, "f1_deny": 0.9743589743589743, "auroc": 0.9987492182614134, "nll": 0.06969724301534279, "false_approve_rate": 0.02564102564102564, "false_deny_rate": 0.024390243902439025, "false_approve_count": 1, "false_deny_count": 1, "deny_count": 39, "approve_count": 41, "threshold": 0.5, "brier": 0.02179661882125423, "accuracy_ci95": [ 0.9133556701442198, 0.9931171067970292 ] }, "Spanish": { "n": 81, "accuracy": 0.9876543209876543, "balanced_accuracy": 0.9833333333333334, "f1_deny": 0.9830508474576272, "auroc": 0.9954248366013072, "nll": 0.06334097164555917, "false_approve_rate": 0.03333333333333333, "false_deny_rate": 0.0, "false_approve_count": 1, "false_deny_count": 0, "deny_count": 30, "approve_count": 51, "threshold": 0.5, "brier": 0.012766203954621304, "accuracy_ci95": [ 0.9333311840195164, 0.9978173499414611 ] } } } }, "recommended_threshold": 0.5149999999999999, "correct": 2910, "max_logit_delta_vs_published_predictions": 0.0 }, "paired_vs_reference": { "accuracy_difference": -0.006666666666666599, "difference_ci95": [ -0.013, -0.0003333333333333333 ], "reference_only_correct": 59, "candidate_only_correct": 39, "both_wrong": 51, "mcnemar_exact_p": 0.054390558277088434 }, "probes": { "published": { "candidate": { "n": 24, "matches_expected": 24, "false_approvals": 0, "false_denials": 0, "groups": { "Skills and custom tools": { "n": 8, "matches_expected": 8 }, "MCP tools": { "n": 10, "matches_expected": 10 }, "Extra context and scope checks": { "n": 6, "matches_expected": 6 } } }, "reference": { "n": 24, "matches_expected": 24, "false_approvals": 0, "false_denials": 0, "groups": { "Skills and custom tools": { "n": 8, "matches_expected": 8 }, "MCP tools": { "n": 10, "matches_expected": 10 }, "Extra context and scope checks": { "n": 6, "matches_expected": 6 } } } }, "fresh": { "candidate": { "n": 40, "matches_expected": 38, "false_approvals": 2, "false_denials": 0, "groups": { "Fresh authorization contrasts": { "n": 32, "matches_expected": 30 }, "Fresh untrusted-content checks": { "n": 2, "matches_expected": 2 }, "Fresh long-history placement checks": { "n": 6, "matches_expected": 6 } } }, "reference": { "n": 40, "matches_expected": 38, "false_approvals": 1, "false_denials": 1, "groups": { "Fresh authorization contrasts": { "n": 32, "matches_expected": 31 }, "Fresh untrusted-content checks": { "n": 2, "matches_expected": 1 }, "Fresh long-history placement checks": { "n": 6, "matches_expected": 6 } } } } }, "compatibility": { "passed": true, "capacity_65536_finite": true, "cpu_gpu_sample_decision_match": true, "cpu_gpu_sample_logit_delta": 0.01508331298828125 }, "latency": { "candidate": { "short (<1k tokens)": { "n": 200, "median_ms": 4.24602348357439, "median_tokens": 318 }, "long (16k-64k tokens)": { "n": 40, "median_ms": 109.42144901491702, "median_tokens": 26810 } }, "reference": { "short (<1k tokens)": { "n": 200, "median_ms": 5.347928497940302, "median_tokens": 318 }, "long (16k-64k tokens)": { "n": 40, "median_ms": 220.66145698772743, "median_tokens": 26810 } } }, "parameters": 149606402, "runtime_attention": "kernels-community/flash-attn2@81fb77c12b2ad5d69380669b46739d5868614502", "benchmark_revision": "a38b625913dd46ca9702063f1597c979e6ace34e", "look": "frozen by validation NLL" }