| { |
| "methodology_note": "RQ1 (calibration) computed on the FULL held-out validation split (11666 frames / 1049 videos), 20-pass MC-Dropout (the plugin's designed default). RQ2/RQ3 (explanation-quality correlation, triage transferability) computed on a class-balanced stratified sample (up to 2000 real + 2000 fake frames per model, drawn from the same val split) because the perturbation-based explanation-stability metric requires one forward+backward pass per sample and is not batchable -- full-scale would be many hours of single-image GPU work per model. num_perturbations=3 matches backend/main.py's actual deployed setting exactly.", |
| "val_frame_count": 11666, |
| "val_video_count": 1049, |
| "models": { |
| "xception": { |
| "rq1_calibration": { |
| "frame_ece": 0.032874380892149876, |
| "video_ece": 0.037745453723143835, |
| "frame_auroc": 0.9976220768807692, |
| "video_auroc": 0.9995995550611791, |
| "frame_reliability": { |
| "bin_accuracies": [ |
| 0.9478943611705924, |
| 0.15151515151515152, |
| 0.09090909090909091, |
| 0.044444444444444446, |
| 0.0625, |
| 0.9193548387096774, |
| 0.9393939393939394, |
| 1.0, |
| 1.0, |
| 0.9993681283963098 |
| ], |
| "bin_confidences": [ |
| 0.9961150191419919, |
| 0.8451707138482368, |
| 0.7526288335973567, |
| 0.6554717653327518, |
| 0.5481232423335314, |
| 0.552825871013826, |
| 0.6581012985923074, |
| 0.7533314636973447, |
| 0.8542808221161731, |
| 0.9956830007063067 |
| ], |
| "bin_counts": [ |
| 2802, |
| 66, |
| 55, |
| 45, |
| 48, |
| 62, |
| 66, |
| 86, |
| 163, |
| 7913 |
| ], |
| "bin_boundaries": [ |
| 0.0, |
| 0.1, |
| 0.2, |
| 0.30000000000000004, |
| 0.4, |
| 0.5, |
| 0.6000000000000001, |
| 0.7000000000000001, |
| 0.8, |
| 0.9, |
| 1.0 |
| ] |
| }, |
| "video_reliability": { |
| "bin_accuracies": [ |
| 0.9733333333333334, |
| 0.4, |
| 1.0, |
| 0.3333333333333333, |
| 0.0, |
| 1.0, |
| 1.0, |
| 1.0, |
| 1.0, |
| 1.0 |
| ], |
| "bin_confidences": [ |
| 0.9954425858807838, |
| 0.8405209202261176, |
| 0.7561018553096801, |
| 0.6423113923558654, |
| 0.5399914891505614, |
| 0.5524718514084817, |
| 0.6568172973223576, |
| 0.7598564254777798, |
| 0.8647716873423498, |
| 0.9896538321624408 |
| ], |
| "bin_counts": [ |
| 150, |
| 5, |
| 1, |
| 3, |
| 5, |
| 5, |
| 13, |
| 34, |
| 54, |
| 777 |
| ], |
| "bin_boundaries": [ |
| 0.0, |
| 0.1, |
| 0.2, |
| 0.30000000000000004, |
| 0.4, |
| 0.5, |
| 0.6000000000000001, |
| 0.7000000000000001, |
| 0.8, |
| 0.9, |
| 1.0 |
| ] |
| } |
| }, |
| "rq2_uncertainty_explanation_correlation": { |
| "n_samples": 4000, |
| "n_skipped_timeout_or_error": 0, |
| "spearman_rho": -0.05301441006945229, |
| "p_value": 0.000795843020519685 |
| }, |
| "rq3_triage_transferability": { |
| "n_samples": 4000, |
| "n_false_negatives": 83, |
| "false_negative_escalation_rate": 0.9518072289156626, |
| "overall_escalation_rate": 0.65725, |
| "capture_ratio": 1.4481661908188095 |
| } |
| }, |
| "sbi": { |
| "rq1_calibration": { |
| "frame_ece": 0.11547506888157706, |
| "video_ece": 0.13027604551714325, |
| "frame_auroc": 0.9894253117772963, |
| "video_auroc": 0.9981015943641083, |
| "frame_reliability": { |
| "bin_accuracies": [ |
| 0.8203422053231939, |
| 0.1948529411764706, |
| 0.07853403141361257, |
| 0.03870967741935484, |
| 0.05228758169934641, |
| 0.973404255319149, |
| 0.976303317535545, |
| 0.9893992932862191, |
| 0.9911504424778761, |
| 0.9996946098641014 |
| ], |
| "bin_confidences": [ |
| 0.9901359978412765, |
| 0.8499580145101336, |
| 0.7507045780020859, |
| 0.6493786873355989, |
| 0.5502231080158084, |
| 0.5524401097221577, |
| 0.6561112276750718, |
| 0.7510433140154862, |
| 0.85728356730094, |
| 0.990656955576358 |
| ], |
| "bin_counts": [ |
| 3156, |
| 272, |
| 191, |
| 155, |
| 153, |
| 188, |
| 211, |
| 283, |
| 452, |
| 6549 |
| ], |
| "bin_boundaries": [ |
| 0.0, |
| 0.1, |
| 0.2, |
| 0.30000000000000004, |
| 0.4, |
| 0.5, |
| 0.6000000000000001, |
| 0.7000000000000001, |
| 0.8, |
| 0.9, |
| 1.0 |
| ] |
| }, |
| "video_reliability": { |
| "bin_accuracies": [ |
| 0.9290322580645162, |
| 0.2631578947368421, |
| 0.05555555555555555, |
| 0.0, |
| 0.0, |
| 1.0, |
| 1.0, |
| 1.0, |
| 1.0, |
| 1.0 |
| ], |
| "bin_confidences": [ |
| 0.9882727553806603, |
| 0.8421360934238213, |
| 0.7384568966313964, |
| 0.6322817423980108, |
| 0.5434941488096956, |
| 0.55045107472363, |
| 0.6569964145652193, |
| 0.7434793035583374, |
| 0.854589222789593, |
| 0.9773286883893305 |
| ], |
| "bin_counts": [ |
| 155, |
| 19, |
| 18, |
| 17, |
| 20, |
| 37, |
| 48, |
| 71, |
| 132, |
| 532 |
| ], |
| "bin_boundaries": [ |
| 0.0, |
| 0.1, |
| 0.2, |
| 0.30000000000000004, |
| 0.4, |
| 0.5, |
| 0.6000000000000001, |
| 0.7000000000000001, |
| 0.8, |
| 0.9, |
| 1.0 |
| ] |
| } |
| }, |
| "rq2_uncertainty_explanation_correlation": { |
| "n_samples": 3165, |
| "n_skipped_timeout_or_error": 0, |
| "spearman_rho": 0.010463770750667711, |
| "p_value": 0.5562250049960079 |
| }, |
| "rq3_triage_transferability": { |
| "n_samples": 3165, |
| "n_false_negatives": 179, |
| "false_negative_escalation_rate": 0.9608938547486033, |
| "overall_escalation_rate": 0.9330173775671406, |
| "capture_ratio": 1.0298777684657396 |
| } |
| }, |
| "vit": { |
| "rq1_calibration": { |
| "frame_ece": 0.07731910229013703, |
| "video_ece": 0.09058113050468003, |
| "frame_auroc": 0.9829844889966801, |
| "video_auroc": 0.9924212087504636, |
| "frame_reliability": { |
| "bin_accuracies": [ |
| 0.8682613352374152, |
| 0.36466165413533835, |
| 0.22560975609756098, |
| 0.20118343195266272, |
| 0.19135802469135801, |
| 0.9121621621621622, |
| 0.900990099009901, |
| 0.9351851851851852, |
| 0.9904458598726115, |
| 0.99873132224415 |
| ], |
| "bin_confidences": [ |
| 0.9844166620647967, |
| 0.8532709505380993, |
| 0.750156934032353, |
| 0.6524830922572571, |
| 0.5520629060489161, |
| 0.5504080693464022, |
| 0.6507839592966703, |
| 0.7540436188379923, |
| 0.8542215421700933, |
| 0.9930967474726512 |
| ], |
| "bin_counts": [ |
| 2801, |
| 266, |
| 164, |
| 169, |
| 162, |
| 148, |
| 202, |
| 216, |
| 314, |
| 7094 |
| ], |
| "bin_boundaries": [ |
| 0.0, |
| 0.1, |
| 0.2, |
| 0.30000000000000004, |
| 0.4, |
| 0.5, |
| 0.6000000000000001, |
| 0.7000000000000001, |
| 0.8, |
| 0.9, |
| 1.0 |
| ] |
| }, |
| "video_reliability": { |
| "bin_accuracies": [ |
| 0.927536231884058, |
| 0.5789473684210527, |
| 0.3125, |
| 0.2, |
| 0.041666666666666664, |
| 1.0, |
| 1.0, |
| 1.0, |
| 1.0, |
| 1.0 |
| ], |
| "bin_confidences": [ |
| 0.9797159383956952, |
| 0.8658041195733333, |
| 0.7462968135710981, |
| 0.641425583120901, |
| 0.5507501248813544, |
| 0.5517592517046356, |
| 0.6596009343343142, |
| 0.7582476830778178, |
| 0.8616442756492688, |
| 0.9851612873775182 |
| ], |
| "bin_counts": [ |
| 138, |
| 19, |
| 16, |
| 25, |
| 24, |
| 19, |
| 26, |
| 50, |
| 96, |
| 636 |
| ], |
| "bin_boundaries": [ |
| 0.0, |
| 0.1, |
| 0.2, |
| 0.30000000000000004, |
| 0.4, |
| 0.5, |
| 0.6000000000000001, |
| 0.7000000000000001, |
| 0.8, |
| 0.9, |
| 1.0 |
| ] |
| } |
| }, |
| "rq2_uncertainty_explanation_correlation": { |
| "n_samples": 4000, |
| "n_skipped_timeout_or_error": 0, |
| "spearman_rho": -0.05378002393888393, |
| "p_value": 0.000667120597725451 |
| }, |
| "rq3_triage_transferability": { |
| "n_samples": 4000, |
| "n_false_negatives": 215, |
| "false_negative_escalation_rate": 0.9302325581395349, |
| "overall_escalation_rate": 0.8495, |
| "capture_ratio": 1.0950353833308237 |
| } |
| }, |
| "lsda": { |
| "rq1_calibration": { |
| "frame_ece": 0.06321588433009974, |
| "video_ece": 0.0728054912142667, |
| "frame_auroc": 0.9929977668661246, |
| "video_auroc": 0.9984649610678531, |
| "frame_reliability": { |
| "bin_accuracies": [ |
| 0.9033523537803139, |
| 0.358974358974359, |
| 0.20634920634920634, |
| 0.2, |
| 0.10891089108910891, |
| 0.9057971014492754, |
| 0.9612903225806452, |
| 0.975, |
| 0.9826086956521739, |
| 0.9998658798283262 |
| ], |
| "bin_confidences": [ |
| 0.9915206229645029, |
| 0.8522127576745473, |
| 0.7492105944289101, |
| 0.6514855871075078, |
| 0.5517038422645909, |
| 0.5537072739739349, |
| 0.6505845727459076, |
| 0.7510491004586219, |
| 0.8561639751213185, |
| 0.9927714535398928 |
| ], |
| "bin_counts": [ |
| 2804, |
| 195, |
| 126, |
| 95, |
| 101, |
| 138, |
| 155, |
| 200, |
| 345, |
| 7456 |
| ], |
| "bin_boundaries": [ |
| 0.0, |
| 0.1, |
| 0.2, |
| 0.30000000000000004, |
| 0.4, |
| 0.5, |
| 0.6000000000000001, |
| 0.7000000000000001, |
| 0.8, |
| 0.9, |
| 1.0 |
| ] |
| }, |
| "video_reliability": { |
| "bin_accuracies": [ |
| 0.9459459459459459, |
| 0.6666666666666666, |
| 0.2, |
| 0.125, |
| 0.0, |
| 1.0, |
| 1.0, |
| 1.0, |
| 1.0, |
| 1.0 |
| ], |
| "bin_confidences": [ |
| 0.9833511600765061, |
| 0.8563631980825789, |
| 0.7432081695180386, |
| 0.6470798112364718, |
| 0.5438026031218071, |
| 0.5389168736881402, |
| 0.6474304313786188, |
| 0.7481252152760411, |
| 0.8615439276635047, |
| 0.9858788564813138 |
| ], |
| "bin_counts": [ |
| 148, |
| 12, |
| 5, |
| 8, |
| 19, |
| 16, |
| 26, |
| 54, |
| 84, |
| 677 |
| ], |
| "bin_boundaries": [ |
| 0.0, |
| 0.1, |
| 0.2, |
| 0.30000000000000004, |
| 0.4, |
| 0.5, |
| 0.6000000000000001, |
| 0.7000000000000001, |
| 0.8, |
| 0.9, |
| 1.0 |
| ] |
| } |
| }, |
| "rq2_uncertainty_explanation_correlation": { |
| "n_samples": 4000, |
| "n_skipped_timeout_or_error": 0, |
| "spearman_rho": -0.08215462739973001, |
| "p_value": 1.9600416835477606e-07 |
| }, |
| "rq3_triage_transferability": { |
| "n_samples": 4000, |
| "n_false_negatives": 152, |
| "false_negative_escalation_rate": 0.8881578947368421, |
| "overall_escalation_rate": 0.76025, |
| "capture_ratio": 1.168244517904429 |
| } |
| } |
| } |
| } |
