| { |
| "paper": "Personal VAD 2.0 (arXiv:2204.03793)", |
| "backbone": "conformer", |
| "conditioning_mode": "film_dvector_cos", |
| "loss_type": "cross_entropy", |
| "final_train_loss": 0.5947727542934995, |
| "artifacts": { |
| "weights_path": "/usr/local/google/home/quanw/Data/pvad_MLS_Watch_models/pvad2_e5_conformer_unified/model.weights.h5", |
| "safetensors_path": "/usr/local/google/home/quanw/Data/pvad_MLS_Watch_models/pvad2_e5_conformer_unified/model.safetensors", |
| "config_path": "/usr/local/google/home/quanw/Data/pvad_MLS_Watch_models/pvad2_e5_conformer_unified/model_config.json", |
| "fp32_tflite_path": "/usr/local/google/home/quanw/Data/pvad_MLS_Watch_models/pvad2_e5_conformer_unified/model_fp32.tflite", |
| "quant_tflite_path": "/usr/local/google/home/quanw/Data/pvad_MLS_Watch_models/pvad2_e5_conformer_unified/model_quantized.tflite", |
| "fp32_size_kb": 4391.39, |
| "quant_size_kb": 1367.07, |
| "safetensors_size_kb": 4278.78, |
| "num_parameters": 1091267 |
| }, |
| "evaluations": { |
| "personal_vad_non_concat": { |
| "eval_mode": "personal_vad", |
| "num_frames": 20160, |
| "overall_accuracy": 0.7103174603174603, |
| "class_accuracies": { |
| "Speech (tss)": 0.6577914798206278, |
| "Silence (ns)": 0.8770911573916013, |
| "Non-Target Speech (ntss)": 0.6262908177504884 |
| }, |
| "average_precisions": { |
| "Speech (tss)": 0.6892420164744152, |
| "Silence (ns)": 0.9357029409584907, |
| "Non-Target Speech (ntss)": 0.7268596677348595 |
| }, |
| "mean_average_precision_micro": 0.7919295853317531, |
| "roc_aucs": { |
| "Speech (tss)": 0.8259080360266248, |
| "Silence (ns)": 0.9699089350874723, |
| "Non-Target Speech (ntss)": 0.8210536349222516 |
| }, |
| "bootstrap_ci": { |
| "resample_unit": "utterance_mixture", |
| "num_clusters": 360, |
| "n_resamples": 1000, |
| "mAP_micro_95_ci": [ |
| 0.7586, |
| 0.8207 |
| ], |
| "tss_ap_95_ci": [ |
| 0.6147, |
| 0.7567 |
| ], |
| "ns_ap_95_ci": [ |
| 0.9263, |
| 0.9448 |
| ], |
| "ntss_ap_95_ci": [ |
| 0.6709, |
| 0.7816 |
| ] |
| } |
| }, |
| "personal_vad_concat": { |
| "eval_mode": "personal_vad", |
| "num_frames": 40320, |
| "overall_accuracy": 0.7016617063492063, |
| "class_accuracies": { |
| "Speech (tss)": 0.6557051423526815, |
| "Silence (ns)": 0.855155343120519, |
| "Non-Target Speech (ntss)": 0.6234761175138231 |
| }, |
| "average_precisions": { |
| "Speech (tss)": 0.6772764605640786, |
| "Silence (ns)": 0.9265294274976209, |
| "Non-Target Speech (ntss)": 0.7326544180781862 |
| }, |
| "mean_average_precision_micro": 0.7850382284324038, |
| "roc_aucs": { |
| "Speech (tss)": 0.8246058077862795, |
| "Silence (ns)": 0.9648233021866168, |
| "Non-Target Speech (ntss)": 0.8136358820553207 |
| }, |
| "bootstrap_ci": { |
| "resample_unit": "utterance_mixture", |
| "num_clusters": 343, |
| "n_resamples": 1000, |
| "mAP_micro_95_ci": [ |
| 0.7639, |
| 0.8038 |
| ], |
| "tss_ap_95_ci": [ |
| 0.6391, |
| 0.7201 |
| ], |
| "ns_ap_95_ci": [ |
| 0.9175, |
| 0.9337 |
| ], |
| "ntss_ap_95_ci": [ |
| 0.7045, |
| 0.7574 |
| ] |
| } |
| }, |
| "personal_vad_concat_mtr": { |
| "eval_mode": "personal_vad", |
| "num_frames": 40320, |
| "overall_accuracy": 0.5465277777777777, |
| "class_accuracies": { |
| "Speech (tss)": 0.47266975649231224, |
| "Silence (ns)": 0.7238818709457152, |
| "Non-Target Speech (ntss)": 0.47498501099193924 |
| }, |
| "average_precisions": { |
| "Speech (tss)": 0.5395266815820644, |
| "Silence (ns)": 0.7835283893492777, |
| "Non-Target Speech (ntss)": 0.550999424919393 |
| }, |
| "mean_average_precision_micro": 0.6202588749375713, |
| "roc_aucs": { |
| "Speech (tss)": 0.701350691302156, |
| "Silence (ns)": 0.8876932646134721, |
| "Non-Target Speech (ntss)": 0.6661654002363839 |
| }, |
| "bootstrap_ci": { |
| "resample_unit": "utterance_mixture", |
| "num_clusters": 343, |
| "n_resamples": 1000, |
| "mAP_micro_95_ci": [ |
| 0.5988, |
| 0.642 |
| ], |
| "tss_ap_95_ci": [ |
| 0.5011, |
| 0.5792 |
| ], |
| "ns_ap_95_ci": [ |
| 0.7673, |
| 0.7991 |
| ], |
| "ntss_ap_95_ci": [ |
| 0.5156, |
| 0.5825 |
| ] |
| } |
| }, |
| "personal_vad_non_concat_tflite_fp32": { |
| "eval_mode": "personal_vad", |
| "num_frames": 20160, |
| "overall_accuracy": 0.7103174603174603, |
| "class_accuracies": { |
| "Speech (tss)": 0.6577914798206278, |
| "Silence (ns)": 0.8770911573916013, |
| "Non-Target Speech (ntss)": 0.6262908177504884 |
| }, |
| "average_precisions": { |
| "Speech (tss)": 0.6892420387976984, |
| "Silence (ns)": 0.9357029490548648, |
| "Non-Target Speech (ntss)": 0.7268597643877056 |
| }, |
| "mean_average_precision_micro": 0.7919295941246614, |
| "roc_aucs": { |
| "Speech (tss)": 0.8259080844453428, |
| "Silence (ns)": 0.9699089470233363, |
| "Non-Target Speech (ntss)": 0.8210537100981061 |
| }, |
| "bootstrap_ci": { |
| "resample_unit": "utterance_mixture", |
| "num_clusters": 360, |
| "n_resamples": 200, |
| "mAP_micro_95_ci": [ |
| 0.7622, |
| 0.8228 |
| ], |
| "tss_ap_95_ci": [ |
| 0.6146, |
| 0.7625 |
| ], |
| "ns_ap_95_ci": [ |
| 0.9272, |
| 0.945 |
| ], |
| "ntss_ap_95_ci": [ |
| 0.6731, |
| 0.7836 |
| ] |
| } |
| }, |
| "personal_vad_non_concat_tflite_quant": { |
| "eval_mode": "personal_vad", |
| "num_frames": 20160, |
| "overall_accuracy": 0.7097718253968254, |
| "class_accuracies": { |
| "Speech (tss)": 0.6579316143497758, |
| "Silence (ns)": 0.8752133834073063, |
| "Non-Target Speech (ntss)": 0.6261512698855708 |
| }, |
| "average_precisions": { |
| "Speech (tss)": 0.6893415696135827, |
| "Silence (ns)": 0.9358276567867474, |
| "Non-Target Speech (ntss)": 0.7269565770766615 |
| }, |
| "mean_average_precision_micro": 0.7917433650517681, |
| "roc_aucs": { |
| "Speech (tss)": 0.8258096653315439, |
| "Silence (ns)": 0.9699485203801774, |
| "Non-Target Speech (ntss)": 0.8208741901573671 |
| }, |
| "bootstrap_ci": { |
| "resample_unit": "utterance_mixture", |
| "num_clusters": 360, |
| "n_resamples": 200, |
| "mAP_micro_95_ci": [ |
| 0.762, |
| 0.8225 |
| ], |
| "tss_ap_95_ci": [ |
| 0.6148, |
| 0.7628 |
| ], |
| "ns_ap_95_ci": [ |
| 0.9275, |
| 0.945 |
| ], |
| "ntss_ap_95_ci": [ |
| 0.6731, |
| 0.7822 |
| ] |
| } |
| }, |
| "personal_vad_concat_tflite_fp32": { |
| "eval_mode": "personal_vad", |
| "num_frames": 40320, |
| "overall_accuracy": 0.7016617063492063, |
| "class_accuracies": { |
| "Speech (tss)": 0.6557051423526815, |
| "Silence (ns)": 0.855155343120519, |
| "Non-Target Speech (ntss)": 0.6234761175138231 |
| }, |
| "average_precisions": { |
| "Speech (tss)": 0.6772765009353281, |
| "Silence (ns)": 0.9265294367475656, |
| "Non-Target Speech (ntss)": 0.7326544699148918 |
| }, |
| "mean_average_precision_micro": 0.7850381579670648, |
| "roc_aucs": { |
| "Speech (tss)": 0.8246058160439154, |
| "Silence (ns)": 0.9648233066625658, |
| "Non-Target Speech (ntss)": 0.8136359123253771 |
| }, |
| "bootstrap_ci": { |
| "resample_unit": "utterance_mixture", |
| "num_clusters": 343, |
| "n_resamples": 200, |
| "mAP_micro_95_ci": [ |
| 0.7633, |
| 0.8064 |
| ], |
| "tss_ap_95_ci": [ |
| 0.6391, |
| 0.7167 |
| ], |
| "ns_ap_95_ci": [ |
| 0.9179, |
| 0.9337 |
| ], |
| "ntss_ap_95_ci": [ |
| 0.7064, |
| 0.758 |
| ] |
| } |
| }, |
| "personal_vad_concat_tflite_quant": { |
| "eval_mode": "personal_vad", |
| "num_frames": 40320, |
| "overall_accuracy": 0.7013392857142857, |
| "class_accuracies": { |
| "Speech (tss)": 0.6555580077981313, |
| "Silence (ns)": 0.8547285763059065, |
| "Non-Target Speech (ntss)": 0.6230764106322031 |
| }, |
| "average_precisions": { |
| "Speech (tss)": 0.6775659071400901, |
| "Silence (ns)": 0.9265073834296618, |
| "Non-Target Speech (ntss)": 0.7323164660648535 |
| }, |
| "mean_average_precision_micro": 0.7848514382066165, |
| "roc_aucs": { |
| "Speech (tss)": 0.8243797976709123, |
| "Silence (ns)": 0.9647974043458671, |
| "Non-Target Speech (ntss)": 0.8134056611675335 |
| }, |
| "bootstrap_ci": { |
| "resample_unit": "utterance_mixture", |
| "num_clusters": 343, |
| "n_resamples": 200, |
| "mAP_micro_95_ci": [ |
| 0.7631, |
| 0.806 |
| ], |
| "tss_ap_95_ci": [ |
| 0.6394, |
| 0.7168 |
| ], |
| "ns_ap_95_ci": [ |
| 0.9179, |
| 0.9337 |
| ], |
| "ntss_ap_95_ci": [ |
| 0.7055, |
| 0.758 |
| ] |
| } |
| }, |
| "per_language_concat": { |
| "de": { |
| "num_speakers": 5, |
| "num_eval_mixtures": 42, |
| "num_frames": 5040, |
| "average_precisions": { |
| "Speech (tss)": 0.5843, |
| "Silence (ns)": 0.8297, |
| "Non-Target Speech (ntss)": 0.7236 |
| }, |
| "mean_average_precision_micro": 0.6937 |
| }, |
| "en": { |
| "num_speakers": 6, |
| "num_eval_mixtures": 54, |
| "num_frames": 6048, |
| "average_precisions": { |
| "Speech (tss)": 0.6705, |
| "Silence (ns)": 0.9365, |
| "Non-Target Speech (ntss)": 0.697 |
| }, |
| "mean_average_precision_micro": 0.7707 |
| }, |
| "es": { |
| "num_speakers": 5, |
| "num_eval_mixtures": 42, |
| "num_frames": 5040, |
| "average_precisions": { |
| "Speech (tss)": 0.6926, |
| "Silence (ns)": 0.9542, |
| "Non-Target Speech (ntss)": 0.6632 |
| }, |
| "mean_average_precision_micro": 0.7766 |
| }, |
| "fr": { |
| "num_speakers": 5, |
| "num_eval_mixtures": 42, |
| "num_frames": 5040, |
| "average_precisions": { |
| "Speech (tss)": 0.5237, |
| "Silence (ns)": 0.9294, |
| "Non-Target Speech (ntss)": 0.6177 |
| }, |
| "mean_average_precision_micro": 0.6825 |
| }, |
| "it": { |
| "num_speakers": 5, |
| "num_eval_mixtures": 42, |
| "num_frames": 5040, |
| "average_precisions": { |
| "Speech (tss)": 0.7897, |
| "Silence (ns)": 0.962, |
| "Non-Target Speech (ntss)": 0.8087 |
| }, |
| "mean_average_precision_micro": 0.8498 |
| }, |
| "nl": { |
| "num_speakers": 5, |
| "num_eval_mixtures": 42, |
| "num_frames": 5040, |
| "average_precisions": { |
| "Speech (tss)": 0.7495, |
| "Silence (ns)": 0.9472, |
| "Non-Target Speech (ntss)": 0.7659 |
| }, |
| "mean_average_precision_micro": 0.8147 |
| }, |
| "pl": { |
| "num_speakers": 4, |
| "num_eval_mixtures": 37, |
| "num_frames": 4032, |
| "average_precisions": { |
| "Speech (tss)": 0.7613, |
| "Silence (ns)": 0.9413, |
| "Non-Target Speech (ntss)": 0.688 |
| }, |
| "mean_average_precision_micro": 0.7806 |
| }, |
| "pt": { |
| "num_speakers": 5, |
| "num_eval_mixtures": 42, |
| "num_frames": 5040, |
| "average_precisions": { |
| "Speech (tss)": 0.8458, |
| "Silence (ns)": 0.9451, |
| "Non-Target Speech (ntss)": 0.8644 |
| }, |
| "mean_average_precision_micro": 0.8723 |
| } |
| }, |
| "no_enrollment_non_concat": { |
| "eval_mode": "personal_vad_std_eval", |
| "num_frames": 20160, |
| "overall_accuracy": 0.915327380952381, |
| "class_accuracies": { |
| "Speech": 0.9136484407775136, |
| "Silence": 0.9194264254011608 |
| }, |
| "average_precisions": { |
| "Speech": 0.9872015134035821, |
| "Silence": 0.9437945725594598 |
| }, |
| "mean_average_precision_micro": 0.9667837244571811, |
| "roc_aucs": { |
| "Speech": 0.9721312198801457, |
| "Silence": 0.974523757835835 |
| }, |
| "bootstrap_ci": { |
| "resample_unit": "utterance_mixture", |
| "num_clusters": 360, |
| "n_resamples": 1000, |
| "mAP_micro_95_ci": [ |
| 0.9607, |
| 0.9723 |
| ] |
| } |
| }, |
| "no_enrollment_concat": { |
| "eval_mode": "personal_vad_std_eval", |
| "num_frames": 40320, |
| "overall_accuracy": 0.9083333333333333, |
| "class_accuracies": { |
| "Speech": 0.9118654733603692, |
| "Silence": 0.8997097985660635 |
| }, |
| "average_precisions": { |
| "Speech": 0.9850832771369401, |
| "Silence": 0.9365338947237541 |
| }, |
| "mean_average_precision_micro": 0.9613701610833925, |
| "roc_aucs": { |
| "Speech": 0.9679380330168913, |
| "Silence": 0.9704484107731388 |
| }, |
| "bootstrap_ci": { |
| "resample_unit": "utterance_mixture", |
| "num_clusters": 355, |
| "n_resamples": 1000, |
| "mAP_micro_95_ci": [ |
| 0.9566, |
| 0.9658 |
| ] |
| } |
| }, |
| "no_enrollment_concat_tflite_quant": { |
| "eval_mode": "personal_vad_std_eval", |
| "num_frames": 40320, |
| "overall_accuracy": 0.908531746031746, |
| "class_accuracies": { |
| "Speech": 0.9120752342329744, |
| "Silence": 0.8998805052919086 |
| }, |
| "average_precisions": { |
| "Speech": 0.9850984184356474, |
| "Silence": 0.9365649290677195 |
| }, |
| "mean_average_precision_micro": 0.9616643373133097, |
| "roc_aucs": { |
| "Speech": 0.9679613452511184, |
| "Silence": 0.970470941208279 |
| }, |
| "bootstrap_ci": { |
| "resample_unit": "utterance_mixture", |
| "num_clusters": 355, |
| "n_resamples": 200, |
| "mAP_micro_95_ci": [ |
| 0.9569, |
| 0.9658 |
| ] |
| } |
| } |
| } |
| } |
