diff --git "a/evaluation/final/modern400/summary.json" "b/evaluation/final/modern400/summary.json" new file mode 100644--- /dev/null +++ "b/evaluation/final/modern400/summary.json" @@ -0,0 +1,3779 @@ +{ + "format": "inflect_v2_modern400_dual_asr_v1", + "prompt_count_per_system": 400, + "total_audio_clips": 3200, + "asr_models": [ + "Qwen/Qwen3-ASR-1.7B", + "nvidia/nemotron-3.5-asr-streaming-0.6b" + ], + "systems": { + "inflect-micro-v2": { + "label": "Inflect-Micro-v2", + "qwen3_asr": { + "raw": { + "utterances": 400, + "errors": 159, + "words": 6794, + "wer": 0.023403002649396525 + }, + "raw_95ci": [ + 0.017408416590977898, + 0.030176415970287838 + ], + "semantic": { + "utterances": 400, + "errors": 172, + "words": 6823, + "wer": 0.02520885241096292 + }, + "semantic_95ci": [ + 0.018783412801618264, + 0.03235815602836879 + ], + "categories": { + "conversational": { + "raw": { + "utterances": 25, + "errors": 0, + "words": 296, + "wer": 0.0 + }, + "semantic": { + "utterances": 25, + "errors": 0, + "words": 309, + "wer": 0.0 + } + }, + "descriptive": { + "raw": { + "utterances": 16, + "errors": 0, + "words": 217, + "wer": 0.0 + }, + "semantic": { + "utterances": 16, + "errors": 0, + "words": 226, + "wer": 0.0 + } + }, + "emotional": { + "raw": { + "utterances": 24, + "errors": 0, + "words": 314, + "wer": 0.0 + }, + "semantic": { + "utterances": 24, + "errors": 0, + "words": 329, + "wer": 0.0 + } + }, + "external_heldout": { + "raw": { + "utterances": 200, + "errors": 114, + "words": 4250, + "wer": 0.026823529411764704 + }, + "semantic": { + "utterances": 200, + "errors": 148, + "words": 4215, + "wer": 0.035112692763938316 + } + }, + "homographs": { + "raw": { + "utterances": 6, + "errors": 0, + "words": 70, + "wer": 0.0 + }, + "semantic": { + "utterances": 6, + "errors": 0, + "words": 70, + "wer": 0.0 + } + }, + "long": { + "raw": { + "utterances": 15, + "errors": 1, + "words": 412, + "wer": 0.0024271844660194173 + }, + "semantic": { + "utterances": 15, + "errors": 1, + "words": 432, + "wer": 0.0023148148148148147 + } + }, + "long_range": { + "raw": { + "utterances": 6, + "errors": 0, + "words": 121, + "wer": 0.0 + }, + "semantic": { + "utterances": 6, + "errors": 0, + "words": 121, + "wer": 0.0 + } + }, + "names_places": { + "raw": { + "utterances": 6, + "errors": 6, + "words": 57, + "wer": 0.10526315789473684 + }, + "semantic": { + "utterances": 6, + "errors": 6, + "words": 57, + "wer": 0.10526315789473684 + } + }, + "narrative": { + "raw": { + "utterances": 24, + "errors": 0, + "words": 342, + "wer": 0.0 + }, + "semantic": { + "utterances": 24, + "errors": 0, + "words": 352, + "wer": 0.0 + } + }, + "numbers": { + "raw": { + "utterances": 6, + "errors": 21, + "words": 81, + "wer": 0.25925925925925924 + }, + "semantic": { + "utterances": 6, + "errors": 4, + "words": 64, + "wer": 0.0625 + } + }, + "punctuation": { + "raw": { + "utterances": 6, + "errors": 0, + "words": 65, + "wer": 0.0 + }, + "semantic": { + "utterances": 6, + "errors": 0, + "words": 65, + "wer": 0.0 + } + }, + "question": { + "raw": { + "utterances": 15, + "errors": 0, + "words": 163, + "wer": 0.0 + }, + "semantic": { + "utterances": 15, + "errors": 0, + "words": 165, + "wer": 0.0 + } + }, + "rare_words": { + "raw": { + "utterances": 6, + "errors": 3, + "words": 57, + "wer": 0.05263157894736842 + }, + "semantic": { + "utterances": 6, + "errors": 3, + "words": 57, + "wer": 0.05263157894736842 + } + }, + "short": { + "raw": { + "utterances": 24, + "errors": 3, + "words": 98, + "wer": 0.030612244897959183 + }, + "semantic": { + "utterances": 24, + "errors": 3, + "words": 105, + "wer": 0.02857142857142857 + } + }, + "technical": { + "raw": { + "utterances": 21, + "errors": 11, + "words": 251, + "wer": 0.043824701195219126 + }, + "semantic": { + "utterances": 21, + "errors": 7, + "words": 256, + "wer": 0.02734375 + } + } + } + }, + "nemotron35_asr": { + "raw": { + "utterances": 400, + "errors": 429, + "words": 6794, + "wer": 0.06314395054459818 + }, + "raw_95ci": [ + 0.052087944518223406, + 0.07540394973070018 + ], + "semantic": { + "utterances": 400, + "errors": 372, + "words": 6823, + "wer": 0.05452147149347794 + }, + "semantic_95ci": [ + 0.04496430198997418, + 0.0649688901750832 + ], + "categories": { + "conversational": { + "raw": { + "utterances": 25, + "errors": 3, + "words": 296, + "wer": 0.010135135135135136 + }, + "semantic": { + "utterances": 25, + "errors": 3, + "words": 309, + "wer": 0.009708737864077669 + } + }, + "descriptive": { + "raw": { + "utterances": 16, + "errors": 2, + "words": 217, + "wer": 0.009216589861751152 + }, + "semantic": { + "utterances": 16, + "errors": 2, + "words": 226, + "wer": 0.008849557522123894 + } + }, + "emotional": { + "raw": { + "utterances": 24, + "errors": 3, + "words": 314, + "wer": 0.009554140127388535 + }, + "semantic": { + "utterances": 24, + "errors": 1, + "words": 329, + "wer": 0.00303951367781155 + } + }, + "external_heldout": { + "raw": { + "utterances": 200, + "errors": 359, + "words": 4250, + "wer": 0.08447058823529412 + }, + "semantic": { + "utterances": 200, + "errors": 306, + "words": 4215, + "wer": 0.07259786476868327 + } + }, + "homographs": { + "raw": { + "utterances": 6, + "errors": 5, + "words": 70, + "wer": 0.07142857142857142 + }, + "semantic": { + "utterances": 6, + "errors": 5, + "words": 70, + "wer": 0.07142857142857142 + } + }, + "long": { + "raw": { + "utterances": 15, + "errors": 7, + "words": 412, + "wer": 0.01699029126213592 + }, + "semantic": { + "utterances": 15, + "errors": 6, + "words": 432, + "wer": 0.013888888888888888 + } + }, + "long_range": { + "raw": { + "utterances": 6, + "errors": 2, + "words": 121, + "wer": 0.01652892561983471 + }, + "semantic": { + "utterances": 6, + "errors": 2, + "words": 121, + "wer": 0.01652892561983471 + } + }, + "names_places": { + "raw": { + "utterances": 6, + "errors": 13, + "words": 57, + "wer": 0.22807017543859648 + }, + "semantic": { + "utterances": 6, + "errors": 13, + "words": 57, + "wer": 0.22807017543859648 + } + }, + "narrative": { + "raw": { + "utterances": 24, + "errors": 4, + "words": 342, + "wer": 0.011695906432748537 + }, + "semantic": { + "utterances": 24, + "errors": 4, + "words": 352, + "wer": 0.011363636363636364 + } + }, + "numbers": { + "raw": { + "utterances": 6, + "errors": 5, + "words": 81, + "wer": 0.06172839506172839 + }, + "semantic": { + "utterances": 6, + "errors": 4, + "words": 64, + "wer": 0.0625 + } + }, + "punctuation": { + "raw": { + "utterances": 6, + "errors": 0, + "words": 65, + "wer": 0.0 + }, + "semantic": { + "utterances": 6, + "errors": 0, + "words": 65, + "wer": 0.0 + } + }, + "question": { + "raw": { + "utterances": 15, + "errors": 0, + "words": 163, + "wer": 0.0 + }, + "semantic": { + "utterances": 15, + "errors": 0, + "words": 165, + "wer": 0.0 + } + }, + "rare_words": { + "raw": { + "utterances": 6, + "errors": 6, + "words": 57, + "wer": 0.10526315789473684 + }, + "semantic": { + "utterances": 6, + "errors": 6, + "words": 57, + "wer": 0.10526315789473684 + } + }, + "short": { + "raw": { + "utterances": 24, + "errors": 6, + "words": 98, + "wer": 0.061224489795918366 + }, + "semantic": { + "utterances": 24, + "errors": 6, + "words": 105, + "wer": 0.05714285714285714 + } + }, + "technical": { + "raw": { + "utterances": 21, + "errors": 14, + "words": 251, + "wer": 0.055776892430278883 + }, + "semantic": { + "utterances": 21, + "errors": 14, + "words": 256, + "wer": 0.0546875 + } + } + } + } + }, + "inflect-nano-v2": { + "label": "Inflect-Nano-v2", + "qwen3_asr": { + "raw": { + "utterances": 400, + "errors": 179, + "words": 6794, + "wer": 0.026346776567559613 + }, + "raw_95ci": [ + 0.020206362854686157, + 0.033170442766178 + ], + "semantic": { + "utterances": 400, + "errors": 190, + "words": 6823, + "wer": 0.02784698812838927 + }, + "semantic_95ci": [ + 0.021251648834823392, + 0.0351635682374982 + ], + "categories": { + "conversational": { + "raw": { + "utterances": 25, + "errors": 3, + "words": 296, + "wer": 0.010135135135135136 + }, + "semantic": { + "utterances": 25, + "errors": 1, + "words": 309, + "wer": 0.003236245954692557 + } + }, + "descriptive": { + "raw": { + "utterances": 16, + "errors": 1, + "words": 217, + "wer": 0.004608294930875576 + }, + "semantic": { + "utterances": 16, + "errors": 1, + "words": 226, + "wer": 0.004424778761061947 + } + }, + "emotional": { + "raw": { + "utterances": 24, + "errors": 0, + "words": 314, + "wer": 0.0 + }, + "semantic": { + "utterances": 24, + "errors": 0, + "words": 329, + "wer": 0.0 + } + }, + "external_heldout": { + "raw": { + "utterances": 200, + "errors": 123, + "words": 4250, + "wer": 0.028941176470588234 + }, + "semantic": { + "utterances": 200, + "errors": 159, + "words": 4215, + "wer": 0.03772241992882562 + } + }, + "homographs": { + "raw": { + "utterances": 6, + "errors": 0, + "words": 70, + "wer": 0.0 + }, + "semantic": { + "utterances": 6, + "errors": 0, + "words": 70, + "wer": 0.0 + } + }, + "long": { + "raw": { + "utterances": 15, + "errors": 2, + "words": 412, + "wer": 0.0048543689320388345 + }, + "semantic": { + "utterances": 15, + "errors": 2, + "words": 432, + "wer": 0.004629629629629629 + } + }, + "long_range": { + "raw": { + "utterances": 6, + "errors": 0, + "words": 121, + "wer": 0.0 + }, + "semantic": { + "utterances": 6, + "errors": 0, + "words": 121, + "wer": 0.0 + } + }, + "names_places": { + "raw": { + "utterances": 6, + "errors": 10, + "words": 57, + "wer": 0.17543859649122806 + }, + "semantic": { + "utterances": 6, + "errors": 8, + "words": 57, + "wer": 0.14035087719298245 + } + }, + "narrative": { + "raw": { + "utterances": 24, + "errors": 1, + "words": 342, + "wer": 0.0029239766081871343 + }, + "semantic": { + "utterances": 24, + "errors": 1, + "words": 352, + "wer": 0.002840909090909091 + } + }, + "numbers": { + "raw": { + "utterances": 6, + "errors": 20, + "words": 81, + "wer": 0.24691358024691357 + }, + "semantic": { + "utterances": 6, + "errors": 3, + "words": 64, + "wer": 0.046875 + } + }, + "punctuation": { + "raw": { + "utterances": 6, + "errors": 0, + "words": 65, + "wer": 0.0 + }, + "semantic": { + "utterances": 6, + "errors": 0, + "words": 65, + "wer": 0.0 + } + }, + "question": { + "raw": { + "utterances": 15, + "errors": 0, + "words": 163, + "wer": 0.0 + }, + "semantic": { + "utterances": 15, + "errors": 0, + "words": 165, + "wer": 0.0 + } + }, + "rare_words": { + "raw": { + "utterances": 6, + "errors": 3, + "words": 57, + "wer": 0.05263157894736842 + }, + "semantic": { + "utterances": 6, + "errors": 3, + "words": 57, + "wer": 0.05263157894736842 + } + }, + "short": { + "raw": { + "utterances": 24, + "errors": 6, + "words": 98, + "wer": 0.061224489795918366 + }, + "semantic": { + "utterances": 24, + "errors": 6, + "words": 105, + "wer": 0.05714285714285714 + } + }, + "technical": { + "raw": { + "utterances": 21, + "errors": 10, + "words": 251, + "wer": 0.0398406374501992 + }, + "semantic": { + "utterances": 21, + "errors": 6, + "words": 256, + "wer": 0.0234375 + } + } + } + }, + "nemotron35_asr": { + "raw": { + "utterances": 400, + "errors": 461, + "words": 6794, + "wer": 0.06785398881365912 + }, + "raw_95ci": [ + 0.05698005698005698, + 0.07978646659933632 + ], + "semantic": { + "utterances": 400, + "errors": 384, + "words": 6823, + "wer": 0.056280228638428846 + }, + "semantic_95ci": [ + 0.04723173515981735, + 0.06624651912648395 + ], + "categories": { + "conversational": { + "raw": { + "utterances": 25, + "errors": 8, + "words": 296, + "wer": 0.02702702702702703 + }, + "semantic": { + "utterances": 25, + "errors": 6, + "words": 309, + "wer": 0.019417475728155338 + } + }, + "descriptive": { + "raw": { + "utterances": 16, + "errors": 3, + "words": 217, + "wer": 0.013824884792626729 + }, + "semantic": { + "utterances": 16, + "errors": 3, + "words": 226, + "wer": 0.01327433628318584 + } + }, + "emotional": { + "raw": { + "utterances": 24, + "errors": 2, + "words": 314, + "wer": 0.006369426751592357 + }, + "semantic": { + "utterances": 24, + "errors": 2, + "words": 329, + "wer": 0.0060790273556231 + } + }, + "external_heldout": { + "raw": { + "utterances": 200, + "errors": 377, + "words": 4250, + "wer": 0.08870588235294118 + }, + "semantic": { + "utterances": 200, + "errors": 303, + "words": 4215, + "wer": 0.07188612099644127 + } + }, + "homographs": { + "raw": { + "utterances": 6, + "errors": 8, + "words": 70, + "wer": 0.11428571428571428 + }, + "semantic": { + "utterances": 6, + "errors": 8, + "words": 70, + "wer": 0.11428571428571428 + } + }, + "long": { + "raw": { + "utterances": 15, + "errors": 3, + "words": 412, + "wer": 0.007281553398058253 + }, + "semantic": { + "utterances": 15, + "errors": 3, + "words": 432, + "wer": 0.006944444444444444 + } + }, + "long_range": { + "raw": { + "utterances": 6, + "errors": 3, + "words": 121, + "wer": 0.024793388429752067 + }, + "semantic": { + "utterances": 6, + "errors": 3, + "words": 121, + "wer": 0.024793388429752067 + } + }, + "names_places": { + "raw": { + "utterances": 6, + "errors": 12, + "words": 57, + "wer": 0.21052631578947367 + }, + "semantic": { + "utterances": 6, + "errors": 12, + "words": 57, + "wer": 0.21052631578947367 + } + }, + "narrative": { + "raw": { + "utterances": 24, + "errors": 10, + "words": 342, + "wer": 0.029239766081871343 + }, + "semantic": { + "utterances": 24, + "errors": 10, + "words": 352, + "wer": 0.028409090909090908 + } + }, + "numbers": { + "raw": { + "utterances": 6, + "errors": 4, + "words": 81, + "wer": 0.04938271604938271 + }, + "semantic": { + "utterances": 6, + "errors": 3, + "words": 64, + "wer": 0.046875 + } + }, + "punctuation": { + "raw": { + "utterances": 6, + "errors": 2, + "words": 65, + "wer": 0.03076923076923077 + }, + "semantic": { + "utterances": 6, + "errors": 2, + "words": 65, + "wer": 0.03076923076923077 + } + }, + "question": { + "raw": { + "utterances": 15, + "errors": 0, + "words": 163, + "wer": 0.0 + }, + "semantic": { + "utterances": 15, + "errors": 0, + "words": 165, + "wer": 0.0 + } + }, + "rare_words": { + "raw": { + "utterances": 6, + "errors": 6, + "words": 57, + "wer": 0.10526315789473684 + }, + "semantic": { + "utterances": 6, + "errors": 6, + "words": 57, + "wer": 0.10526315789473684 + } + }, + "short": { + "raw": { + "utterances": 24, + "errors": 13, + "words": 98, + "wer": 0.1326530612244898 + }, + "semantic": { + "utterances": 24, + "errors": 13, + "words": 105, + "wer": 0.12380952380952381 + } + }, + "technical": { + "raw": { + "utterances": 21, + "errors": 10, + "words": 251, + "wer": 0.0398406374501992 + }, + "semantic": { + "utterances": 21, + "errors": 10, + "words": 256, + "wer": 0.0390625 + } + } + } + } + }, + "kitten-nano-bruno": { + "label": "KittenTTS Nano - Bruno", + "qwen3_asr": { + "raw": { + "utterances": 400, + "errors": 139, + "words": 6794, + "wer": 0.02045922873123344 + }, + "raw_95ci": [ + 0.014941302027748132, + 0.026771187694128087 + ], + "semantic": { + "utterances": 400, + "errors": 147, + "words": 6823, + "wer": 0.02154477502564854 + }, + "semantic_95ci": [ + 0.015435835351089588, + 0.0284862043251305 + ], + "categories": { + "conversational": { + "raw": { + "utterances": 25, + "errors": 1, + "words": 296, + "wer": 0.0033783783783783786 + }, + "semantic": { + "utterances": 25, + "errors": 1, + "words": 309, + "wer": 0.003236245954692557 + } + }, + "descriptive": { + "raw": { + "utterances": 16, + "errors": 0, + "words": 217, + "wer": 0.0 + }, + "semantic": { + "utterances": 16, + "errors": 0, + "words": 226, + "wer": 0.0 + } + }, + "emotional": { + "raw": { + "utterances": 24, + "errors": 0, + "words": 314, + "wer": 0.0 + }, + "semantic": { + "utterances": 24, + "errors": 0, + "words": 329, + "wer": 0.0 + } + }, + "external_heldout": { + "raw": { + "utterances": 200, + "errors": 94, + "words": 4250, + "wer": 0.02211764705882353 + }, + "semantic": { + "utterances": 200, + "errors": 125, + "words": 4215, + "wer": 0.02965599051008304 + } + }, + "homographs": { + "raw": { + "utterances": 6, + "errors": 1, + "words": 70, + "wer": 0.014285714285714285 + }, + "semantic": { + "utterances": 6, + "errors": 1, + "words": 70, + "wer": 0.014285714285714285 + } + }, + "long": { + "raw": { + "utterances": 15, + "errors": 1, + "words": 412, + "wer": 0.0024271844660194173 + }, + "semantic": { + "utterances": 15, + "errors": 1, + "words": 432, + "wer": 0.0023148148148148147 + } + }, + "long_range": { + "raw": { + "utterances": 6, + "errors": 0, + "words": 121, + "wer": 0.0 + }, + "semantic": { + "utterances": 6, + "errors": 0, + "words": 121, + "wer": 0.0 + } + }, + "names_places": { + "raw": { + "utterances": 6, + "errors": 8, + "words": 57, + "wer": 0.14035087719298245 + }, + "semantic": { + "utterances": 6, + "errors": 6, + "words": 57, + "wer": 0.10526315789473684 + } + }, + "narrative": { + "raw": { + "utterances": 24, + "errors": 2, + "words": 342, + "wer": 0.005847953216374269 + }, + "semantic": { + "utterances": 24, + "errors": 2, + "words": 352, + "wer": 0.005681818181818182 + } + }, + "numbers": { + "raw": { + "utterances": 6, + "errors": 19, + "words": 81, + "wer": 0.2345679012345679 + }, + "semantic": { + "utterances": 6, + "errors": 2, + "words": 64, + "wer": 0.03125 + } + }, + "punctuation": { + "raw": { + "utterances": 6, + "errors": 0, + "words": 65, + "wer": 0.0 + }, + "semantic": { + "utterances": 6, + "errors": 0, + "words": 65, + "wer": 0.0 + } + }, + "question": { + "raw": { + "utterances": 15, + "errors": 0, + "words": 163, + "wer": 0.0 + }, + "semantic": { + "utterances": 15, + "errors": 0, + "words": 165, + "wer": 0.0 + } + }, + "rare_words": { + "raw": { + "utterances": 6, + "errors": 3, + "words": 57, + "wer": 0.05263157894736842 + }, + "semantic": { + "utterances": 6, + "errors": 3, + "words": 57, + "wer": 0.05263157894736842 + } + }, + "short": { + "raw": { + "utterances": 24, + "errors": 1, + "words": 98, + "wer": 0.01020408163265306 + }, + "semantic": { + "utterances": 24, + "errors": 1, + "words": 105, + "wer": 0.009523809523809525 + } + }, + "technical": { + "raw": { + "utterances": 21, + "errors": 9, + "words": 251, + "wer": 0.035856573705179286 + }, + "semantic": { + "utterances": 21, + "errors": 5, + "words": 256, + "wer": 0.01953125 + } + } + } + }, + "nemotron35_asr": { + "raw": { + "utterances": 400, + "errors": 300, + "words": 6794, + "wer": 0.044156608772446274 + }, + "raw_95ci": [ + 0.03619243784022363, + 0.052726998035957096 + ], + "semantic": { + "utterances": 400, + "errors": 270, + "words": 6823, + "wer": 0.03957203576139528 + }, + "semantic_95ci": [ + 0.03191024333381903, + 0.04806156526789604 + ], + "categories": { + "conversational": { + "raw": { + "utterances": 25, + "errors": 2, + "words": 296, + "wer": 0.006756756756756757 + }, + "semantic": { + "utterances": 25, + "errors": 2, + "words": 309, + "wer": 0.006472491909385114 + } + }, + "descriptive": { + "raw": { + "utterances": 16, + "errors": 1, + "words": 217, + "wer": 0.004608294930875576 + }, + "semantic": { + "utterances": 16, + "errors": 1, + "words": 226, + "wer": 0.004424778761061947 + } + }, + "emotional": { + "raw": { + "utterances": 24, + "errors": 7, + "words": 314, + "wer": 0.022292993630573247 + }, + "semantic": { + "utterances": 24, + "errors": 7, + "words": 329, + "wer": 0.02127659574468085 + } + }, + "external_heldout": { + "raw": { + "utterances": 200, + "errors": 243, + "words": 4250, + "wer": 0.057176470588235294 + }, + "semantic": { + "utterances": 200, + "errors": 214, + "words": 4215, + "wer": 0.050771055753262156 + } + }, + "homographs": { + "raw": { + "utterances": 6, + "errors": 3, + "words": 70, + "wer": 0.04285714285714286 + }, + "semantic": { + "utterances": 6, + "errors": 3, + "words": 70, + "wer": 0.04285714285714286 + } + }, + "long": { + "raw": { + "utterances": 15, + "errors": 2, + "words": 412, + "wer": 0.0048543689320388345 + }, + "semantic": { + "utterances": 15, + "errors": 2, + "words": 432, + "wer": 0.004629629629629629 + } + }, + "long_range": { + "raw": { + "utterances": 6, + "errors": 1, + "words": 121, + "wer": 0.008264462809917356 + }, + "semantic": { + "utterances": 6, + "errors": 1, + "words": 121, + "wer": 0.008264462809917356 + } + }, + "names_places": { + "raw": { + "utterances": 6, + "errors": 7, + "words": 57, + "wer": 0.12280701754385964 + }, + "semantic": { + "utterances": 6, + "errors": 7, + "words": 57, + "wer": 0.12280701754385964 + } + }, + "narrative": { + "raw": { + "utterances": 24, + "errors": 6, + "words": 342, + "wer": 0.017543859649122806 + }, + "semantic": { + "utterances": 24, + "errors": 6, + "words": 352, + "wer": 0.017045454545454544 + } + }, + "numbers": { + "raw": { + "utterances": 6, + "errors": 4, + "words": 81, + "wer": 0.04938271604938271 + }, + "semantic": { + "utterances": 6, + "errors": 5, + "words": 64, + "wer": 0.078125 + } + }, + "punctuation": { + "raw": { + "utterances": 6, + "errors": 2, + "words": 65, + "wer": 0.03076923076923077 + }, + "semantic": { + "utterances": 6, + "errors": 2, + "words": 65, + "wer": 0.03076923076923077 + } + }, + "question": { + "raw": { + "utterances": 15, + "errors": 0, + "words": 163, + "wer": 0.0 + }, + "semantic": { + "utterances": 15, + "errors": 0, + "words": 165, + "wer": 0.0 + } + }, + "rare_words": { + "raw": { + "utterances": 6, + "errors": 4, + "words": 57, + "wer": 0.07017543859649122 + }, + "semantic": { + "utterances": 6, + "errors": 4, + "words": 57, + "wer": 0.07017543859649122 + } + }, + "short": { + "raw": { + "utterances": 24, + "errors": 10, + "words": 98, + "wer": 0.10204081632653061 + }, + "semantic": { + "utterances": 24, + "errors": 10, + "words": 105, + "wer": 0.09523809523809523 + } + }, + "technical": { + "raw": { + "utterances": 21, + "errors": 8, + "words": 251, + "wer": 0.03187250996015936 + }, + "semantic": { + "utterances": 21, + "errors": 6, + "words": 256, + "wer": 0.0234375 + } + } + } + } + }, + "kitten-nano-hugo": { + "label": "KittenTTS Nano - Hugo", + "qwen3_asr": { + "raw": { + "utterances": 400, + "errors": 150, + "words": 6794, + "wer": 0.022078304386223137 + }, + "raw_95ci": [ + 0.016605437178545186, + 0.028407418310273772 + ], + "semantic": { + "utterances": 400, + "errors": 163, + "words": 6823, + "wer": 0.023889784552249743 + }, + "semantic_95ci": [ + 0.0173235166738848, + 0.03107674350744091 + ], + "categories": { + "conversational": { + "raw": { + "utterances": 25, + "errors": 2, + "words": 296, + "wer": 0.006756756756756757 + }, + "semantic": { + "utterances": 25, + "errors": 2, + "words": 309, + "wer": 0.006472491909385114 + } + }, + "descriptive": { + "raw": { + "utterances": 16, + "errors": 3, + "words": 217, + "wer": 0.013824884792626729 + }, + "semantic": { + "utterances": 16, + "errors": 2, + "words": 226, + "wer": 0.008849557522123894 + } + }, + "emotional": { + "raw": { + "utterances": 24, + "errors": 0, + "words": 314, + "wer": 0.0 + }, + "semantic": { + "utterances": 24, + "errors": 0, + "words": 329, + "wer": 0.0 + } + }, + "external_heldout": { + "raw": { + "utterances": 200, + "errors": 100, + "words": 4250, + "wer": 0.023529411764705882 + }, + "semantic": { + "utterances": 200, + "errors": 133, + "words": 4215, + "wer": 0.03155397390272835 + } + }, + "homographs": { + "raw": { + "utterances": 6, + "errors": 0, + "words": 70, + "wer": 0.0 + }, + "semantic": { + "utterances": 6, + "errors": 0, + "words": 70, + "wer": 0.0 + } + }, + "long": { + "raw": { + "utterances": 15, + "errors": 1, + "words": 412, + "wer": 0.0024271844660194173 + }, + "semantic": { + "utterances": 15, + "errors": 1, + "words": 432, + "wer": 0.0023148148148148147 + } + }, + "long_range": { + "raw": { + "utterances": 6, + "errors": 0, + "words": 121, + "wer": 0.0 + }, + "semantic": { + "utterances": 6, + "errors": 0, + "words": 121, + "wer": 0.0 + } + }, + "names_places": { + "raw": { + "utterances": 6, + "errors": 7, + "words": 57, + "wer": 0.12280701754385964 + }, + "semantic": { + "utterances": 6, + "errors": 5, + "words": 57, + "wer": 0.08771929824561403 + } + }, + "narrative": { + "raw": { + "utterances": 24, + "errors": 4, + "words": 342, + "wer": 0.011695906432748537 + }, + "semantic": { + "utterances": 24, + "errors": 4, + "words": 352, + "wer": 0.011363636363636364 + } + }, + "numbers": { + "raw": { + "utterances": 6, + "errors": 20, + "words": 81, + "wer": 0.24691358024691357 + }, + "semantic": { + "utterances": 6, + "errors": 7, + "words": 64, + "wer": 0.109375 + } + }, + "punctuation": { + "raw": { + "utterances": 6, + "errors": 0, + "words": 65, + "wer": 0.0 + }, + "semantic": { + "utterances": 6, + "errors": 0, + "words": 65, + "wer": 0.0 + } + }, + "question": { + "raw": { + "utterances": 15, + "errors": 0, + "words": 163, + "wer": 0.0 + }, + "semantic": { + "utterances": 15, + "errors": 0, + "words": 165, + "wer": 0.0 + } + }, + "rare_words": { + "raw": { + "utterances": 6, + "errors": 3, + "words": 57, + "wer": 0.05263157894736842 + }, + "semantic": { + "utterances": 6, + "errors": 3, + "words": 57, + "wer": 0.05263157894736842 + } + }, + "short": { + "raw": { + "utterances": 24, + "errors": 1, + "words": 98, + "wer": 0.01020408163265306 + }, + "semantic": { + "utterances": 24, + "errors": 1, + "words": 105, + "wer": 0.009523809523809525 + } + }, + "technical": { + "raw": { + "utterances": 21, + "errors": 9, + "words": 251, + "wer": 0.035856573705179286 + }, + "semantic": { + "utterances": 21, + "errors": 5, + "words": 256, + "wer": 0.01953125 + } + } + } + }, + "nemotron35_asr": { + "raw": { + "utterances": 400, + "errors": 287, + "words": 6794, + "wer": 0.04224315572564027 + }, + "raw_95ci": [ + 0.034229828850855744, + 0.05104311543810849 + ], + "semantic": { + "utterances": 400, + "errors": 259, + "words": 6823, + "wer": 0.037959841711856956 + }, + "semantic_95ci": [ + 0.030281077948420748, + 0.04660587639311044 + ], + "categories": { + "conversational": { + "raw": { + "utterances": 25, + "errors": 3, + "words": 296, + "wer": 0.010135135135135136 + }, + "semantic": { + "utterances": 25, + "errors": 3, + "words": 309, + "wer": 0.009708737864077669 + } + }, + "descriptive": { + "raw": { + "utterances": 16, + "errors": 3, + "words": 217, + "wer": 0.013824884792626729 + }, + "semantic": { + "utterances": 16, + "errors": 3, + "words": 226, + "wer": 0.01327433628318584 + } + }, + "emotional": { + "raw": { + "utterances": 24, + "errors": 1, + "words": 314, + "wer": 0.0031847133757961785 + }, + "semantic": { + "utterances": 24, + "errors": 1, + "words": 329, + "wer": 0.00303951367781155 + } + }, + "external_heldout": { + "raw": { + "utterances": 200, + "errors": 237, + "words": 4250, + "wer": 0.05576470588235294 + }, + "semantic": { + "utterances": 200, + "errors": 208, + "words": 4215, + "wer": 0.04934756820877817 + } + }, + "homographs": { + "raw": { + "utterances": 6, + "errors": 4, + "words": 70, + "wer": 0.05714285714285714 + }, + "semantic": { + "utterances": 6, + "errors": 4, + "words": 70, + "wer": 0.05714285714285714 + } + }, + "long": { + "raw": { + "utterances": 15, + "errors": 3, + "words": 412, + "wer": 0.007281553398058253 + }, + "semantic": { + "utterances": 15, + "errors": 3, + "words": 432, + "wer": 0.006944444444444444 + } + }, + "long_range": { + "raw": { + "utterances": 6, + "errors": 2, + "words": 121, + "wer": 0.01652892561983471 + }, + "semantic": { + "utterances": 6, + "errors": 2, + "words": 121, + "wer": 0.01652892561983471 + } + }, + "names_places": { + "raw": { + "utterances": 6, + "errors": 7, + "words": 57, + "wer": 0.12280701754385964 + }, + "semantic": { + "utterances": 6, + "errors": 7, + "words": 57, + "wer": 0.12280701754385964 + } + }, + "narrative": { + "raw": { + "utterances": 24, + "errors": 5, + "words": 342, + "wer": 0.014619883040935672 + }, + "semantic": { + "utterances": 24, + "errors": 5, + "words": 352, + "wer": 0.014204545454545454 + } + }, + "numbers": { + "raw": { + "utterances": 6, + "errors": 4, + "words": 81, + "wer": 0.04938271604938271 + }, + "semantic": { + "utterances": 6, + "errors": 6, + "words": 64, + "wer": 0.09375 + } + }, + "punctuation": { + "raw": { + "utterances": 6, + "errors": 0, + "words": 65, + "wer": 0.0 + }, + "semantic": { + "utterances": 6, + "errors": 0, + "words": 65, + "wer": 0.0 + } + }, + "question": { + "raw": { + "utterances": 15, + "errors": 0, + "words": 163, + "wer": 0.0 + }, + "semantic": { + "utterances": 15, + "errors": 0, + "words": 165, + "wer": 0.0 + } + }, + "rare_words": { + "raw": { + "utterances": 6, + "errors": 4, + "words": 57, + "wer": 0.07017543859649122 + }, + "semantic": { + "utterances": 6, + "errors": 4, + "words": 57, + "wer": 0.07017543859649122 + } + }, + "short": { + "raw": { + "utterances": 24, + "errors": 6, + "words": 98, + "wer": 0.061224489795918366 + }, + "semantic": { + "utterances": 24, + "errors": 6, + "words": 105, + "wer": 0.05714285714285714 + } + }, + "technical": { + "raw": { + "utterances": 21, + "errors": 8, + "words": 251, + "wer": 0.03187250996015936 + }, + "semantic": { + "utterances": 21, + "errors": 7, + "words": 256, + "wer": 0.02734375 + } + } + } + } + }, + "piper-ryan-low": { + "label": "Piper - Ryan Low", + "qwen3_asr": { + "raw": { + "utterances": 400, + "errors": 181, + "words": 6794, + "wer": 0.02664115395937592 + }, + "raw_95ci": [ + 0.02094850160023276, + 0.03278688524590164 + ], + "semantic": { + "utterances": 400, + "errors": 192, + "words": 6823, + "wer": 0.028140114319214423 + }, + "semantic_95ci": [ + 0.021386536529352664, + 0.03574504737295435 + ], + "categories": { + "conversational": { + "raw": { + "utterances": 25, + "errors": 2, + "words": 296, + "wer": 0.006756756756756757 + }, + "semantic": { + "utterances": 25, + "errors": 2, + "words": 309, + "wer": 0.006472491909385114 + } + }, + "descriptive": { + "raw": { + "utterances": 16, + "errors": 0, + "words": 217, + "wer": 0.0 + }, + "semantic": { + "utterances": 16, + "errors": 0, + "words": 226, + "wer": 0.0 + } + }, + "emotional": { + "raw": { + "utterances": 24, + "errors": 0, + "words": 314, + "wer": 0.0 + }, + "semantic": { + "utterances": 24, + "errors": 0, + "words": 329, + "wer": 0.0 + } + }, + "external_heldout": { + "raw": { + "utterances": 200, + "errors": 130, + "words": 4250, + "wer": 0.03058823529411765 + }, + "semantic": { + "utterances": 200, + "errors": 153, + "words": 4215, + "wer": 0.036298932384341634 + } + }, + "homographs": { + "raw": { + "utterances": 6, + "errors": 3, + "words": 70, + "wer": 0.04285714285714286 + }, + "semantic": { + "utterances": 6, + "errors": 3, + "words": 70, + "wer": 0.04285714285714286 + } + }, + "long": { + "raw": { + "utterances": 15, + "errors": 3, + "words": 412, + "wer": 0.007281553398058253 + }, + "semantic": { + "utterances": 15, + "errors": 3, + "words": 432, + "wer": 0.006944444444444444 + } + }, + "long_range": { + "raw": { + "utterances": 6, + "errors": 1, + "words": 121, + "wer": 0.008264462809917356 + }, + "semantic": { + "utterances": 6, + "errors": 1, + "words": 121, + "wer": 0.008264462809917356 + } + }, + "names_places": { + "raw": { + "utterances": 6, + "errors": 14, + "words": 57, + "wer": 0.24561403508771928 + }, + "semantic": { + "utterances": 6, + "errors": 14, + "words": 57, + "wer": 0.24561403508771928 + } + }, + "narrative": { + "raw": { + "utterances": 24, + "errors": 1, + "words": 342, + "wer": 0.0029239766081871343 + }, + "semantic": { + "utterances": 24, + "errors": 1, + "words": 352, + "wer": 0.002840909090909091 + } + }, + "numbers": { + "raw": { + "utterances": 6, + "errors": 11, + "words": 81, + "wer": 0.13580246913580246 + }, + "semantic": { + "utterances": 6, + "errors": 3, + "words": 64, + "wer": 0.046875 + } + }, + "punctuation": { + "raw": { + "utterances": 6, + "errors": 0, + "words": 65, + "wer": 0.0 + }, + "semantic": { + "utterances": 6, + "errors": 0, + "words": 65, + "wer": 0.0 + } + }, + "question": { + "raw": { + "utterances": 15, + "errors": 0, + "words": 163, + "wer": 0.0 + }, + "semantic": { + "utterances": 15, + "errors": 0, + "words": 165, + "wer": 0.0 + } + }, + "rare_words": { + "raw": { + "utterances": 6, + "errors": 4, + "words": 57, + "wer": 0.07017543859649122 + }, + "semantic": { + "utterances": 6, + "errors": 4, + "words": 57, + "wer": 0.07017543859649122 + } + }, + "short": { + "raw": { + "utterances": 24, + "errors": 3, + "words": 98, + "wer": 0.030612244897959183 + }, + "semantic": { + "utterances": 24, + "errors": 3, + "words": 105, + "wer": 0.02857142857142857 + } + }, + "technical": { + "raw": { + "utterances": 21, + "errors": 9, + "words": 251, + "wer": 0.035856573705179286 + }, + "semantic": { + "utterances": 21, + "errors": 5, + "words": 256, + "wer": 0.01953125 + } + } + } + }, + "nemotron35_asr": { + "raw": { + "utterances": 400, + "errors": 442, + "words": 6794, + "wer": 0.06505740359140418 + }, + "raw_95ci": [ + 0.054938545831482305, + 0.07616320687186828 + ], + "semantic": { + "utterances": 400, + "errors": 376, + "words": 6823, + "wer": 0.055107723875128245 + }, + "semantic_95ci": [ + 0.04650152107779226, + 0.06445341707791237 + ], + "categories": { + "conversational": { + "raw": { + "utterances": 25, + "errors": 10, + "words": 296, + "wer": 0.033783783783783786 + }, + "semantic": { + "utterances": 25, + "errors": 8, + "words": 309, + "wer": 0.025889967637540454 + } + }, + "descriptive": { + "raw": { + "utterances": 16, + "errors": 7, + "words": 217, + "wer": 0.03225806451612903 + }, + "semantic": { + "utterances": 16, + "errors": 7, + "words": 226, + "wer": 0.030973451327433628 + } + }, + "emotional": { + "raw": { + "utterances": 24, + "errors": 1, + "words": 314, + "wer": 0.0031847133757961785 + }, + "semantic": { + "utterances": 24, + "errors": 1, + "words": 329, + "wer": 0.00303951367781155 + } + }, + "external_heldout": { + "raw": { + "utterances": 200, + "errors": 347, + "words": 4250, + "wer": 0.0816470588235294 + }, + "semantic": { + "utterances": 200, + "errors": 284, + "words": 4215, + "wer": 0.06737841043890866 + } + }, + "homographs": { + "raw": { + "utterances": 6, + "errors": 7, + "words": 70, + "wer": 0.1 + }, + "semantic": { + "utterances": 6, + "errors": 7, + "words": 70, + "wer": 0.1 + } + }, + "long": { + "raw": { + "utterances": 15, + "errors": 9, + "words": 412, + "wer": 0.021844660194174758 + }, + "semantic": { + "utterances": 15, + "errors": 9, + "words": 432, + "wer": 0.020833333333333332 + } + }, + "long_range": { + "raw": { + "utterances": 6, + "errors": 1, + "words": 121, + "wer": 0.008264462809917356 + }, + "semantic": { + "utterances": 6, + "errors": 1, + "words": 121, + "wer": 0.008264462809917356 + } + }, + "names_places": { + "raw": { + "utterances": 6, + "errors": 24, + "words": 57, + "wer": 0.42105263157894735 + }, + "semantic": { + "utterances": 6, + "errors": 24, + "words": 57, + "wer": 0.42105263157894735 + } + }, + "narrative": { + "raw": { + "utterances": 24, + "errors": 4, + "words": 342, + "wer": 0.011695906432748537 + }, + "semantic": { + "utterances": 24, + "errors": 4, + "words": 352, + "wer": 0.011363636363636364 + } + }, + "numbers": { + "raw": { + "utterances": 6, + "errors": 5, + "words": 81, + "wer": 0.06172839506172839 + }, + "semantic": { + "utterances": 6, + "errors": 5, + "words": 64, + "wer": 0.078125 + } + }, + "punctuation": { + "raw": { + "utterances": 6, + "errors": 1, + "words": 65, + "wer": 0.015384615384615385 + }, + "semantic": { + "utterances": 6, + "errors": 1, + "words": 65, + "wer": 0.015384615384615385 + } + }, + "question": { + "raw": { + "utterances": 15, + "errors": 0, + "words": 163, + "wer": 0.0 + }, + "semantic": { + "utterances": 15, + "errors": 0, + "words": 165, + "wer": 0.0 + } + }, + "rare_words": { + "raw": { + "utterances": 6, + "errors": 6, + "words": 57, + "wer": 0.10526315789473684 + }, + "semantic": { + "utterances": 6, + "errors": 6, + "words": 57, + "wer": 0.10526315789473684 + } + }, + "short": { + "raw": { + "utterances": 24, + "errors": 10, + "words": 98, + "wer": 0.10204081632653061 + }, + "semantic": { + "utterances": 24, + "errors": 10, + "words": 105, + "wer": 0.09523809523809523 + } + }, + "technical": { + "raw": { + "utterances": 21, + "errors": 10, + "words": 251, + "wer": 0.0398406374501992 + }, + "semantic": { + "utterances": 21, + "errors": 9, + "words": 256, + "wer": 0.03515625 + } + } + } + } + }, + "piper-danny-low": { + "label": "Piper - Danny Low", + "qwen3_asr": { + "raw": { + "utterances": 400, + "errors": 175, + "words": 6794, + "wer": 0.025758021783926995 + }, + "raw_95ci": [ + 0.01956489998539933, + 0.03292759051186017 + ], + "semantic": { + "utterances": 400, + "errors": 179, + "words": 6823, + "wer": 0.026234794078850945 + }, + "semantic_95ci": [ + 0.019979876383498633, + 0.03340659340659341 + ], + "categories": { + "conversational": { + "raw": { + "utterances": 25, + "errors": 2, + "words": 296, + "wer": 0.006756756756756757 + }, + "semantic": { + "utterances": 25, + "errors": 2, + "words": 309, + "wer": 0.006472491909385114 + } + }, + "descriptive": { + "raw": { + "utterances": 16, + "errors": 2, + "words": 217, + "wer": 0.009216589861751152 + }, + "semantic": { + "utterances": 16, + "errors": 2, + "words": 226, + "wer": 0.008849557522123894 + } + }, + "emotional": { + "raw": { + "utterances": 24, + "errors": 1, + "words": 314, + "wer": 0.0031847133757961785 + }, + "semantic": { + "utterances": 24, + "errors": 1, + "words": 329, + "wer": 0.00303951367781155 + } + }, + "external_heldout": { + "raw": { + "utterances": 200, + "errors": 131, + "words": 4250, + "wer": 0.030823529411764704 + }, + "semantic": { + "utterances": 200, + "errors": 150, + "words": 4215, + "wer": 0.03558718861209965 + } + }, + "homographs": { + "raw": { + "utterances": 6, + "errors": 0, + "words": 70, + "wer": 0.0 + }, + "semantic": { + "utterances": 6, + "errors": 0, + "words": 70, + "wer": 0.0 + } + }, + "long": { + "raw": { + "utterances": 15, + "errors": 5, + "words": 412, + "wer": 0.012135922330097087 + }, + "semantic": { + "utterances": 15, + "errors": 5, + "words": 432, + "wer": 0.011574074074074073 + } + }, + "long_range": { + "raw": { + "utterances": 6, + "errors": 0, + "words": 121, + "wer": 0.0 + }, + "semantic": { + "utterances": 6, + "errors": 0, + "words": 121, + "wer": 0.0 + } + }, + "names_places": { + "raw": { + "utterances": 6, + "errors": 6, + "words": 57, + "wer": 0.10526315789473684 + }, + "semantic": { + "utterances": 6, + "errors": 6, + "words": 57, + "wer": 0.10526315789473684 + } + }, + "narrative": { + "raw": { + "utterances": 24, + "errors": 0, + "words": 342, + "wer": 0.0 + }, + "semantic": { + "utterances": 24, + "errors": 0, + "words": 352, + "wer": 0.0 + } + }, + "numbers": { + "raw": { + "utterances": 6, + "errors": 13, + "words": 81, + "wer": 0.16049382716049382 + }, + "semantic": { + "utterances": 6, + "errors": 2, + "words": 64, + "wer": 0.03125 + } + }, + "punctuation": { + "raw": { + "utterances": 6, + "errors": 1, + "words": 65, + "wer": 0.015384615384615385 + }, + "semantic": { + "utterances": 6, + "errors": 1, + "words": 65, + "wer": 0.015384615384615385 + } + }, + "question": { + "raw": { + "utterances": 15, + "errors": 0, + "words": 163, + "wer": 0.0 + }, + "semantic": { + "utterances": 15, + "errors": 0, + "words": 165, + "wer": 0.0 + } + }, + "rare_words": { + "raw": { + "utterances": 6, + "errors": 3, + "words": 57, + "wer": 0.05263157894736842 + }, + "semantic": { + "utterances": 6, + "errors": 3, + "words": 57, + "wer": 0.05263157894736842 + } + }, + "short": { + "raw": { + "utterances": 24, + "errors": 2, + "words": 98, + "wer": 0.02040816326530612 + }, + "semantic": { + "utterances": 24, + "errors": 2, + "words": 105, + "wer": 0.01904761904761905 + } + }, + "technical": { + "raw": { + "utterances": 21, + "errors": 9, + "words": 251, + "wer": 0.035856573705179286 + }, + "semantic": { + "utterances": 21, + "errors": 5, + "words": 256, + "wer": 0.01953125 + } + } + } + }, + "nemotron35_asr": { + "raw": { + "utterances": 400, + "errors": 438, + "words": 6794, + "wer": 0.06446864880777156 + }, + "raw_95ci": [ + 0.05425912670007158, + 0.07542874487782668 + ], + "semantic": { + "utterances": 400, + "errors": 382, + "words": 6823, + "wer": 0.055987102447603694 + }, + "semantic_95ci": [ + 0.04722822174226062, + 0.06521079769487413 + ], + "categories": { + "conversational": { + "raw": { + "utterances": 25, + "errors": 4, + "words": 296, + "wer": 0.013513513513513514 + }, + "semantic": { + "utterances": 25, + "errors": 4, + "words": 309, + "wer": 0.012944983818770227 + } + }, + "descriptive": { + "raw": { + "utterances": 16, + "errors": 4, + "words": 217, + "wer": 0.018433179723502304 + }, + "semantic": { + "utterances": 16, + "errors": 4, + "words": 226, + "wer": 0.017699115044247787 + } + }, + "emotional": { + "raw": { + "utterances": 24, + "errors": 3, + "words": 314, + "wer": 0.009554140127388535 + }, + "semantic": { + "utterances": 24, + "errors": 3, + "words": 329, + "wer": 0.00911854103343465 + } + }, + "external_heldout": { + "raw": { + "utterances": 200, + "errors": 349, + "words": 4250, + "wer": 0.08211764705882353 + }, + "semantic": { + "utterances": 200, + "errors": 295, + "words": 4215, + "wer": 0.06998813760379596 + } + }, + "homographs": { + "raw": { + "utterances": 6, + "errors": 3, + "words": 70, + "wer": 0.04285714285714286 + }, + "semantic": { + "utterances": 6, + "errors": 3, + "words": 70, + "wer": 0.04285714285714286 + } + }, + "long": { + "raw": { + "utterances": 15, + "errors": 19, + "words": 412, + "wer": 0.04611650485436893 + }, + "semantic": { + "utterances": 15, + "errors": 19, + "words": 432, + "wer": 0.04398148148148148 + } + }, + "long_range": { + "raw": { + "utterances": 6, + "errors": 4, + "words": 121, + "wer": 0.03305785123966942 + }, + "semantic": { + "utterances": 6, + "errors": 4, + "words": 121, + "wer": 0.03305785123966942 + } + }, + "names_places": { + "raw": { + "utterances": 6, + "errors": 13, + "words": 57, + "wer": 0.22807017543859648 + }, + "semantic": { + "utterances": 6, + "errors": 13, + "words": 57, + "wer": 0.22807017543859648 + } + }, + "narrative": { + "raw": { + "utterances": 24, + "errors": 1, + "words": 342, + "wer": 0.0029239766081871343 + }, + "semantic": { + "utterances": 24, + "errors": 1, + "words": 352, + "wer": 0.002840909090909091 + } + }, + "numbers": { + "raw": { + "utterances": 6, + "errors": 5, + "words": 81, + "wer": 0.06172839506172839 + }, + "semantic": { + "utterances": 6, + "errors": 4, + "words": 64, + "wer": 0.0625 + } + }, + "punctuation": { + "raw": { + "utterances": 6, + "errors": 1, + "words": 65, + "wer": 0.015384615384615385 + }, + "semantic": { + "utterances": 6, + "errors": 1, + "words": 65, + "wer": 0.015384615384615385 + } + }, + "question": { + "raw": { + "utterances": 15, + "errors": 1, + "words": 163, + "wer": 0.006134969325153374 + }, + "semantic": { + "utterances": 15, + "errors": 1, + "words": 165, + "wer": 0.006060606060606061 + } + }, + "rare_words": { + "raw": { + "utterances": 6, + "errors": 4, + "words": 57, + "wer": 0.07017543859649122 + }, + "semantic": { + "utterances": 6, + "errors": 4, + "words": 57, + "wer": 0.07017543859649122 + } + }, + "short": { + "raw": { + "utterances": 24, + "errors": 3, + "words": 98, + "wer": 0.030612244897959183 + }, + "semantic": { + "utterances": 24, + "errors": 3, + "words": 105, + "wer": 0.02857142857142857 + } + }, + "technical": { + "raw": { + "utterances": 21, + "errors": 24, + "words": 251, + "wer": 0.09561752988047809 + }, + "semantic": { + "utterances": 21, + "errors": 23, + "words": 256, + "wer": 0.08984375 + } + } + } + } + }, + "supertonic3-m2-3step": { + "label": "Supertonic 3 - M2 - 3-step", + "qwen3_asr": { + "raw": { + "utterances": 400, + "errors": 200, + "words": 6794, + "wer": 0.02943773918163085 + }, + "raw_95ci": [ + 0.023245740687265377, + 0.03628800917957545 + ], + "semantic": { + "utterances": 400, + "errors": 207, + "words": 6823, + "wer": 0.03033856075040305 + }, + "semantic_95ci": [ + 0.02375875723423698, + 0.03767123287671233 + ], + "categories": { + "conversational": { + "raw": { + "utterances": 25, + "errors": 0, + "words": 296, + "wer": 0.0 + }, + "semantic": { + "utterances": 25, + "errors": 0, + "words": 309, + "wer": 0.0 + } + }, + "descriptive": { + "raw": { + "utterances": 16, + "errors": 1, + "words": 217, + "wer": 0.004608294930875576 + }, + "semantic": { + "utterances": 16, + "errors": 1, + "words": 226, + "wer": 0.004424778761061947 + } + }, + "emotional": { + "raw": { + "utterances": 24, + "errors": 1, + "words": 314, + "wer": 0.0031847133757961785 + }, + "semantic": { + "utterances": 24, + "errors": 1, + "words": 329, + "wer": 0.00303951367781155 + } + }, + "external_heldout": { + "raw": { + "utterances": 200, + "errors": 120, + "words": 4250, + "wer": 0.02823529411764706 + }, + "semantic": { + "utterances": 200, + "errors": 153, + "words": 4215, + "wer": 0.036298932384341634 + } + }, + "homographs": { + "raw": { + "utterances": 6, + "errors": 4, + "words": 70, + "wer": 0.05714285714285714 + }, + "semantic": { + "utterances": 6, + "errors": 4, + "words": 70, + "wer": 0.05714285714285714 + } + }, + "long": { + "raw": { + "utterances": 15, + "errors": 6, + "words": 412, + "wer": 0.014563106796116505 + }, + "semantic": { + "utterances": 15, + "errors": 6, + "words": 432, + "wer": 0.013888888888888888 + } + }, + "long_range": { + "raw": { + "utterances": 6, + "errors": 1, + "words": 121, + "wer": 0.008264462809917356 + }, + "semantic": { + "utterances": 6, + "errors": 1, + "words": 121, + "wer": 0.008264462809917356 + } + }, + "names_places": { + "raw": { + "utterances": 6, + "errors": 10, + "words": 57, + "wer": 0.17543859649122806 + }, + "semantic": { + "utterances": 6, + "errors": 10, + "words": 57, + "wer": 0.17543859649122806 + } + }, + "narrative": { + "raw": { + "utterances": 24, + "errors": 0, + "words": 342, + "wer": 0.0 + }, + "semantic": { + "utterances": 24, + "errors": 0, + "words": 352, + "wer": 0.0 + } + }, + "numbers": { + "raw": { + "utterances": 6, + "errors": 24, + "words": 81, + "wer": 0.2962962962962963 + }, + "semantic": { + "utterances": 6, + "errors": 2, + "words": 64, + "wer": 0.03125 + } + }, + "punctuation": { + "raw": { + "utterances": 6, + "errors": 3, + "words": 65, + "wer": 0.046153846153846156 + }, + "semantic": { + "utterances": 6, + "errors": 3, + "words": 65, + "wer": 0.046153846153846156 + } + }, + "question": { + "raw": { + "utterances": 15, + "errors": 1, + "words": 163, + "wer": 0.006134969325153374 + }, + "semantic": { + "utterances": 15, + "errors": 1, + "words": 165, + "wer": 0.006060606060606061 + } + }, + "rare_words": { + "raw": { + "utterances": 6, + "errors": 7, + "words": 57, + "wer": 0.12280701754385964 + }, + "semantic": { + "utterances": 6, + "errors": 7, + "words": 57, + "wer": 0.12280701754385964 + } + }, + "short": { + "raw": { + "utterances": 24, + "errors": 13, + "words": 98, + "wer": 0.1326530612244898 + }, + "semantic": { + "utterances": 24, + "errors": 13, + "words": 105, + "wer": 0.12380952380952381 + } + }, + "technical": { + "raw": { + "utterances": 21, + "errors": 9, + "words": 251, + "wer": 0.035856573705179286 + }, + "semantic": { + "utterances": 21, + "errors": 5, + "words": 256, + "wer": 0.01953125 + } + } + } + }, + "nemotron35_asr": { + "raw": { + "utterances": 400, + "errors": 446, + "words": 6794, + "wer": 0.0656461583750368 + }, + "raw_95ci": [ + 0.05547445255474453, + 0.07639304973037747 + ], + "semantic": { + "utterances": 400, + "errors": 412, + "words": 6823, + "wer": 0.060383995309980946 + }, + "semantic_95ci": [ + 0.05129658495111237, + 0.07025560474082536 + ], + "categories": { + "conversational": { + "raw": { + "utterances": 25, + "errors": 7, + "words": 296, + "wer": 0.02364864864864865 + }, + "semantic": { + "utterances": 25, + "errors": 7, + "words": 309, + "wer": 0.022653721682847898 + } + }, + "descriptive": { + "raw": { + "utterances": 16, + "errors": 1, + "words": 217, + "wer": 0.004608294930875576 + }, + "semantic": { + "utterances": 16, + "errors": 1, + "words": 226, + "wer": 0.004424778761061947 + } + }, + "emotional": { + "raw": { + "utterances": 24, + "errors": 7, + "words": 314, + "wer": 0.022292993630573247 + }, + "semantic": { + "utterances": 24, + "errors": 7, + "words": 329, + "wer": 0.02127659574468085 + } + }, + "external_heldout": { + "raw": { + "utterances": 200, + "errors": 324, + "words": 4250, + "wer": 0.07623529411764705 + }, + "semantic": { + "utterances": 200, + "errors": 292, + "words": 4215, + "wer": 0.06927639383155397 + } + }, + "homographs": { + "raw": { + "utterances": 6, + "errors": 7, + "words": 70, + "wer": 0.1 + }, + "semantic": { + "utterances": 6, + "errors": 7, + "words": 70, + "wer": 0.1 + } + }, + "long": { + "raw": { + "utterances": 15, + "errors": 9, + "words": 412, + "wer": 0.021844660194174758 + }, + "semantic": { + "utterances": 15, + "errors": 7, + "words": 432, + "wer": 0.016203703703703703 + } + }, + "long_range": { + "raw": { + "utterances": 6, + "errors": 4, + "words": 121, + "wer": 0.03305785123966942 + }, + "semantic": { + "utterances": 6, + "errors": 4, + "words": 121, + "wer": 0.03305785123966942 + } + }, + "names_places": { + "raw": { + "utterances": 6, + "errors": 12, + "words": 57, + "wer": 0.21052631578947367 + }, + "semantic": { + "utterances": 6, + "errors": 12, + "words": 57, + "wer": 0.21052631578947367 + } + }, + "narrative": { + "raw": { + "utterances": 24, + "errors": 1, + "words": 342, + "wer": 0.0029239766081871343 + }, + "semantic": { + "utterances": 24, + "errors": 1, + "words": 352, + "wer": 0.002840909090909091 + } + }, + "numbers": { + "raw": { + "utterances": 6, + "errors": 4, + "words": 81, + "wer": 0.04938271604938271 + }, + "semantic": { + "utterances": 6, + "errors": 3, + "words": 64, + "wer": 0.046875 + } + }, + "punctuation": { + "raw": { + "utterances": 6, + "errors": 8, + "words": 65, + "wer": 0.12307692307692308 + }, + "semantic": { + "utterances": 6, + "errors": 9, + "words": 65, + "wer": 0.13846153846153847 + } + }, + "question": { + "raw": { + "utterances": 15, + "errors": 9, + "words": 163, + "wer": 0.05521472392638037 + }, + "semantic": { + "utterances": 15, + "errors": 9, + "words": 165, + "wer": 0.05454545454545454 + } + }, + "rare_words": { + "raw": { + "utterances": 6, + "errors": 10, + "words": 57, + "wer": 0.17543859649122806 + }, + "semantic": { + "utterances": 6, + "errors": 10, + "words": 57, + "wer": 0.17543859649122806 + } + }, + "short": { + "raw": { + "utterances": 24, + "errors": 31, + "words": 98, + "wer": 0.3163265306122449 + }, + "semantic": { + "utterances": 24, + "errors": 31, + "words": 105, + "wer": 0.29523809523809524 + } + }, + "technical": { + "raw": { + "utterances": 21, + "errors": 12, + "words": 251, + "wer": 0.04780876494023904 + }, + "semantic": { + "utterances": 21, + "errors": 12, + "words": 256, + "wer": 0.046875 + } + } + } + } + }, + "supertonic3-m2-8step": { + "label": "Supertonic 3 - M2 - 8-step", + "qwen3_asr": { + "raw": { + "utterances": 400, + "errors": 137, + "words": 6794, + "wer": 0.020164851339417134 + }, + "raw_95ci": [ + 0.014720886168197056, + 0.026397286535909156 + ], + "semantic": { + "utterances": 400, + "errors": 140, + "words": 6823, + "wer": 0.020518833357760516 + }, + "semantic_95ci": [ + 0.014671013633669235, + 0.02735562310030395 + ], + "categories": { + "conversational": { + "raw": { + "utterances": 25, + "errors": 0, + "words": 296, + "wer": 0.0 + }, + "semantic": { + "utterances": 25, + "errors": 0, + "words": 309, + "wer": 0.0 + } + }, + "descriptive": { + "raw": { + "utterances": 16, + "errors": 1, + "words": 217, + "wer": 0.004608294930875576 + }, + "semantic": { + "utterances": 16, + "errors": 0, + "words": 226, + "wer": 0.0 + } + }, + "emotional": { + "raw": { + "utterances": 24, + "errors": 1, + "words": 314, + "wer": 0.0031847133757961785 + }, + "semantic": { + "utterances": 24, + "errors": 1, + "words": 329, + "wer": 0.00303951367781155 + } + }, + "external_heldout": { + "raw": { + "utterances": 200, + "errors": 89, + "words": 4250, + "wer": 0.020941176470588234 + }, + "semantic": { + "utterances": 200, + "errors": 119, + "words": 4215, + "wer": 0.02823250296559905 + } + }, + "homographs": { + "raw": { + "utterances": 6, + "errors": 0, + "words": 70, + "wer": 0.0 + }, + "semantic": { + "utterances": 6, + "errors": 0, + "words": 70, + "wer": 0.0 + } + }, + "long": { + "raw": { + "utterances": 15, + "errors": 3, + "words": 412, + "wer": 0.007281553398058253 + }, + "semantic": { + "utterances": 15, + "errors": 3, + "words": 432, + "wer": 0.006944444444444444 + } + }, + "long_range": { + "raw": { + "utterances": 6, + "errors": 0, + "words": 121, + "wer": 0.0 + }, + "semantic": { + "utterances": 6, + "errors": 0, + "words": 121, + "wer": 0.0 + } + }, + "names_places": { + "raw": { + "utterances": 6, + "errors": 5, + "words": 57, + "wer": 0.08771929824561403 + }, + "semantic": { + "utterances": 6, + "errors": 5, + "words": 57, + "wer": 0.08771929824561403 + } + }, + "narrative": { + "raw": { + "utterances": 24, + "errors": 0, + "words": 342, + "wer": 0.0 + }, + "semantic": { + "utterances": 24, + "errors": 0, + "words": 352, + "wer": 0.0 + } + }, + "numbers": { + "raw": { + "utterances": 6, + "errors": 24, + "words": 81, + "wer": 0.2962962962962963 + }, + "semantic": { + "utterances": 6, + "errors": 2, + "words": 64, + "wer": 0.03125 + } + }, + "punctuation": { + "raw": { + "utterances": 6, + "errors": 0, + "words": 65, + "wer": 0.0 + }, + "semantic": { + "utterances": 6, + "errors": 0, + "words": 65, + "wer": 0.0 + } + }, + "question": { + "raw": { + "utterances": 15, + "errors": 0, + "words": 163, + "wer": 0.0 + }, + "semantic": { + "utterances": 15, + "errors": 0, + "words": 165, + "wer": 0.0 + } + }, + "rare_words": { + "raw": { + "utterances": 6, + "errors": 4, + "words": 57, + "wer": 0.07017543859649122 + }, + "semantic": { + "utterances": 6, + "errors": 4, + "words": 57, + "wer": 0.07017543859649122 + } + }, + "short": { + "raw": { + "utterances": 24, + "errors": 1, + "words": 98, + "wer": 0.01020408163265306 + }, + "semantic": { + "utterances": 24, + "errors": 1, + "words": 105, + "wer": 0.009523809523809525 + } + }, + "technical": { + "raw": { + "utterances": 21, + "errors": 9, + "words": 251, + "wer": 0.035856573705179286 + }, + "semantic": { + "utterances": 21, + "errors": 5, + "words": 256, + "wer": 0.01953125 + } + } + } + }, + "nemotron35_asr": { + "raw": { + "utterances": 400, + "errors": 284, + "words": 6794, + "wer": 0.04180158963791581 + }, + "raw_95ci": [ + 0.033854950346072825, + 0.050436615477265884 + ], + "semantic": { + "utterances": 400, + "errors": 243, + "words": 6823, + "wer": 0.035614832185255754 + }, + "semantic_95ci": [ + 0.028567347521782603, + 0.043412264723740136 + ], + "categories": { + "conversational": { + "raw": { + "utterances": 25, + "errors": 1, + "words": 296, + "wer": 0.0033783783783783786 + }, + "semantic": { + "utterances": 25, + "errors": 1, + "words": 309, + "wer": 0.003236245954692557 + } + }, + "descriptive": { + "raw": { + "utterances": 16, + "errors": 3, + "words": 217, + "wer": 0.013824884792626729 + }, + "semantic": { + "utterances": 16, + "errors": 3, + "words": 226, + "wer": 0.01327433628318584 + } + }, + "emotional": { + "raw": { + "utterances": 24, + "errors": 1, + "words": 314, + "wer": 0.0031847133757961785 + }, + "semantic": { + "utterances": 24, + "errors": 1, + "words": 329, + "wer": 0.00303951367781155 + } + }, + "external_heldout": { + "raw": { + "utterances": 200, + "errors": 245, + "words": 4250, + "wer": 0.05764705882352941 + }, + "semantic": { + "utterances": 200, + "errors": 205, + "words": 4215, + "wer": 0.04863582443653618 + } + }, + "homographs": { + "raw": { + "utterances": 6, + "errors": 2, + "words": 70, + "wer": 0.02857142857142857 + }, + "semantic": { + "utterances": 6, + "errors": 2, + "words": 70, + "wer": 0.02857142857142857 + } + }, + "long": { + "raw": { + "utterances": 15, + "errors": 4, + "words": 412, + "wer": 0.009708737864077669 + }, + "semantic": { + "utterances": 15, + "errors": 4, + "words": 432, + "wer": 0.009259259259259259 + } + }, + "long_range": { + "raw": { + "utterances": 6, + "errors": 0, + "words": 121, + "wer": 0.0 + }, + "semantic": { + "utterances": 6, + "errors": 0, + "words": 121, + "wer": 0.0 + } + }, + "names_places": { + "raw": { + "utterances": 6, + "errors": 11, + "words": 57, + "wer": 0.19298245614035087 + }, + "semantic": { + "utterances": 6, + "errors": 11, + "words": 57, + "wer": 0.19298245614035087 + } + }, + "narrative": { + "raw": { + "utterances": 24, + "errors": 0, + "words": 342, + "wer": 0.0 + }, + "semantic": { + "utterances": 24, + "errors": 0, + "words": 352, + "wer": 0.0 + } + }, + "numbers": { + "raw": { + "utterances": 6, + "errors": 3, + "words": 81, + "wer": 0.037037037037037035 + }, + "semantic": { + "utterances": 6, + "errors": 2, + "words": 64, + "wer": 0.03125 + } + }, + "punctuation": { + "raw": { + "utterances": 6, + "errors": 0, + "words": 65, + "wer": 0.0 + }, + "semantic": { + "utterances": 6, + "errors": 0, + "words": 65, + "wer": 0.0 + } + }, + "question": { + "raw": { + "utterances": 15, + "errors": 0, + "words": 163, + "wer": 0.0 + }, + "semantic": { + "utterances": 15, + "errors": 0, + "words": 165, + "wer": 0.0 + } + }, + "rare_words": { + "raw": { + "utterances": 6, + "errors": 5, + "words": 57, + "wer": 0.08771929824561403 + }, + "semantic": { + "utterances": 6, + "errors": 5, + "words": 57, + "wer": 0.08771929824561403 + } + }, + "short": { + "raw": { + "utterances": 24, + "errors": 3, + "words": 98, + "wer": 0.030612244897959183 + }, + "semantic": { + "utterances": 24, + "errors": 3, + "words": 105, + "wer": 0.02857142857142857 + } + }, + "technical": { + "raw": { + "utterances": 21, + "errors": 6, + "words": 251, + "wer": 0.02390438247011952 + }, + "semantic": { + "utterances": 21, + "errors": 6, + "words": 256, + "wer": 0.0234375 + } + } + } + } + } + } +}