Tune raw reference audio conditioning

README.md

@@ -21,9 +21,9 @@ models:
 ## 模式
 
 - **內建語者**：預設為內建語者 A；Speaker B 使用模型庫 `checkpoints/speaker_b_embedding.pt`，固定 seed 56789 與 projector scale 1.125。長文會自動以最多 80 個字元分段。
-- **參考音色**：錄音直接送入模型原生的 raw reference-audio conditioning，不經 ECAPA centroid；固定 seed 1337。長文同樣自動分段。
+- **參考音色**：錄音直接送入模型原生的 raw reference-audio conditioning，不經 ECAPA centroid；超過 6 秒固定取中央 6 秒，CFG 2.0，固定 seed 1337。長文同樣自動分段。
 
 兩種模式的每個文字分段都只生成一次，分段間加入短靜音串接；不需要切換獨立長文模式。
 
 TTS model snapshot 固定為
-`6f7cab914a1e27c56b504ec663c0144dc25cc0a3`，預設 CFG 為 3.0，NFE 為 10。
+`6f7cab914a1e27c56b504ec663c0144dc25cc0a3`，內建語者預設 CFG 為 3.0，參考音色預設 CFG 為 2.0，NFE 為 10。

app.py

@@ -9,6 +9,7 @@ import json
 import os
 from pathlib import Path
 import secrets
+import tempfile
 import threading
 from types import MappingProxyType
 
@@ -61,6 +62,7 @@ except ImportError:
 
 import gradio as gr
 import numpy as np
+import soundfile as sf
 import torch
 from huggingface_hub import snapshot_download
 from transformers import PreTrainedTokenizerFast
@@ -199,6 +201,7 @@ MODEL_REVISION = "6f7cab914a1e27c56b504ec663c0144dc25cc0a3"
 ECAPA_REPO_ID = "speechbrain/spkrec-ecapa-voxceleb"
 ECAPA_REVISION = "0f99f2d0ebe89ac095bcc5903c4dd8f72b367286"
 DEFAULT_CFG = 3.0
+REFERENCE_CFG = 2.0
 DEFAULT_STEPS = 10
 TARGET_CPS = 4.0
 ACTIVE_PACE_TARGET_CPS = 4.00
@@ -276,6 +279,7 @@ SPEAKER_B_EMBEDDING_SHA256 = (
     "e9556e14723c140985a104c1659d1ff8a5078d2fa28ce2fb756f04906641a8a7"
 )
 REFERENCE_GENERATION_SEED = 1_337
+REFERENCE_AUDIO_SECONDS = 6.0
 GLYPH_ASSET_MANIFEST_RELATIVE_PATH = "glyph_assets_v1/manifest.json"
 # The asset builder replaces this unavailable sentinel with the exact frozen
 # manifest SHA-256 before a candidate can expose the V5 profile.
@@ -2363,6 +2367,36 @@ def _fast_text_chunks(text: str) -> tuple[str, ...]:
     return tuple(chunks)
 
 
+def _prepare_reference_audio(reference_wav: str) -> tuple[str, str | None]:
+    """Deterministically center-crop long references to the trained 6s cap."""
+
+    info = sf.info(reference_wav)
+    max_frames = int(round(REFERENCE_AUDIO_SECONDS * int(info.samplerate)))
+    if int(info.frames) <= max_frames:
+        return reference_wav, None
+    start = max(0, (int(info.frames) - max_frames) // 2)
+    waveform, sample_rate = sf.read(
+        reference_wav,
+        start=start,
+        frames=max_frames,
+        dtype="float32",
+        always_2d=True,
+    )
+    with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as handle:
+        temporary_path = handle.name
+    try:
+        sf.write(
+            temporary_path,
+            waveform,
+            sample_rate,
+            subtype="PCM_16",
+        )
+    except Exception:
+        os.unlink(temporary_path)
+        raise
+    return temporary_path, temporary_path
+
+
 def _generate_fast_chunk(
     text: str,
     centroid: torch.Tensor | None,
@@ -2371,6 +2405,7 @@ def _generate_fast_chunk(
     steps: int,
     request_seed: int,
     reference_wav_path: str = "",
+    short_text_min_cfg: float = SHORT_TEXT_CFG_MIN,
 ) -> np.ndarray:
     """Run one model call and return it without candidate qualification."""
 
@@ -2390,7 +2425,7 @@ def _generate_fast_chunk(
         text,
         float(cfg),
         short_text_unit_threshold=SHORT_TEXT_CFG_UNITS,
-        short_text_min_cfg=SHORT_TEXT_CFG_MIN,
+        short_text_min_cfg=float(short_text_min_cfg),
     )
     set_generation_seed(request_seed)
     kwargs = {
@@ -2453,6 +2488,7 @@ def _synthesize(
     request_seed: int | None = None,
     speaker_projector_scale: float = 1.0,
     reference_wav_path: str = "",
+    short_text_min_cfg: float = SHORT_TEXT_CFG_MIN,
 ) -> tuple[int, np.ndarray]:
     """Latency-first single-pass synthesis with no output qualification."""
 
@@ -2486,6 +2522,7 @@ def _synthesize(
                     steps=int(steps),
                     request_seed=seed,
                     reference_wav_path=reference_wav_path,
+                    short_text_min_cfg=short_text_min_cfg,
                 )
                 for chunk in chunks
             )
@@ -3841,23 +3878,29 @@ def tts_speaker(
 def tts_reference(
     text: str,
     reference_wav: str,
-    cfg: float = DEFAULT_CFG,
+    cfg: float = REFERENCE_CFG,
     steps: int = DEFAULT_STEPS,
     speed: float = 1.0,
 ):
     if not reference_wav:
         raise gr.Error("請先錄音或上傳參考音檔。")
-    return pcm16_audio_output(
-        *_synthesize(
-            text,
-            None,
-            cfg=cfg,
-            steps=steps,
-            speed=speed,
-            request_seed=REFERENCE_GENERATION_SEED,
-            reference_wav_path=reference_wav,
+    prepared_reference, temporary_reference = _prepare_reference_audio(reference_wav)
+    try:
+        return pcm16_audio_output(
+            *_synthesize(
+                text,
+                None,
+                cfg=cfg,
+                steps=steps,
+                speed=speed,
+                request_seed=REFERENCE_GENERATION_SEED,
+                reference_wav_path=prepared_reference,
+                short_text_min_cfg=REFERENCE_CFG,
+            )
         )
-    )
+    finally:
+        if temporary_reference is not None:
+            os.unlink(temporary_reference)
 
 EXAMPLE_TEXTS = [
     "今天天氣真好，我們一起去散步吧。",
@@ -3884,7 +3927,12 @@ with gr.Blocks(title="BlueMagpie-TTS Demo", theme=gr.themes.Soft()) as demo:
         with gr.Row():
             cfg_input = gr.Number(
                 value=DEFAULT_CFG,
-                label="CFG",
+                label="內建語者 CFG",
+                interactive=True,
+            )
+            reference_cfg_input = gr.Number(
+                value=REFERENCE_CFG,
+                label="參考���色 CFG",
                 interactive=True,
             )
             steps_input = gr.Number(
@@ -3916,8 +3964,9 @@ with gr.Blocks(title="BlueMagpie-TTS Demo", theme=gr.themes.Soft()) as demo:
 
     with gr.Tab("參考音色"):
         gr.Markdown(
-            "建議錄製 6–12 秒、單一語者且背景乾淨的授權參考音檔；"
-            "錄音會直接作為模型的 reference audio，不轉成 ECAPA 音色。"
+            "建議錄製至少 6 秒、單一語者且背景乾淨的授權參考音檔；"
+            "超過 6 秒會固定取中央 6 秒。錄音會直接作為模型的 "
+            "reference audio，不轉成 ECAPA 音色。"
         )
         with gr.Row():
             with gr.Column():
@@ -3932,7 +3981,13 @@ with gr.Blocks(title="BlueMagpie-TTS Demo", theme=gr.themes.Soft()) as demo:
                 reference_output = gr.Audio(label="合成結果", type="numpy")
         reference_button.click(
             tts_reference,
-            [reference_text, reference_audio, cfg_input, steps_input, speed_input],
+            [
+                reference_text,
+                reference_audio,
+                reference_cfg_input,
+                steps_input,
+                speed_input,
+            ],
             reference_output,
         )
 

tests/test_latency_first_mode.py

@@ -4,6 +4,7 @@ import os
 from pathlib import Path
 
 import numpy as np
+import soundfile as sf
 import torch
 
 
@@ -222,12 +223,16 @@ def test_reference_conditioning_uses_raw_reference_audio_once():
     synthesize_source = ast.get_source_segment(source, _function_node("_synthesize"))
 
     assert "REFERENCE_GENERATION_SEED = 1_337" in source
+    assert "REFERENCE_CFG = 2.0" in source
+    assert "REFERENCE_AUDIO_SECONDS = 6.0" in source
     assert reference_source is not None
     assert "extract_windowed_speaker_embedding(" not in reference_source
     assert "_get_ecapa_encoder(" not in reference_source
     assert "None," in reference_source
     assert "request_seed=REFERENCE_GENERATION_SEED" in reference_source
-    assert "reference_wav_path=reference_wav" in reference_source
+    assert "cfg: float = REFERENCE_CFG" in reference_source
+    assert "reference_wav_path=prepared_reference" in reference_source
+    assert "short_text_min_cfg=REFERENCE_CFG" in reference_source
     assert "retry" not in reference_source
     assert fast_source is not None
     assert 'kwargs["reference_wav_path"] = reference_wav_path' in fast_source
@@ -235,6 +240,38 @@ def test_reference_conditioning_uses_raw_reference_audio_once():
     assert "reference_wav_path=reference_wav_path" in synthesize_source
 
 
+def test_reference_audio_center_crop_matches_the_trained_six_second_cap(tmp_path):
+    prepare = _isolated_function(
+        "_prepare_reference_audio",
+        {
+            "os": os,
+            "sf": sf,
+            "tempfile": __import__("tempfile"),
+            "REFERENCE_AUDIO_SECONDS": 6.0,
+        },
+    )
+    sample_rate = 10
+    waveform = np.linspace(-0.5, 0.5, 100, dtype=np.float32)
+    source_path = tmp_path / "reference.wav"
+    sf.write(source_path, waveform, sample_rate, subtype="FLOAT")
+
+    prepared_path, temporary_path = prepare(str(source_path))
+    try:
+        prepared, prepared_rate = sf.read(prepared_path, dtype="float32")
+        assert prepared_rate == sample_rate
+        assert prepared.shape == (60,)
+        np.testing.assert_allclose(prepared, waveform[20:80], atol=1.0 / 32768.0)
+    finally:
+        assert temporary_path is not None
+        os.unlink(temporary_path)
+
+    short_path = tmp_path / "short.wav"
+    sf.write(short_path, waveform[:40], sample_rate, subtype="FLOAT")
+    unchanged_path, unchanged_temporary = prepare(str(short_path))
+    assert unchanged_path == str(short_path)
+    assert unchanged_temporary is None
+
+
 def test_reference_projector_gain_is_scoped_and_removed():
     source = APP_PATH.read_text(encoding="utf-8")
     synthesize_source = ast.get_source_segment(source, _function_node("_synthesize"))