Use raw reference audio for voice cloning

README.md

@@ -21,7 +21,7 @@ models:
 ## 模式
 
 - **內建語者**：預設為內建語者 A；Speaker B 使用模型庫 `checkpoints/speaker_b_embedding.pt`，固定 seed 56789 與 projector scale 1.125。
-- **參考音色**：以 3 秒窗、1.5 秒 hop（最多 4 個窗）平均成單一 speaker embedding，固定 centroid/projector scale 1.25 與 seed 1337；每個文字分段仍只生成一次。
+- **參考音色**：錄音直接送入模型原生的 raw reference-audio conditioning，不經 ECAPA centroid；固定 seed 1337，每個文字分段仍只生成一次。
 - **長文**：以最多 80 個字元快速分段，各分段單次生成後加入短靜音串接。
 
 TTS model snapshot 固定為

app.py

@@ -106,7 +106,6 @@ from production import (
     email_domain_mail_generation_variant,
     endpoint_generation_plan,
     estimate_step_seconds,
-    extract_windowed_speaker_embedding,
     fade_variable_internal_edges,
     finish_audio,
     join_audio_chunks_variable,
@@ -249,14 +248,6 @@ QUALITY_FAST_EXACT_MIN_SPEAKER_SIMILARITY = 0.23
 QUALITY_FAST_EXACT_MAX_BOUNDARY_SPEAKER_DROP = 0.08
 QUALITY_MAX_SEQUENCE_PATHS = 3
 SHORT_AUDIO_SPEAKER_GATE_SECONDS = 1.50
-REFERENCE_MIN_DURATION_SECONDS = 3.0
-# A short overlapping-window centroid is measurably closer to a recorded
-# reference than one whole-clip embedding.  Keep the cap small so extraction
-# remains bounded and generation itself is still exactly single-pass.
-REFERENCE_WINDOW_SECONDS = 3.0
-REFERENCE_HOP_SECONDS = 1.5
-REFERENCE_MAX_WINDOWS = 4
-REFERENCE_FULL_CLIP_MAX_SECONDS = 0.0
 SPEAKER_GENERATION_SEED = 56_789
 SPEAKER_B_LABEL = "內建語者 B"
 SPEAKER_B_PROJECTOR_SCALE = 1.125
@@ -264,8 +255,6 @@ SPEAKER_B_EMBEDDING_SHA256 = (
     "e9556e14723c140985a104c1659d1ff8a5078d2fa28ce2fb756f04906641a8a7"
 )
 REFERENCE_GENERATION_SEED = 1_337
-REFERENCE_CONDITIONING_SCALE = 1.25
-REFERENCE_PROJECTOR_SCALE = 1.25
 GLYPH_ASSET_MANIFEST_RELATIVE_PATH = "glyph_assets_v1/manifest.json"
 # The asset builder replaces this unavailable sentinel with the exact frozen
 # manifest SHA-256 before a candidate can expose the V5 profile.
@@ -2355,11 +2344,12 @@ def _fast_text_chunks(text: str) -> tuple[str, ...]:
 
 def _generate_fast_chunk(
     text: str,
-    centroid: torch.Tensor,
+    centroid: torch.Tensor | None,
     *,
     cfg: float,
     steps: int,
     request_seed: int,
+    reference_wav_path: str = "",
 ) -> np.ndarray:
     """Run one model call and return it without candidate qualification."""
 
@@ -2391,6 +2381,8 @@ def _generate_fast_chunk(
         "max_len": hard_stop_steps,
         "retry_badcase": False,
     }
+    if reference_wav_path:
+        kwargs["reference_wav_path"] = reference_wav_path
     if _NATIVE_STOP_POLICY:
         kwargs["stop_threshold"] = STOP_THRESHOLD
         kwargs["stop_consecutive"] = STOP_CONSECUTIVE
@@ -2431,7 +2423,7 @@ def _assemble_fast_audio(trajectory: tuple[np.ndarray, ...]) -> np.ndarray:
 
 def _synthesize(
     text: str,
-    centroid: torch.Tensor,
+    centroid: torch.Tensor | None,
     *,
     cfg: float,
     steps: int,
@@ -2439,6 +2431,7 @@ def _synthesize(
     generation_audit: _GlyphCarrierGenerationAudit | None = None,
     request_seed: int | None = None,
     speaker_projector_scale: float = 1.0,
+    reference_wav_path: str = "",
 ) -> tuple[int, np.ndarray]:
     """Latency-first single-pass synthesis with no output qualification."""
 
@@ -2471,6 +2464,7 @@ def _synthesize(
                     cfg=float(cfg),
                     steps=int(steps),
                     request_seed=seed,
+                    reference_wav_path=reference_wav_path,
                 )
                 for chunk in chunks
             )
@@ -3832,30 +3826,15 @@ def tts_reference(
 ):
     if not reference_wav:
         raise gr.Error("請先錄音或上傳參考音檔。")
-    try:
-        with latency_stage("ecapa"):
-            centroid = extract_windowed_speaker_embedding(
-                reference_wav,
-                _get_ecapa_encoder(),
-                device="cpu",
-                min_duration_seconds=REFERENCE_MIN_DURATION_SECONDS,
-                window_seconds=REFERENCE_WINDOW_SECONDS,
-                hop_seconds=REFERENCE_HOP_SECONDS,
-                max_windows=REFERENCE_MAX_WINDOWS,
-                full_clip_max_seconds=REFERENCE_FULL_CLIP_MAX_SECONDS,
-            )
-            centroid = centroid * REFERENCE_CONDITIONING_SCALE
-    except ValueError as error:
-        raise gr.Error(str(error)) from error
     return pcm16_audio_output(
         *_synthesize(
             text,
-            centroid,
+            None,
             cfg=cfg,
             steps=steps,
             speed=speed,
             request_seed=REFERENCE_GENERATION_SEED,
-            speaker_projector_scale=REFERENCE_PROJECTOR_SCALE,
+            reference_wav_path=reference_wav,
         )
     )
 
@@ -3949,7 +3928,7 @@ with gr.Blocks(title="BlueMagpie-TTS Demo", theme=gr.themes.Soft()) as demo:
     with gr.Tab("參考音色"):
         gr.Markdown(
             "建議錄製 6–12 秒、單一語者且背景乾淨的授權參考音檔；"
-            "至少需要 3 秒。"
+            "錄音會直接作為模型的 reference audio，不轉成 ECAPA 音色。"
         )
         with gr.Row():
             with gr.Column():

tests/test_latency_first_mode.py

@@ -203,28 +203,24 @@ def test_builtin_speaker_b_uses_the_pinned_checkpoint_embedding(tmp_path):
     )
 
 
-def test_reference_conditioning_uses_one_fixed_embedding():
+def test_reference_conditioning_uses_raw_reference_audio_once():
     source = APP_PATH.read_text(encoding="utf-8")
     reference_source = ast.get_source_segment(source, _function_node("tts_reference"))
+    fast_source = ast.get_source_segment(source, _function_node("_generate_fast_chunk"))
+    synthesize_source = ast.get_source_segment(source, _function_node("_synthesize"))
 
-    assert "REFERENCE_WINDOW_SECONDS = 3.0" in source
-    assert "REFERENCE_HOP_SECONDS = 1.5" in source
-    assert "REFERENCE_MAX_WINDOWS = 4" in source
-    assert "REFERENCE_FULL_CLIP_MAX_SECONDS = 0.0" in source
-    assert "REFERENCE_CONDITIONING_SCALE = 1.25" in source
     assert "REFERENCE_GENERATION_SEED = 1_337" in source
-    assert "REFERENCE_PROJECTOR_SCALE = 1.25" in source
     assert reference_source is not None
-    assert "extract_windowed_speaker_embedding(" in reference_source
-    assert "min_duration_seconds=REFERENCE_MIN_DURATION_SECONDS" in reference_source
-    assert "window_seconds=REFERENCE_WINDOW_SECONDS" in reference_source
-    assert "hop_seconds=REFERENCE_HOP_SECONDS" in reference_source
-    assert "max_windows=REFERENCE_MAX_WINDOWS" in reference_source
-    assert "full_clip_max_seconds=REFERENCE_FULL_CLIP_MAX_SECONDS" in reference_source
-    assert "centroid = centroid * REFERENCE_CONDITIONING_SCALE" in reference_source
+    assert "extract_windowed_speaker_embedding(" not in reference_source
+    assert "_get_ecapa_encoder(" not in reference_source
+    assert "None," in reference_source
     assert "request_seed=REFERENCE_GENERATION_SEED" in reference_source
-    assert "speaker_projector_scale=REFERENCE_PROJECTOR_SCALE" in reference_source
+    assert "reference_wav_path=reference_wav" in reference_source
     assert "retry" not in reference_source
+    assert fast_source is not None
+    assert 'kwargs["reference_wav_path"] = reference_wav_path' in fast_source
+    assert synthesize_source is not None
+    assert "reference_wav_path=reference_wav_path" in synthesize_source
 
 
 def test_reference_projector_gain_is_scoped_and_removed():