diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index e548fd43199e453494b415ae8584c90de682cb1b..0000000000000000000000000000000000000000 --- a/.gitattributes +++ /dev/null @@ -1,55 +0,0 @@ -*.7z filter=lfs diff=lfs merge=lfs -text -*.arrow filter=lfs diff=lfs merge=lfs -text -*.bin filter=lfs diff=lfs merge=lfs -text -*.bz2 filter=lfs diff=lfs merge=lfs -text -*.ckpt filter=lfs diff=lfs merge=lfs -text -*.ftz filter=lfs diff=lfs merge=lfs -text -*.gz filter=lfs diff=lfs merge=lfs -text -*.h5 filter=lfs diff=lfs merge=lfs -text -*.joblib filter=lfs diff=lfs merge=lfs -text -*.lfs.* filter=lfs diff=lfs merge=lfs -text -*.mlmodel filter=lfs diff=lfs merge=lfs -text -*.model filter=lfs diff=lfs merge=lfs -text -*.msgpack filter=lfs diff=lfs merge=lfs -text -*.npy filter=lfs diff=lfs merge=lfs -text -*.npz filter=lfs diff=lfs merge=lfs -text -*.onnx filter=lfs diff=lfs merge=lfs -text -*.ot filter=lfs diff=lfs merge=lfs -text -*.parquet filter=lfs diff=lfs merge=lfs -text -*.pb filter=lfs diff=lfs merge=lfs -text -*.pickle filter=lfs diff=lfs merge=lfs -text -*.pkl filter=lfs diff=lfs merge=lfs -text -*.pt filter=lfs diff=lfs merge=lfs -text -*.pth filter=lfs diff=lfs merge=lfs -text -*.rar filter=lfs diff=lfs merge=lfs -text -*.safetensors filter=lfs diff=lfs merge=lfs -text -saved_model/**/* filter=lfs diff=lfs merge=lfs -text -*.tar.* filter=lfs diff=lfs merge=lfs -text -*.tar filter=lfs diff=lfs merge=lfs -text -*.tflite filter=lfs diff=lfs merge=lfs -text -*.tgz filter=lfs diff=lfs merge=lfs -text -*.wasm filter=lfs diff=lfs merge=lfs -text -*.xz filter=lfs diff=lfs merge=lfs -text -*.zip filter=lfs diff=lfs merge=lfs -text -*.zst filter=lfs diff=lfs merge=lfs -text -*tfevents* filter=lfs diff=lfs merge=lfs -text -sample/Bình[[:space:]](nam[[:space:]]miền[[:space:]]Bắc).wav filter=lfs diff=lfs merge=lfs -text -sample/Dung[[:space:]](nữ[[:space:]]miền[[:space:]]Nam).wav filter=lfs diff=lfs merge=lfs -text -sample/Đoan[[:space:]](nữ[[:space:]]miền[[:space:]]Nam).wav filter=lfs diff=lfs merge=lfs -text -sample/Hương[[:space:]](nữ[[:space:]]miền[[:space:]]Bắc).wav filter=lfs diff=lfs merge=lfs -text -sample/Ly[[:space:]](nữ[[:space:]]miền[[:space:]]Bắc).wav filter=lfs diff=lfs merge=lfs -text -sample/Ngọc[[:space:]](nữ[[:space:]]miền[[:space:]]Bắc).wav filter=lfs diff=lfs merge=lfs -text -sample/Nguyên[[:space:]](nam[[:space:]]miền[[:space:]]Nam).wav filter=lfs diff=lfs merge=lfs -text -sample/Sơn[[:space:]](nam[[:space:]]miền[[:space:]]Nam).wav filter=lfs diff=lfs merge=lfs -text -sample/Tuyên[[:space:]](nam[[:space:]]miền[[:space:]]Bắc).wav filter=lfs diff=lfs merge=lfs -text -sample/Vĩnh[[:space:]](nam[[:space:]]miền[[:space:]]Nam).wav filter=lfs diff=lfs merge=lfs -text -utils/phoneme_dict.json filter=lfs diff=lfs merge=lfs -text -examples/audio_ref/example_2.wav filter=lfs diff=lfs merge=lfs -text -examples/audio_ref/example_4.wav filter=lfs diff=lfs merge=lfs -text -examples/audio_ref/example.wav filter=lfs diff=lfs merge=lfs -text -src/vieneu/assets/samples/Bình[[:space:]](nam[[:space:]]miền[[:space:]]Bắc).wav filter=lfs diff=lfs merge=lfs -text -src/vieneu/assets/samples/Đoan[[:space:]](nữ[[:space:]]miền[[:space:]]Nam).wav filter=lfs diff=lfs merge=lfs -text -src/vieneu/assets/samples/Ly[[:space:]](nữ[[:space:]]miền[[:space:]]Bắc).wav filter=lfs diff=lfs merge=lfs -text -src/vieneu/assets/samples/Ngọc[[:space:]](nữ[[:space:]]miền[[:space:]]Bắc).wav filter=lfs diff=lfs merge=lfs -text -src/vieneu/assets/samples/Tuyên[[:space:]](nam[[:space:]]miền[[:space:]]Bắc).wav filter=lfs diff=lfs merge=lfs -text -src/vieneu/assets/samples/Vĩnh[[:space:]](nam[[:space:]]miền[[:space:]]Nam).wav filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md index 6eae0eb1dcd025c12d3b24e7c822bb05085e9f3d..cdb4178be4fb169e8f4637d938cfd1460b614211 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,39 @@ --- -title: VieNeu-TTS-v2-Turbo +title: VieNeu-TTS v3 Turbo emoji: 🦜 -colorFrom: pink -colorTo: yellow +colorFrom: indigo +colorTo: blue sdk: gradio -sdk_version: 6.2.0 +sdk_version: 5.49.1 +python_version: "3.12" app_file: app.py -pinned: true +pinned: false license: apache-2.0 -short_description: Demo for VieNeu-TTS-0.3B +short_description: Vietnamese TTS · 48kHz · giọng dựng sẵn + nhân bản giọng +models: + - pnnbao-ump/VieNeu-TTS-v3-Turbo + - OpenMOSS-Team/MOSS-Audio-Tokenizer-Nano +tags: + - text-to-speech + - tts + - vietnamese + - voice-cloning --- -Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference \ No newline at end of file +# 🦜 VieNeu-TTS v3 Turbo + +Text-to-Speech tiếng Việt, **48 kHz**, với giọng dựng sẵn và **nhân bản giọng tức thì** +từ một đoạn mẫu 3–5 giây. Chạy trên **ZeroGPU** (PyTorch / CUDA). + +- Model: [`pnnbao-ump/VieNeu-TTS-v3-Turbo`](https://huggingface.co/pnnbao-ump/VieNeu-TTS-v3-Turbo) +- Mã nguồn: [github.com/pnnbao97/VieNeu-TTS](https://github.com/pnnbao97/VieNeu-TTS) + +## Tính năng +- 10 giọng dựng sẵn (nam/nữ, nhiều sắc thái). +- Nhân bản giọng từ audio mẫu (tab *Nhân bản giọng*). +- Tag cảm xúc thử nghiệm chèn trong văn bản: `[cười]`, `[thở dài]`, `[hắng giọng]`. + +## ⚙️ Lưu ý cấu hình Space +Vào **Settings → Hardware** của Space và chọn **ZeroGPU** +(cần tài khoản PRO hoặc tổ chức Team/Enterprise để bật ZeroGPU). +GPU chỉ được cấp phát khi hàm `@spaces.GPU` chạy; phần còn lại chạy trên CPU. diff --git a/app.py b/app.py index fb10ef2ce7be8fc9334661edcd4ab27fdbc58fc4..82f689569cf2d81fe29fc48cccb6bf0e170948c2 100644 --- a/app.py +++ b/app.py @@ -1,192 +1,186 @@ -import spaces # MUST be before any other imports +""" +VieNeu-TTS v3 Turbo — Hugging Face ZeroGPU Space +================================================ +Vietnamese text-to-speech, 48 kHz, with built-in voices + instant voice cloning. + +ZeroGPU notes: + * The model is placed on ``cuda`` at module import (ZeroGPU runs a CUDA + emulation outside ``@spaces.GPU`` so this is the recommended, fastest path). + * Real GPU compute happens only inside the ``@spaces.GPU`` decorated function. +""" import os -import sys - -# Support directory structure on HF Spaces: Add 'src' to search path -src_path = os.path.join(os.path.dirname(__file__), "src") -if os.path.exists(src_path) and src_path not in sys.path: - sys.path.append(src_path) +import numpy as np import gradio as gr -import soundfile as sf -import tempfile -import torch +import spaces + from vieneu import Vieneu -import time -import numpy as np -print("⏳ Starting VieNeu-TTS Studio v2 (HF Space Edition)...") +# ── Load model once, on GPU (CUDA emulation makes this valid at startup) ─────── +print("⏳ Loading VieNeu-TTS v3 Turbo (PyTorch / CUDA) ...") +tts = Vieneu( + mode="v3turbo", + device="cuda", # ZeroGPU: keep weights on cuda from the start + backend="pytorch", # force the PyTorch engine (ONNX is the CPU-only path) + hf_token=os.getenv("HF_TOKEN"), +) +print("✅ Model ready.") -# --- 1. SETUP MODEL --- -device = "cuda" if torch.cuda.is_available() else "cpu" -print(f"🖥️ Using device: {device.upper()}") +PRESET_VOICES = tts.list_preset_voices() # [(label, voice_id), ...] +VOICE_CHOICES = [(label, vid) for label, vid in PRESET_VOICES] +DEFAULT_VOICE = tts._default_voice or (VOICE_CHOICES[0][1] if VOICE_CHOICES else None) -try: - # Initialize with turbo_gpu mode for V2-Turbo - tts = Vieneu( - mode="turbo_gpu", - backbone_repo="pnnbao-ump/VieNeu-TTS-v2-Turbo", - decoder_repo="pnnbao-ump/VieNeu-Codec", - device=device, - backend="standard" # Standard (Transformers) is more stable for ZeroGPU tasks - ) - print("✅ VieNeu-TTS-v2-Turbo initialized successfully!") -except Exception as e: - print(f"⚠️ Warning: Failed to load model: {e}") - import traceback - traceback.print_exc() - # Mock for UI testing if model can't be loaded (e.g. during build) - class MockTTS: - def list_preset_voices(self): return [("Lỗi tải model", "error")] - def get_preset_voice(self, v): return {"codes": np.zeros((1, 128)), "text": ""} - def encode_reference(self, path): return np.zeros((1, 128)) - def infer(self, text, **kwargs): return np.random.uniform(-0.1, 0.1, 24000) - tts = MockTTS() - -# --- 2. DATA --- -try: - PRESET_VOICES = tts.list_preset_voices() - VOICE_CHOICES = [v[0] for v in PRESET_VOICES] - VOICE_MAP = {v[0]: v[1] for v in PRESET_VOICES} -except: - VOICE_CHOICES = ["Xuân Vĩnh (nam miền Nam)"] - VOICE_MAP = {"Xuân Vĩnh (nam miền Nam)": "xuan_vinh"} - -# --- 3. INFERENCE --- -@spaces.GPU(duration=60) -def synthesize_speech(text, voice_choice, custom_audio, mode_tab, temperature): - global tts - if tts is None: - # Re-initialize tts if it is lost in the ZeroGPU worker context - from vieneu import Vieneu - tts = Vieneu( - mode="turbo_gpu", - backbone_repo="pnnbao-ump/VieNeu-TTS-v2-Turbo", - decoder_repo="pnnbao-ump/VieNeu-Codec", - device="cuda" - ) - - try: - if not text or not text.strip(): - return None, "⚠️ Vui lòng nhập văn bản cần tổng hợp!" - - if len(text) > 600: - return None, f"❌ Văn bản quá dài ({len(text)}/600 ký tự)!" - - # Handle Reference Logic - if mode_tab == "custom_mode": - if custom_audio is None: - return None, "⚠️ Vui lòng tải lên Audio để Voice Cloning." - # Voice Cloning on Turbo GPU: encode_reference returns the embedding vector - ref_codes = tts.encode_reference(custom_audio) - ref_text_raw = "" # Turbo v2 doesn't require reference text - else: # Preset mode - actual_voice_id = VOICE_MAP.get(voice_choice) - voice_data = tts.get_preset_voice(actual_voice_id) - ref_codes = voice_data['codes'] - ref_text_raw = voice_data['text'] - - # Start Inference - start_time = time.time() - # V2 Turbo uses specific infer signatures - wav = tts.infer( - text, - ref_codes=ref_codes, - temperature=float(temperature), - skip_normalize=False # Let the model handle normalization - ) - process_time = time.time() - start_time - - # Save to temporary file - with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as tmp_file: - sf.write(tmp_file.name, wav, 24000) - output_path = tmp_file.name - - rtf = process_time / (len(wav) / 24000) - return output_path, f"✅ Tổng hợp xong! | Thời gian: {process_time:.2f}s | RTF: {rtf:.3f}" - - except Exception as e: - import traceback - traceback.print_exc() - return None, f"❌ Lỗi: {str(e)}" - -# --- 4. UI --- -theme = gr.themes.Soft( - primary_hue="indigo", - secondary_hue="blue", - neutral_hue="slate", - font=[gr.themes.GoogleFont('Inter'), 'system-ui'] -).set( - button_primary_background_fill="linear-gradient(90deg, #4f46e5 0%, #3b82f6 100%)", - block_shadow="0 10px 15px -3px rgba(0, 0, 0, 0.1)", +EMOTIONS = [("Tự nhiên", "natural"), ("Kể chuyện", "storytelling")] + +DEFAULT_TEXT = ( + "Mình từng nghĩ giọng nói AI bây giờ nghe kiểu gì cũng bị đơ đơ, máy móc... " + "nhưng mà VieNeu xuất hiện làm mình thay đổi hẳn 180 độ luôn á! [cười]\n\n" + "Trời ơi, cái giọng nó tự nhiên mà nó mượt mà dã man, nghe không khác gì người thật luôn. " + "Giờ thì tha hồ mà quẩy content với cả kho giọng nói đa dạng, đủ mọi sắc thái biểu cảm. " + "Mọi người bật loa lên rồi cùng trải nghiệm thử với mình nhé!" ) -css = """ -.container { max-width: 1000px; margin: auto; } -.header { text-align: center; margin-bottom: 30px; padding: 40px; background: #0f172a; border-radius: 16px; color: white; } -.header h1 { font-size: 3rem; margin-bottom: 10px; background: linear-gradient(90deg, #818cf8, #38bdf8); -webkit-background-clip: text; -webkit-text-fill-color: transparent; } -.header p { font-size: 1.1rem; opacity: 0.8; } -.status-box { background: transparent !important; border: none !important; font-weight: bold; text-align: center; } -.footer { text-align: center; margin-top: 30px; font-size: 0.9rem; opacity: 0.6; } + +def _gpu_duration(text, *args, **kwargs): + """Dynamic ZeroGPU budget: scale with text length, capped at 3 minutes.""" + n = len(text or "") + return int(min(180, 30 + n // 8)) + + +@spaces.GPU(duration=_gpu_duration) +def synthesize( + text, + voice, + ref_audio, + emotion, + temperature, + top_k, + top_p, + repetition_penalty, + max_new_frames, + max_chars, +): + text = (text or "").strip() + if not text: + raise gr.Error("Vui lòng nhập văn bản cần đọc.") + + kwargs = dict( + emotion=emotion, + temperature=float(temperature), + top_k=int(top_k), + top_p=float(top_p), + repetition_penalty=float(repetition_penalty), + max_new_frames=int(max_new_frames), + max_chars=int(max_chars), + ) + + # An uploaded reference clip takes precedence → voice cloning. + # Otherwise use the selected built-in voice (speaker-token path). + if ref_audio: + wav = tts.infer(text, ref_audio=ref_audio, **kwargs) + else: + wav = tts.infer(text, voice=voice, **kwargs) + + wav = np.asarray(wav, dtype=np.float32) + return (tts.sample_rate, wav) + + +HEADER = """ +
+
🦜 VieNeu-TTS v3 Turbo
+
+ Text-to-Speech tiếng Việt · 48 kHz · giọng dựng sẵn + nhân bản giọng tức thì +
+
""" -with gr.Blocks(theme=theme, css=css, title="VieNeu-TTS Studio") as demo: - - with gr.Column(elem_classes="container"): - gr.HTML(""" -
-

🦜 VieNeu-TTS v2 Turbo

-

The fastest Vietnamese-English Bilingual TTS engine with Instant Voice Cloning

-
- 🤗 Model • - 💬 Discord • - 🦜 GitHub -
-
- """) - - with gr.Row(): - with gr.Column(scale=3): - text_input = gr.Textbox( - label="Văn bản (Nhập Tiếng Việt hoặc Tiếng Anh)", - placeholder="Hello! Chào mừng bạn đến với VieNeu-TTS v2...", - lines=6, - value="Vật lý lượng tử là một trong những lĩnh vực phức tạp và thú vị nhất của khoa học hiện đại. Quantum physics studies how particles behave at extremely small scales, nơi mà các quy luật cổ điển không còn đúng nữa." +GUIDE = ( + "**Mẹo:** chèn tag cảm xúc ngay trong văn bản (thử nghiệm): " + "`[cười]`, `[thở dài]`, `[hắng giọng]`.\n\n" + "**Nhân bản giọng:** tải lên một đoạn mẫu **3–5 giây** ở tab *Nhân bản giọng* — " + "khi có audio mẫu, hệ thống sẽ ưu tiên dùng nó thay cho giọng dựng sẵn." +) + +theme = gr.themes.Soft(primary_hue="indigo", secondary_hue="cyan", neutral_hue="slate") + +with gr.Blocks(theme=theme, title="VieNeu-TTS v3 Turbo") as demo: + gr.HTML(HEADER) + gr.Markdown(GUIDE) + + with gr.Row(): + with gr.Column(scale=3): + text_in = gr.Textbox( + label="Văn bản", + value=DEFAULT_TEXT, + lines=8, + placeholder="Nhập văn bản tiếng Việt...", + ) + with gr.Tabs(): + with gr.Tab("Giọng dựng sẵn"): + voice_in = gr.Dropdown( + label="Chọn giọng", + choices=VOICE_CHOICES, + value=DEFAULT_VOICE, + ) + with gr.Tab("Nhân bản giọng"): + ref_audio_in = gr.Audio( + label="Audio mẫu (3–5 giây)", + type="filepath", + sources=["upload", "microphone"], + ) + gr.Markdown( + "_Có audio mẫu ở đây sẽ **ghi đè** giọng dựng sẵn. " + "Xoá audio để quay lại giọng dựng sẵn._" + ) + + with gr.Accordion("Tuỳ chọn nâng cao", open=False): + emotion_in = gr.Dropdown( + label="Sắc thái (áp dụng khi nhân bản giọng)", + choices=EMOTIONS, + value="natural", ) - - with gr.Tabs() as tabs: - with gr.TabItem("👤 Preset Voice", id="preset_mode") as tab_preset: - voice_select = gr.Dropdown(choices=VOICE_CHOICES, value=VOICE_CHOICES[0] if VOICE_CHOICES else None, label="Chọn nhân vật") - with gr.TabItem("🦜 Voice Cloning", id="custom_mode") as tab_custom: - gr.Markdown("💡 **Mẹo:** Tải lên audio giọng nói rõ ràng, độ dài từ 5-10 giây để có kết quả tốt nhất.") - custom_audio = gr.Audio(label="Audio mẫu (.wav / .mp3)", type="filepath") - - with gr.Accordion("⚙️ Cài đặt nâng cao", open=False): - temperature = gr.Slider(minimum=0.1, maximum=1.0, value=0.4, step=0.1, label="Độ sáng tạo (Temperature)") - - current_mode = gr.State(value="preset_mode") - btn_generate = gr.Button("⚡ Bắt đầu tổng hợp", variant="primary", size="lg") - - with gr.Column(scale=2): - audio_output = gr.Audio(label="Audio kết quả", type="filepath", autoplay=True) - status_output = gr.Textbox(label="Trạng thái", show_label=False, elem_classes="status-box") - - gr.Markdown(""" - ### ✨ Tính năng mới trên V2-Turbo: - - 🇻🇳 **Song ngữ Việt - Anh**: Tự động nhận diện và đọc code-switching. - - 🚀 **Nhanh gấp 10 lần**: Tối ưu hóa cực mạnh cho cả CPU và GPU. - - 🦜 **Cloning chất lượng cao**: Khả năng bắt chước ngữ điệu chuẩn xác hơn. - """) - - # Logic Tab switching - tab_preset.select(fn=lambda: "preset_mode", outputs=current_mode) - tab_custom.select(fn=lambda: "custom_mode", outputs=current_mode) - - btn_generate.click( - fn=synthesize_speech, - inputs=[text_input, voice_select, custom_audio, current_mode, temperature], - outputs=[audio_output, status_output] + with gr.Row(): + temperature_in = gr.Slider(0.1, 1.5, value=0.8, step=0.05, label="temperature") + top_p_in = gr.Slider(0.1, 1.0, value=0.95, step=0.01, label="top_p") + with gr.Row(): + top_k_in = gr.Slider(1, 100, value=25, step=1, label="top_k") + rep_pen_in = gr.Slider(1.0, 2.0, value=1.2, step=0.05, label="repetition_penalty") + with gr.Row(): + max_frames_in = gr.Slider( + 50, 1200, value=300, step=10, label="max_new_frames (mỗi đoạn)" + ) + max_chars_in = gr.Slider( + 64, 400, value=256, step=8, label="max_chars (cắt đoạn)" + ) + + run_btn = gr.Button("🔊 Tạo giọng nói", variant="primary") + + with gr.Column(scale=2): + audio_out = gr.Audio(label="Kết quả", type="numpy", autoplay=False) + gr.Markdown( + "Model: [pnnbao-ump/VieNeu-TTS-v3-Turbo]" + "(https://huggingface.co/pnnbao-ump/VieNeu-TTS-v3-Turbo) · " + "Code: [github.com/pnnbao97/VieNeu-TTS](https://github.com/pnnbao97/VieNeu-TTS)" + ) + + inputs = [ + text_in, voice_in, ref_audio_in, emotion_in, + temperature_in, top_k_in, top_p_in, rep_pen_in, + max_frames_in, max_chars_in, + ] + run_btn.click(fn=synthesize, inputs=inputs, outputs=audio_out) + + gr.Examples( + examples=[ + [DEFAULT_TEXT, DEFAULT_VOICE], + ["Xin chào, đây là giọng đọc tiếng Việt tự nhiên từ VieNeu-TTS.", DEFAULT_VOICE], + ], + inputs=[text_in, voice_in], ) + if __name__ == "__main__": - demo.queue().launch() \ No newline at end of file + demo.queue().launch() diff --git a/config.yaml b/config.yaml deleted file mode 100644 index dad5193d718ce83ee0ae6978b4b1ddd8a305df2d..0000000000000000000000000000000000000000 --- a/config.yaml +++ /dev/null @@ -1,77 +0,0 @@ -text_settings: - max_chars_per_chunk: 256 - max_total_chars_streaming: 3000 - -backbone_configs: - "VieNeu-TTS (GPU)": - repo: pnnbao-ump/VieNeu-TTS - supports_streaming: false - description: Chất lượng cao nhất, yêu cầu GPU - "VieNeu-TTS-0.3B (GPU)": - repo: pnnbao-ump/VieNeu-TTS-0.3B - supports_streaming: false - description: Phiên bản nhẹ cho GPU, tốc độ nhanh x2 so với phiên bản gốc - "VieNeu-TTS-q8-gguf": - repo: pnnbao-ump/VieNeu-TTS-q8-gguf - supports_streaming: true - description: Phiên bản GGUF có chất lượng cao nhất - "VieNeu-TTS-q4-gguf": - repo: pnnbao-ump/VieNeu-TTS-q4-gguf - supports_streaming: true - description: Cân bằng giữa chất lượng và tốc độ - "VieNeu-TTS-0.3B-q4-gguf": - repo: pnnbao-ump/VieNeu-TTS-0.3B-q4-gguf - supports_streaming: true - description: Phiên bản cực nhẹ, chạy mượt trên CPU - -codec_configs: - "NeuCodec (Standard)": - repo: neuphonic/neucodec - description: Codec chuẩn, tốc độ trung bình - use_preencoded: false - "NeuCodec (Distill)": - repo: neuphonic/distill-neucodec - description: Codec tối ưu, tốc độ cao - use_preencoded: false - "NeuCodec ONNX (Fast CPU)": - repo: neuphonic/neucodec-onnx-decoder-int8 - description: Tối ưu cho CPU, cần pre-encoded codes - use_preencoded: true - -voice_samples: - "Tuyên (nam miền Bắc)": - audio: ./sample/Tuyên (nam miền Bắc).wav - text: ./sample/Tuyên (nam miền Bắc).txt - codes: ./sample/Tuyên (nam miền Bắc).pt - "Vĩnh (nam miền Nam)": - audio: ./sample/Vĩnh (nam miền Nam).wav - text: ./sample/Vĩnh (nam miền Nam).txt - codes: ./sample/Vĩnh (nam miền Nam).pt - "Bình (nam miền Bắc)": - audio: ./sample/Bình (nam miền Bắc).wav - text: ./sample/Bình (nam miền Bắc).txt - codes: ./sample/Bình (nam miền Bắc).pt - "Nguyên (nam miền Nam)": - audio: ./sample/Nguyên (nam miền Nam).wav - text: ./sample/Nguyên (nam miền Nam).txt - codes: ./sample/Nguyên (nam miền Nam).pt - "Sơn (nam miền Nam)": - audio: ./sample/Sơn (nam miền Nam).wav - text: ./sample/Sơn (nam miền Nam).txt - codes: ./sample/Sơn (nam miền Nam).pt - "Đoan (nữ miền Nam)": - audio: ./sample/Đoan (nữ miền Nam).wav - text: ./sample/Đoan (nữ miền Nam).txt - codes: ./sample/Đoan (nữ miền Nam).pt - "Ngọc (nữ miền Bắc)": - audio: ./sample/Ngọc (nữ miền Bắc).wav - text: ./sample/Ngọc (nữ miền Bắc).txt - codes: ./sample/Ngọc (nữ miền Bắc).pt - "Ly (nữ miền Bắc)": - audio: ./sample/Ly (nữ miền Bắc).wav - text: ./sample/Ly (nữ miền Bắc).txt - codes: ./sample/Ly (nữ miền Bắc).pt - "Dung (nữ miền Nam)": - audio: ./sample/Dung (nữ miền Nam).wav - text: ./sample/Dung (nữ miền Nam).txt - codes: ./sample/Dung (nữ miền Nam).pt diff --git a/examples/audio_ref/example.txt b/examples/audio_ref/example.txt deleted file mode 100644 index dd256a7c427fca01ad4b9479ed7d16fd5158b7b0..0000000000000000000000000000000000000000 --- a/examples/audio_ref/example.txt +++ /dev/null @@ -1 +0,0 @@ -ví dụ 2. tính trung bình của dãy số. \ No newline at end of file diff --git a/examples/audio_ref/example.wav b/examples/audio_ref/example.wav deleted file mode 100644 index 9469beafee622cc900490bd5a9bb180666962fe5..0000000000000000000000000000000000000000 --- a/examples/audio_ref/example.wav +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a723ba224a2f98f421a8fd6e850f0f5989ec59d9d7d9ca2f2710b5e7caf73b2c -size 118862 diff --git a/examples/audio_ref/example_2.txt b/examples/audio_ref/example_2.txt deleted file mode 100644 index c9fe64610c3df7690ef83364642d57b767081745..0000000000000000000000000000000000000000 --- a/examples/audio_ref/example_2.txt +++ /dev/null @@ -1 +0,0 @@ -Trên thực tế, các nghi ngờ đã bắt đầu xuất hiện. \ No newline at end of file diff --git a/examples/audio_ref/example_2.wav b/examples/audio_ref/example_2.wav deleted file mode 100644 index 1a2d8bb35da9d0bc610c0a942d64e894ea766430..0000000000000000000000000000000000000000 --- a/examples/audio_ref/example_2.wav +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8f6733df04e5f3477a00136c6baeaf7a196c93df0ee13b9bfa3d8ba61034f063 -size 174044 diff --git a/examples/audio_ref/example_3.txt b/examples/audio_ref/example_3.txt deleted file mode 100644 index e79ef0356e2508ab4fb72e62b7f51a840fa61c01..0000000000000000000000000000000000000000 --- a/examples/audio_ref/example_3.txt +++ /dev/null @@ -1 +0,0 @@ -Cậu có nhìn thấy không? \ No newline at end of file diff --git a/examples/audio_ref/example_3.wav b/examples/audio_ref/example_3.wav deleted file mode 100644 index 8e1a0f737ba94f94c119767cbc6d5af8118c496b..0000000000000000000000000000000000000000 Binary files a/examples/audio_ref/example_3.wav and /dev/null differ diff --git a/examples/audio_ref/example_4.txt b/examples/audio_ref/example_4.txt deleted file mode 100644 index 1c6458927ad1903ea6c6ae9363ce6820044c981a..0000000000000000000000000000000000000000 --- a/examples/audio_ref/example_4.txt +++ /dev/null @@ -1 +0,0 @@ -Tết là dịp mọi người háo hức đón chào một năm mới với nhiều hy vọng và mong ước. \ No newline at end of file diff --git a/examples/audio_ref/example_4.wav b/examples/audio_ref/example_4.wav deleted file mode 100644 index cbd88bdebffa87c9e5278aa2d98f28acd9272de6..0000000000000000000000000000000000000000 --- a/examples/audio_ref/example_4.wav +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:bd817540edf7b34744b305c2966ac16b071729b2bab18e9bdd4f38b665b030b0 -size 192556 diff --git a/examples/encode_ref_audio.py b/examples/encode_ref_audio.py deleted file mode 100644 index b7512c10559c8d605d6ebf1ddfb791c6295d68b1..0000000000000000000000000000000000000000 --- a/examples/encode_ref_audio.py +++ /dev/null @@ -1,43 +0,0 @@ -import torch -from librosa import load -from neucodec import NeuCodec - -def main(ref_audio_path, output_path="output.pt"): - print("Encoding reference audio") - - # Make sure output path ends with .pt - if not output_path.endswith(".pt"): - print("Output path should end with .pt to save the codes.") - return - - # Initialize codec - codec = NeuCodec.from_pretrained("neuphonic/neucodec") - codec.eval().to("cpu") - - # Load and encode reference audio - wav, _ = load(ref_audio_path, sr=16000, mono=True) # load as 16kHz - wav_tensor = torch.from_numpy(wav).float().unsqueeze(0).unsqueeze(0) # [1, 1, T] - ref_codes = codec.encode_code(audio_or_path=wav_tensor).squeeze(0).squeeze(0) - - # Save the codes - torch.save(ref_codes, output_path) - - -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser(description="NeuTTSAir Reference Encoding Example") - parser.add_argument( - "--ref_audio", type=str, default="./sample/Vĩnh (nam miền Nam).wav", help="Path to reference audio" - ) - parser.add_argument( - "--output_path", - type=str, - default="encoded_reference.pt", - help="Path to save the output codes", - ) - args = parser.parse_args() - main( - ref_audio_path=args.ref_audio, - output_path=args.output_path, - ) \ No newline at end of file diff --git a/examples/infer_long_text.py b/examples/infer_long_text.py deleted file mode 100644 index 80b08988ce0722b5105d2840b1a700b12737b250..0000000000000000000000000000000000000000 --- a/examples/infer_long_text.py +++ /dev/null @@ -1,223 +0,0 @@ -import argparse -import os -import re -import sys -from pathlib import Path -from typing import List -import numpy as np -import soundfile as sf -import torch -from vieneu_tts import VieNeuTTS - - -def split_text_into_chunks(text: str, max_chars: int = 256) -> List[str]: - """ - Split raw text into chunks no longer than max_chars. - Preference is given to sentence boundaries; otherwise falls back to word-based splitting. - """ - sentences = re.split(r"(?<=[\.\!\?\…])\s+", text.strip()) - chunks: List[str] = [] - buffer = "" - - def flush_buffer(): - nonlocal buffer - if buffer: - chunks.append(buffer.strip()) - buffer = "" - - for sentence in sentences: - sentence = sentence.strip() - if not sentence: - continue - - # If single sentence already fits, try to append to current buffer - if len(sentence) <= max_chars: - candidate = f"{buffer} {sentence}".strip() if buffer else sentence - if len(candidate) <= max_chars: - buffer = candidate - else: - flush_buffer() - buffer = sentence - continue - - # Fallback: sentence too long, break by words - flush_buffer() - words = sentence.split() - current = "" - for word in words: - candidate = f"{current} {word}".strip() if current else word - if len(candidate) > max_chars and current: - chunks.append(current.strip()) - current = word - else: - current = candidate - if current: - chunks.append(current.strip()) - - flush_buffer() - return [chunk for chunk in chunks if chunk] - - -def infer_long_text( - text: str, - ref_audio_path: str, - ref_text_path: str, - output_path: str, - chunk_dir: str | None = None, - max_chars: int = 256, - backbone_repo: str = "pnnbao-ump/VieNeu-TTS", - codec_repo: str = "neuphonic/neucodec", - device: str | None = None, -) -> str: - """ - Generate speech for long-form text by chunking into manageable segments. - - Returns: - The path to the combined audio file. - """ - - device = device or ("cuda" if torch.cuda.is_available() else "cpu") - if device not in {"cuda", "cpu"}: - raise ValueError("Device must be either 'cuda' or 'cpu'.") - - raw_text = text.strip() - if not raw_text: - raise ValueError("Input text is empty.") - - chunks = split_text_into_chunks(raw_text, max_chars=max_chars) - if not chunks: - raise ValueError("Text could not be segmented into valid chunks.") - - print(f"📄 Total chunks: {len(chunks)} (≤ {max_chars} chars each)") - - if chunk_dir: - os.makedirs(chunk_dir, exist_ok=True) - - ref_text_raw = Path(ref_text_path).read_text(encoding="utf-8") - - tts = VieNeuTTS( - backbone_repo=backbone_repo, - backbone_device=device, - codec_repo=codec_repo, - codec_device=device, - ) - - print("🎧 Encoding reference audio...") - ref_codes = tts.encode_reference(ref_audio_path) - - generated_segments: List[np.ndarray] = [] - - for idx, chunk in enumerate(chunks, start=1): - print(f"🎙️ Chunk {idx}/{len(chunks)} | {len(chunk)} chars") - wav = tts.infer(chunk, ref_codes, ref_text_raw) - generated_segments.append(wav) - - if chunk_dir: - chunk_path = os.path.join(chunk_dir, f"chunk_{idx:03d}.wav") - sf.write(chunk_path, wav, 24_000) - - combined_audio = np.concatenate(generated_segments) - os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True) - sf.write(output_path, combined_audio, 24_000) - - print(f"✅ Saved combined audio to: {output_path}") - return output_path - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Infer long text with VieNeu-TTS") - text_group = parser.add_mutually_exclusive_group(required=True) - text_group.add_argument( - "--text", - help="Raw UTF-8 text content to synthesize.", - ) - text_group.add_argument( - "--text-file", - help="Path to a UTF-8 text file to synthesize.", - ) - parser.add_argument( - "--ref-audio", - default="./sample/Vĩnh (nam miền Nam).wav", - help="Path to reference audio (.wav). Default: ./sample/Vĩnh (nam miền Nam).wav" - ) - parser.add_argument( - "--ref-text", - default="./sample/Vĩnh (nam miền Nam).txt", - help="Path to reference text (UTF-8). Default: ./sample/Vĩnh (nam miền Nam).txt" - ) - parser.add_argument( - "--output", - default="./output_audio/long_text.wav", - help="Path to save the combined audio output.", - ) - parser.add_argument( - "--chunk-output-dir", - default=None, - help="Optional directory to save individual chunk audio files.", - ) - parser.add_argument( - "--max-chars", - type=int, - default=256, - help="Maximum characters per chunk before TTS inference.", - ) - parser.add_argument( - "--device", - choices=["auto", "cuda", "cpu"], - default="auto", - help="Device to run inference on (auto=CUDA if available).", - ) - parser.add_argument( - "--backbone", - default="pnnbao-ump/VieNeu-TTS", - help="Backbone repository ID or local path.", - ) - parser.add_argument( - "--codec", - default="neuphonic/neucodec", - help="Codec repository ID or local path.", - ) - return parser.parse_args() - - -def main(): - args = parse_args() - ref_audio_path = Path(args.ref_audio) - if not ref_audio_path.exists(): - raise FileNotFoundError(f"Reference audio not found: {ref_audio_path}") - - ref_text_path = Path(args.ref_text) - if not ref_text_path.exists(): - raise FileNotFoundError(f"Reference text not found: {ref_text_path}") - - if args.text_file: - text_path = Path(args.text_file) - if not text_path.exists(): - raise FileNotFoundError(f"Text file not found: {text_path}") - raw_text = text_path.read_text(encoding="utf-8") - else: - raw_text = args.text.strip() - if not raw_text: - raise ValueError("Provided text is empty.") - device = ( - "cuda" - if args.device == "auto" and torch.cuda.is_available() - else ("cpu" if args.device == "auto" else args.device) - ) - - infer_long_text( - text=raw_text, - ref_audio_path=str(ref_audio_path), - ref_text_path=str(ref_text_path), - output_path=args.output, - chunk_dir=args.chunk_output_dir, - max_chars=args.max_chars, - backbone_repo=args.backbone, - codec_repo=args.codec, - device=device, - ) - - -if __name__ == "__main__": - main() - diff --git a/examples/sample_long_text.txt b/examples/sample_long_text.txt deleted file mode 100644 index 2db8c59467bd2ff123893584bbbf3b904023a708..0000000000000000000000000000000000000000 --- a/examples/sample_long_text.txt +++ /dev/null @@ -1,4 +0,0 @@ -Buổi sáng hôm ấy, ánh nắng vàng óng từ từ lan tỏa qua những tán cây xanh mướt, tạo nên những vệt sáng lung linh trên mặt đất. Tiếng chim hót véo von vang lên khắp khu rừng, hòa quyện cùng tiếng suối chảy róc rách từ phía xa. Không khí trong lành, mát mẻ khiến người ta cảm thấy sảng khoái và tràn đầy năng lượng. -Tôi bước chậm rãi trên con đường mòn quanh co, ngắm nhìn những bông hoa dại đủ màu sắc nở rộ bên vệ đường. Có những bông hoa màu tím nhạt, có những bông màu vàng rực rỡ, và cả những bông hoa trắng tinh khôi như những vì sao nhỏ. Gió nhẹ thổi qua, mang theo hương thơm ngào ngạt của hoa lá, khiến lòng người ta thư thái và bình yên đến lạ. -Đi một đoạn nữa, tôi đến một cánh đồng lúa chín vàng trải dài bất tận. Những đợt sóng lúa nhấp nhô theo gió, tạo nên một bức tranh thiên nhiên tuyệt đẹp. Xa xa, những người nông dân đang miệt mài gặt lúa, tiếng cười nói vui vẻ của họ vang lên, hòa cùng tiếng ve kêu râm ran. Đó là bức tranh của một mùa màng bội thu, của sự cần cù và đoàn kết. -Cuộc sống thật đơn giản nhưng đầy ý nghĩa khi ta biết trân trọng những khoảnh khắc nhỏ bé như thế này. Mỗi ngày trôi qua là một món quà, một cơ hội để ta được tận hưởng vẻ đẹp của thiên nhiên và sự ấm áp của tình người. \ No newline at end of file diff --git a/packages.txt b/packages.txt deleted file mode 100644 index de84c4624a835c2f0ad15d5ce2f3360afe24dad6..0000000000000000000000000000000000000000 --- a/packages.txt +++ /dev/null @@ -1,3 +0,0 @@ -espeak-ng -libespeak-ng1 -ffmpeg \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 790d033f439a200b96af0001b93a088c3a02f0f6..bb85e3a7def222c945b0b3de943a468fa9e8168a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,11 @@ -torchaudio -transformers -librosa -soundfile -numpy -onnx -onnxruntime-gpu -phonemizer -sea-g2p -requests -pyyaml -numpy \ No newline at end of file +# VieNeu-TTS v3 Turbo — Hugging Face ZeroGPU Space (PyTorch / CUDA path) +# +# torch 2.8.0 is the floor ZeroGPU supports; HF provides the matching CUDA build. +# The `vieneu` core pulls in sea-g2p, onnxruntime, soundfile, soxr, tokenizers, +# huggingface_hub, perth and gradio automatically. +vieneu==3.0.1 + +torch==2.8.0 +torchaudio==2.8.0 +transformers==4.57.3 +safetensors>=0.4 diff --git "a/sample/B\303\254nh (nam mi\341\273\201n B\341\272\257c).pt" "b/sample/B\303\254nh (nam mi\341\273\201n B\341\272\257c).pt" deleted file mode 100644 index d4daaf33136c91d125d7158fbddc32f962ac5254..0000000000000000000000000000000000000000 --- "a/sample/B\303\254nh (nam mi\341\273\201n B\341\272\257c).pt" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1f896d618fc46c3e131eda7b4168e25e9c2fb2d7ea0e864bedff2577fbd0bd30 -size 2089 diff --git "a/sample/B\303\254nh (nam mi\341\273\201n B\341\272\257c).txt" "b/sample/B\303\254nh (nam mi\341\273\201n B\341\272\257c).txt" deleted file mode 100644 index a8f91a90d0f830c1cd482ac5468ccbc1e1af43ed..0000000000000000000000000000000000000000 --- "a/sample/B\303\254nh (nam mi\341\273\201n B\341\272\257c).txt" +++ /dev/null @@ -1 +0,0 @@ -Anh chỉ muốn được nhìn nhận như là một huấn luyện viên. \ No newline at end of file diff --git "a/sample/B\303\254nh (nam mi\341\273\201n B\341\272\257c).wav" "b/sample/B\303\254nh (nam mi\341\273\201n B\341\272\257c).wav" deleted file mode 100644 index 8219ecdfcac169ade4cc03bbce49372eb0afa766..0000000000000000000000000000000000000000 --- "a/sample/B\303\254nh (nam mi\341\273\201n B\341\272\257c).wav" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:135f087ced48606c4d406b770a11e344d4d9aa6bd7adfb3e5c26f69cd9cc6df1 -size 127054 diff --git "a/sample/Dung (n\341\273\257 mi\341\273\201n Nam).pt" "b/sample/Dung (n\341\273\257 mi\341\273\201n Nam).pt" deleted file mode 100644 index 9f5bbb12264a46e7cf148ff762ab666f3af079a3..0000000000000000000000000000000000000000 --- "a/sample/Dung (n\341\273\257 mi\341\273\201n Nam).pt" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:dc4d65b6504470cb00e46763915060590595fbe4d47912eeacecd2bf1bade262 -size 2153 diff --git "a/sample/Dung (n\341\273\257 mi\341\273\201n Nam).txt" "b/sample/Dung (n\341\273\257 mi\341\273\201n Nam).txt" deleted file mode 100644 index f6cbfa7f5d19f8ed0e2b74c0cb5a7d00a19c0cd1..0000000000000000000000000000000000000000 --- "a/sample/Dung (n\341\273\257 mi\341\273\201n Nam).txt" +++ /dev/null @@ -1 +0,0 @@ -Tục ngữ có câu, sai một li, đi một dặm. \ No newline at end of file diff --git "a/sample/Dung (n\341\273\257 mi\341\273\201n Nam).wav" "b/sample/Dung (n\341\273\257 mi\341\273\201n Nam).wav" deleted file mode 100644 index 8223c31455a13fd1d07ad22d74659861db544401..0000000000000000000000000000000000000000 --- "a/sample/Dung (n\341\273\257 mi\341\273\201n Nam).wav" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:56e42039d0c96ad19e9f78ecb7218853202022b2a8460010d34ffb7879b17409 -size 143438 diff --git "a/sample/H\306\260\306\241ng (n\341\273\257 mi\341\273\201n B\341\272\257c).pt" "b/sample/H\306\260\306\241ng (n\341\273\257 mi\341\273\201n B\341\272\257c).pt" deleted file mode 100644 index a7811e28cfd6af1fbc7755acd46a518671f619c2..0000000000000000000000000000000000000000 --- "a/sample/H\306\260\306\241ng (n\341\273\257 mi\341\273\201n B\341\272\257c).pt" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:919035b7c762956a7d568cebc6e69fea22eb9be02bf906c1d32c1db1d8c7b9ff -size 2217 diff --git "a/sample/H\306\260\306\241ng (n\341\273\257 mi\341\273\201n B\341\272\257c).txt" "b/sample/H\306\260\306\241ng (n\341\273\257 mi\341\273\201n B\341\272\257c).txt" deleted file mode 100644 index 5a5b0624ecd2c952ea476b2ff8208b77971aabf8..0000000000000000000000000000000000000000 --- "a/sample/H\306\260\306\241ng (n\341\273\257 mi\341\273\201n B\341\272\257c).txt" +++ /dev/null @@ -1 +0,0 @@ -Tuy nhiên, lúc này có một vấn đề khó khăn nảy sinh. \ No newline at end of file diff --git "a/sample/H\306\260\306\241ng (n\341\273\257 mi\341\273\201n B\341\272\257c).wav" "b/sample/H\306\260\306\241ng (n\341\273\257 mi\341\273\201n B\341\272\257c).wav" deleted file mode 100644 index 6aefb2843704ef45d09a7d5a104654732d6365b9..0000000000000000000000000000000000000000 --- "a/sample/H\306\260\306\241ng (n\341\273\257 mi\341\273\201n B\341\272\257c).wav" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4c064b4ec64df44ea1306e87b25b84905e540d0fe29885629d2fe8bc8a5e53bc -size 155756 diff --git "a/sample/Ly (n\341\273\257 mi\341\273\201n B\341\272\257c).pt" "b/sample/Ly (n\341\273\257 mi\341\273\201n B\341\272\257c).pt" deleted file mode 100644 index b55700d20101a48ed3859205342e099ae5965d35..0000000000000000000000000000000000000000 --- "a/sample/Ly (n\341\273\257 mi\341\273\201n B\341\272\257c).pt" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:69b6bc9bb1062122dc3755be907d87f232fa8be5129b54f6994dead35f4935c6 -size 2153 diff --git "a/sample/Ly (n\341\273\257 mi\341\273\201n B\341\272\257c).txt" "b/sample/Ly (n\341\273\257 mi\341\273\201n B\341\272\257c).txt" deleted file mode 100644 index f1f0344fdf00040ad471754f3d78f5f24b96910d..0000000000000000000000000000000000000000 --- "a/sample/Ly (n\341\273\257 mi\341\273\201n B\341\272\257c).txt" +++ /dev/null @@ -1 +0,0 @@ -Chúng ta có thể áp dụng logic tương tự với người khác. \ No newline at end of file diff --git "a/sample/Ly (n\341\273\257 mi\341\273\201n B\341\272\257c).wav" "b/sample/Ly (n\341\273\257 mi\341\273\201n B\341\272\257c).wav" deleted file mode 100644 index 0a2fc9655480d46e8471241313c65a98ce6bf2c3..0000000000000000000000000000000000000000 --- "a/sample/Ly (n\341\273\257 mi\341\273\201n B\341\272\257c).wav" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0d4e47cfa5ed0b753c2bed07c58e26da89ee2977ca5e941244a6bbafd8869d5e -size 147534 diff --git "a/sample/Nguy\303\252n (nam mi\341\273\201n Nam).pt" "b/sample/Nguy\303\252n (nam mi\341\273\201n Nam).pt" deleted file mode 100644 index 6c2af76bed8c72bb591a3d466ce7a4234c5fef6c..0000000000000000000000000000000000000000 --- "a/sample/Nguy\303\252n (nam mi\341\273\201n Nam).pt" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0e6ebaa0b2977589afa7e7f811b0553151bd8312c96a70b1b666bd9d0fd50edf -size 2345 diff --git "a/sample/Nguy\303\252n (nam mi\341\273\201n Nam).txt" "b/sample/Nguy\303\252n (nam mi\341\273\201n Nam).txt" deleted file mode 100644 index aee753a7948d9067479bcf64071927f0598d3cf8..0000000000000000000000000000000000000000 --- "a/sample/Nguy\303\252n (nam mi\341\273\201n Nam).txt" +++ /dev/null @@ -1 +0,0 @@ -Hiểu biết về bản thân và người khác bắt đầu từ chính cơ thể mình. \ No newline at end of file diff --git "a/sample/Nguy\303\252n (nam mi\341\273\201n Nam).wav" "b/sample/Nguy\303\252n (nam mi\341\273\201n Nam).wav" deleted file mode 100644 index 5f8504fa79d993ef79f3caf1c8d40228a3edec47..0000000000000000000000000000000000000000 --- "a/sample/Nguy\303\252n (nam mi\341\273\201n Nam).wav" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4fc9655e24a7048c3c908494f2cbaf4c42d3d139d68c21d4f50c60e03aa19727 -size 196124 diff --git "a/sample/Ng\341\273\215c (n\341\273\257 mi\341\273\201n B\341\272\257c).pt" "b/sample/Ng\341\273\215c (n\341\273\257 mi\341\273\201n B\341\272\257c).pt" deleted file mode 100644 index f341086f4e07f3eacb997cf56bf1d05571e51fea..0000000000000000000000000000000000000000 --- "a/sample/Ng\341\273\215c (n\341\273\257 mi\341\273\201n B\341\272\257c).pt" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:78ab670f177092dc8586e45536faea20fdb84471dc8d8a8b1b95dd76a4ed3d0d -size 2281 diff --git "a/sample/Ng\341\273\215c (n\341\273\257 mi\341\273\201n B\341\272\257c).txt" "b/sample/Ng\341\273\215c (n\341\273\257 mi\341\273\201n B\341\272\257c).txt" deleted file mode 100644 index 0b2fc09b6bb5a79a09282dd877529fbc9d89b044..0000000000000000000000000000000000000000 --- "a/sample/Ng\341\273\215c (n\341\273\257 mi\341\273\201n B\341\272\257c).txt" +++ /dev/null @@ -1 +0,0 @@ -Trong phòng rất tù mù, nên có thể dễ dàng che dấu nó. \ No newline at end of file diff --git "a/sample/Ng\341\273\215c (n\341\273\257 mi\341\273\201n B\341\272\257c).wav" "b/sample/Ng\341\273\215c (n\341\273\257 mi\341\273\201n B\341\272\257c).wav" deleted file mode 100644 index b3c930bbe0f449baef6cca4f77d5176e4d02b2ef..0000000000000000000000000000000000000000 --- "a/sample/Ng\341\273\215c (n\341\273\257 mi\341\273\201n B\341\272\257c).wav" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:475a73298fbe86e5d92e7fb95c6c26e897e5a2ffbdc3fa9e062df4025767af93 -size 174956 diff --git "a/sample/S\306\241n (nam mi\341\273\201n Nam).pt" "b/sample/S\306\241n (nam mi\341\273\201n Nam).pt" deleted file mode 100644 index 1489058917672cf1e7efcf56eb899c4edd27faab..0000000000000000000000000000000000000000 --- "a/sample/S\306\241n (nam mi\341\273\201n Nam).pt" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:114cb04ee2357d06de2f038853bbeb0dc57fc8ed30e085118a9e0bf5a70f7857 -size 2281 diff --git "a/sample/S\306\241n (nam mi\341\273\201n Nam).txt" "b/sample/S\306\241n (nam mi\341\273\201n Nam).txt" deleted file mode 100644 index c9fe64610c3df7690ef83364642d57b767081745..0000000000000000000000000000000000000000 --- "a/sample/S\306\241n (nam mi\341\273\201n Nam).txt" +++ /dev/null @@ -1 +0,0 @@ -Trên thực tế, các nghi ngờ đã bắt đầu xuất hiện. \ No newline at end of file diff --git "a/sample/S\306\241n (nam mi\341\273\201n Nam).wav" "b/sample/S\306\241n (nam mi\341\273\201n Nam).wav" deleted file mode 100644 index 1a2d8bb35da9d0bc610c0a942d64e894ea766430..0000000000000000000000000000000000000000 --- "a/sample/S\306\241n (nam mi\341\273\201n Nam).wav" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8f6733df04e5f3477a00136c6baeaf7a196c93df0ee13b9bfa3d8ba61034f063 -size 174044 diff --git "a/sample/Tuy\303\252n (nam mi\341\273\201n B\341\272\257c).pt" "b/sample/Tuy\303\252n (nam mi\341\273\201n B\341\272\257c).pt" deleted file mode 100644 index 108b2be8d9cf38323984fd59f50e6d0a484fb6a0..0000000000000000000000000000000000000000 --- "a/sample/Tuy\303\252n (nam mi\341\273\201n B\341\272\257c).pt" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e79eb6ee9cc7cd35cb4fbbef107249ed3209608b59644c52f55a34941a531873 -size 2473 diff --git "a/sample/Tuy\303\252n (nam mi\341\273\201n B\341\272\257c).txt" "b/sample/Tuy\303\252n (nam mi\341\273\201n B\341\272\257c).txt" deleted file mode 100644 index ab1415c3c83309e9bcd0f5dcf327c99fdea7a5a2..0000000000000000000000000000000000000000 --- "a/sample/Tuy\303\252n (nam mi\341\273\201n B\341\272\257c).txt" +++ /dev/null @@ -1 +0,0 @@ -Bạn cầm khúc cây, và ném vào bãi cỏ xanh tươi rậm rạp ở đằng xa. \ No newline at end of file diff --git "a/sample/Tuy\303\252n (nam mi\341\273\201n B\341\272\257c).wav" "b/sample/Tuy\303\252n (nam mi\341\273\201n B\341\272\257c).wav" deleted file mode 100644 index c3f694983a8706a42cc32b21de4e91595cbb7376..0000000000000000000000000000000000000000 --- "a/sample/Tuy\303\252n (nam mi\341\273\201n B\341\272\257c).wav" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f6b7ac2605db0a2cf634ce0f3a55a87a89f4c2e3bc06f83433e6af583c1f3692 -size 217166 diff --git "a/sample/V\304\251nh (nam mi\341\273\201n Nam).pt" "b/sample/V\304\251nh (nam mi\341\273\201n Nam).pt" deleted file mode 100644 index c53b01f37919263d5feea83b9d3a180bb5976903..0000000000000000000000000000000000000000 --- "a/sample/V\304\251nh (nam mi\341\273\201n Nam).pt" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c87342d3a6a8cbaaf2139c21e7554eea19aba6aa03248e4426238a1c2507e447 -size 2217 diff --git "a/sample/V\304\251nh (nam mi\341\273\201n Nam).txt" "b/sample/V\304\251nh (nam mi\341\273\201n Nam).txt" deleted file mode 100644 index e01f4a0fc1874a6ef1bd73bcd2e50b47ed8c685d..0000000000000000000000000000000000000000 --- "a/sample/V\304\251nh (nam mi\341\273\201n Nam).txt" +++ /dev/null @@ -1 +0,0 @@ -Đến cuối thế kỷ 19, ngành đánh bắt cá được thương mại hóa. \ No newline at end of file diff --git "a/sample/V\304\251nh (nam mi\341\273\201n Nam).wav" "b/sample/V\304\251nh (nam mi\341\273\201n Nam).wav" deleted file mode 100644 index 804e1d23449eae11ab055eb50773e9ed5f2b3aac..0000000000000000000000000000000000000000 --- "a/sample/V\304\251nh (nam mi\341\273\201n Nam).wav" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:632a5c8fa34fe03001cc3c44427b5e0ee70f767377bc788b59a5dc9afa9fba49 -size 164492 diff --git "a/sample/\304\220oan (n\341\273\257 mi\341\273\201n Nam).pt" "b/sample/\304\220oan (n\341\273\257 mi\341\273\201n Nam).pt" deleted file mode 100644 index 35e162cd35f122df414839548bd4a33fe7fba5c6..0000000000000000000000000000000000000000 --- "a/sample/\304\220oan (n\341\273\257 mi\341\273\201n Nam).pt" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:28b48dbae193adc88aa26243086ba3ce862def7035d9793613c2967df29f9afe -size 2793 diff --git "a/sample/\304\220oan (n\341\273\257 mi\341\273\201n Nam).txt" "b/sample/\304\220oan (n\341\273\257 mi\341\273\201n Nam).txt" deleted file mode 100644 index f77d56d5069b7d1449aac7c59d167a4006422273..0000000000000000000000000000000000000000 --- "a/sample/\304\220oan (n\341\273\257 mi\341\273\201n Nam).txt" +++ /dev/null @@ -1 +0,0 @@ -Nuôi con theo phong cách Do Thái, không chỉ tốt cho đứa trẻ, mà còn tốt cho cả các bậc cha mẹ. \ No newline at end of file diff --git "a/sample/\304\220oan (n\341\273\257 mi\341\273\201n Nam).wav" "b/sample/\304\220oan (n\341\273\257 mi\341\273\201n Nam).wav" deleted file mode 100644 index 5dcb58561123b31b5f829ef5b0f6ea429a32c4d2..0000000000000000000000000000000000000000 --- "a/sample/\304\220oan (n\341\273\257 mi\341\273\201n Nam).wav" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:3e319ed45dd2a1458a52edfe43a83a36eff813f19399ac2e59ee3f93cace74be -size 294830 diff --git a/src/vieneu.egg-info/PKG-INFO b/src/vieneu.egg-info/PKG-INFO deleted file mode 100644 index 564efc52922045a0a472f2beb3f083867f8875d2..0000000000000000000000000000000000000000 --- a/src/vieneu.egg-info/PKG-INFO +++ /dev/null @@ -1,325 +0,0 @@ -Metadata-Version: 2.4 -Name: vieneu -Version: 2.1.1 -Summary: Advanced on-device Vietnamese TTS with instant voice cloning -Author-email: Phạm Nguyễn Ngọc Bảo -License: Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -Project-URL: Homepage, https://github.com/pnnbao97/VieNeu-TTS -Project-URL: Repository, https://github.com/pnnbao97/VieNeu-TTS -Project-URL: Bug Tracker, https://github.com/pnnbao97/VieNeu-TTS/issues -Project-URL: Documentation, https://github.com/pnnbao97/VieNeu-TTS/blob/main/README.md -Project-URL: Source Code, https://github.com/pnnbao97/VieNeu-TTS -Project-URL: Changelog, https://github.com/pnnbao97/VieNeu-TTS/releases -Keywords: text-to-speech,tts,vietnamese,voice-cloning,speech-synthesis,real-time,on-device -Classifier: Development Status :: 4 - Beta -Classifier: Intended Audience :: Developers -Classifier: Intended Audience :: Science/Research -Classifier: License :: OSI Approved :: Apache Software License -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Topic :: Multimedia :: Sound/Audio :: Speech -Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence -Classifier: Operating System :: OS Independent -Requires-Python: >=3.10 -Description-Content-Type: text/markdown -License-File: LICENSE -Requires-Dist: sea-g2p>=0.7.5 -Requires-Dist: onnxruntime>=1.23.2 -Requires-Dist: llama-cpp-python>=0.3.16 -Requires-Dist: requests -Requires-Dist: numpy -Requires-Dist: soundfile -Requires-Dist: PyYAML -Requires-Dist: gradio>=5.49.1 -Requires-Dist: perth>=0.2.0 -Provides-Extra: gpu -Requires-Dist: torch; extra == "gpu" -Requires-Dist: torchaudio; extra == "gpu" -Requires-Dist: neucodec>=0.0.4; extra == "gpu" -Requires-Dist: lmdeploy; sys_platform != "darwin" and extra == "gpu" -Requires-Dist: triton-windows; sys_platform == "win32" and extra == "gpu" -Requires-Dist: triton; sys_platform == "linux" and extra == "gpu" -Requires-Dist: transformers; sys_platform == "darwin" and extra == "gpu" -Requires-Dist: accelerate; sys_platform == "darwin" and extra == "gpu" -Dynamic: license-file - -# 🦜 VieNeu-TTS - -**VieNeu-TTS** is an advanced on-device Vietnamese Text-to-Speech (TTS) model with **instant voice cloning** and **English-Vietnamese bilingual** support. - -[![Hugging Face v2 Turbo](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-v2%20Turbo-blue)](https://huggingface.co/pnnbao-ump/VieNeu-TTS-v2-Turbo-GGUF) -[![License](https://img.shields.io/badge/License-Apache%202.0-green.svg)](https://opensource.org/licenses/Apache-2.0) - -## ✨ Key Features -- **Bilingual (English-Vietnamese)**: Seamless transitions between languages (Code-switching) in version 2.0+. -- **Ultra-Fast Turbo Mode**: Optimized for CPU/Mobile using GGUF and ONNX. No dedicated GPU required! -- **Instant Voice Cloning**: Clone any voice with just 3-5s of reference audio (GPU mode). -- **Production Ready**: High-fidelity 24 kHz audio generation, fully offline. -- **AI Identification**: Built-in audio watermarking for responsible AI use. - ---- - -## 📦 Quick Install - -```bash -# Minimal installation (Turbo/CPU Only) -pip install vieneu - -# Optional: Pre-built llama-cpp-python for CPU (if building fails) -pip install vieneu --extra-index-url https://pnnbao97.github.io/llama-cpp-python-v0.3.16/cpu/ - -# Optional: macOS Metal acceleration -pip install vieneu --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/metal/ -``` ---- - -## 🚀 Quick Start (Python SDK) - -The SDK now defaults to **Turbo mode** for maximum out-of-the-box compatibility. - -```python -from vieneu import Vieneu - -# Initialize - Minimal dependencies required! -tts = Vieneu() - -# Synthesis with Bilingual support (Vietnamese + English) -text = "Trước đây, hệ thống điện chủ yếu sử dụng direct current, nhưng Tesla đã chứng minh rằng alternating current is more efficient." -audio = tts.infer(text=text) - -# Save output -tts.save(audio, "output.wav") -print("💾 Saved synthesis to output.wav") -``` - -### Advanced Usage (Remote API) -Connect to a remote VieNeu-TTS server without loading heavy models locally: -```python -tts = Vieneu(mode='remote', api_base='http://your-server:23333/v1') -audio = tts.infer(text="Xin chào!") -``` - ---- - -## 🔬 Model Overview - -| Model | Format | Device | Bilingual | Cloning | Speed | -|---|---|---|---|---|---| -| **VieNeu-v2-Turbo** | GGUF/ONNX | **CPU**/GPU | ✅ | ❌ (Ssoon) | **Extreme** | -| **VieNeu-TTS-v2** | PyTorch | GPU | ✅ | ✅ | **Standard** (Ssoon) | -| **VieNeu-TTS 0.3B** | PyTorch | GPU/CPU | ❌ | ✅ | **Very Fast** | -| **VieNeu-TTS** | PyTorch | GPU/CPU | ❌ | ✅ | **Standard** | - ---- - -## 🤝 Support & Links -- **GitHub:** [pnnbao97/VieNeu-TTS](https://github.com/pnnbao97/VieNeu-TTS) -- **Hugging Face:** [pnnbao-ump](https://huggingface.co/pnnbao-ump) -- **Discord:** [Join our community](https://discord.gg/yJt8kzjzWZ) - ---- - -**Made with ❤️ for the Vietnamese TTS community** diff --git a/src/vieneu.egg-info/SOURCES.txt b/src/vieneu.egg-info/SOURCES.txt deleted file mode 100644 index c5ac5bfbbcc52386930b14ce2ef1d5f2d45d51bc..0000000000000000000000000000000000000000 --- a/src/vieneu.egg-info/SOURCES.txt +++ /dev/null @@ -1,54 +0,0 @@ -LICENSE -README.md -README_PYPI.md -pyproject.toml -apps/__init__.py -apps/gradio_main.py -apps/gradio_xpu.py -apps/web_stream.py -examples/__init__.py -examples/main.py -examples/main_remote.py -src/vieneu/__init__.py -src/vieneu/base.py -src/vieneu/core_xpu.py -src/vieneu/factory.py -src/vieneu/fast.py -src/vieneu/remote.py -src/vieneu/serve.py -src/vieneu/standard.py -src/vieneu/turbo.py -src/vieneu/utils.py -src/vieneu.egg-info/PKG-INFO -src/vieneu.egg-info/SOURCES.txt -src/vieneu.egg-info/dependency_links.txt -src/vieneu.egg-info/entry_points.txt -src/vieneu.egg-info/requires.txt -src/vieneu.egg-info/top_level.txt -src/vieneu/assets/samples/Bình (nam miền Bắc).pt -src/vieneu/assets/samples/Bình (nam miền Bắc).txt -src/vieneu/assets/samples/Bình (nam miền Bắc).wav -src/vieneu/assets/samples/Ly (nữ miền Bắc).pt -src/vieneu/assets/samples/Ly (nữ miền Bắc).txt -src/vieneu/assets/samples/Ly (nữ miền Bắc).wav -src/vieneu/assets/samples/Ngọc (nữ miền Bắc).pt -src/vieneu/assets/samples/Ngọc (nữ miền Bắc).txt -src/vieneu/assets/samples/Ngọc (nữ miền Bắc).wav -src/vieneu/assets/samples/Tuyên (nam miền Bắc).pt -src/vieneu/assets/samples/Tuyên (nam miền Bắc).txt -src/vieneu/assets/samples/Tuyên (nam miền Bắc).wav -src/vieneu/assets/samples/Vĩnh (nam miền Nam).pt -src/vieneu/assets/samples/Vĩnh (nam miền Nam).txt -src/vieneu/assets/samples/Vĩnh (nam miền Nam).wav -src/vieneu/assets/samples/Đoan (nữ miền Nam).pt -src/vieneu/assets/samples/Đoan (nữ miền Nam).txt -src/vieneu/assets/samples/Đoan (nữ miền Nam).wav -src/vieneu_utils/__init__.py -src/vieneu_utils/core_utils.py -src/vieneu_utils/phonemize_text.py -src/vieneu_utils/url_extract.py -tests/test_engine_fast.py -tests/test_engine_remote.py -tests/test_engine_standard.py -tests/test_factory.py -tests/test_utils.py \ No newline at end of file diff --git a/src/vieneu.egg-info/dependency_links.txt b/src/vieneu.egg-info/dependency_links.txt deleted file mode 100644 index 8b137891791fe96927ad78e64b0aad7bded08bdc..0000000000000000000000000000000000000000 --- a/src/vieneu.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/src/vieneu.egg-info/entry_points.txt b/src/vieneu.egg-info/entry_points.txt deleted file mode 100644 index 33fccc96b372a7197b2d548bcc7ffc20428f8fce..0000000000000000000000000000000000000000 --- a/src/vieneu.egg-info/entry_points.txt +++ /dev/null @@ -1,3 +0,0 @@ -[console_scripts] -vieneu-stream = apps.web_stream:main -vieneu-web = apps.gradio_main:main diff --git a/src/vieneu.egg-info/requires.txt b/src/vieneu.egg-info/requires.txt deleted file mode 100644 index c5ddbf11a36d0b70dedc3f2982c36ff5f45ad2f2..0000000000000000000000000000000000000000 --- a/src/vieneu.egg-info/requires.txt +++ /dev/null @@ -1,27 +0,0 @@ -sea-g2p>=0.7.5 -onnxruntime>=1.23.2 -llama-cpp-python>=0.3.16 -requests -numpy -soundfile -PyYAML -gradio>=5.49.1 -perth>=0.2.0 - -[gpu] -torch -torchaudio -neucodec>=0.0.4 - -[gpu:sys_platform != "darwin"] -lmdeploy - -[gpu:sys_platform == "darwin"] -transformers -accelerate - -[gpu:sys_platform == "linux"] -triton - -[gpu:sys_platform == "win32"] -triton-windows diff --git a/src/vieneu.egg-info/top_level.txt b/src/vieneu.egg-info/top_level.txt deleted file mode 100644 index 022afe8f9e6860b55e7e1daeea0068d10a75fbe9..0000000000000000000000000000000000000000 --- a/src/vieneu.egg-info/top_level.txt +++ /dev/null @@ -1,4 +0,0 @@ -apps -examples -vieneu -vieneu_utils diff --git a/src/vieneu/__init__.py b/src/vieneu/__init__.py deleted file mode 100644 index 3758c071358ee55c6a5e60f7a8859b4f0ff81605..0000000000000000000000000000000000000000 --- a/src/vieneu/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .factory import Vieneu - -__all__ = ["Vieneu"] diff --git a/src/vieneu/__pycache__/__init__.cpython-312.pyc b/src/vieneu/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 51b3c310229d6d764454b4123899d63ed99a4bc2..0000000000000000000000000000000000000000 Binary files a/src/vieneu/__pycache__/__init__.cpython-312.pyc and /dev/null differ diff --git a/src/vieneu/__pycache__/base.cpython-312.pyc b/src/vieneu/__pycache__/base.cpython-312.pyc deleted file mode 100644 index e7b590a5450207baf3cd71ddd2a2e5138b74e59f..0000000000000000000000000000000000000000 Binary files a/src/vieneu/__pycache__/base.cpython-312.pyc and /dev/null differ diff --git a/src/vieneu/__pycache__/factory.cpython-312.pyc b/src/vieneu/__pycache__/factory.cpython-312.pyc deleted file mode 100644 index 68f650bd057646399934a2ba41513499026f366a..0000000000000000000000000000000000000000 Binary files a/src/vieneu/__pycache__/factory.cpython-312.pyc and /dev/null differ diff --git a/src/vieneu/__pycache__/fast.cpython-312.pyc b/src/vieneu/__pycache__/fast.cpython-312.pyc deleted file mode 100644 index 9dde181a5d4e8822973ee90f41cfc79357f5b1ac..0000000000000000000000000000000000000000 Binary files a/src/vieneu/__pycache__/fast.cpython-312.pyc and /dev/null differ diff --git a/src/vieneu/__pycache__/standard.cpython-312.pyc b/src/vieneu/__pycache__/standard.cpython-312.pyc deleted file mode 100644 index f4d7a04e4672f7a04fff48b625398216df28d88d..0000000000000000000000000000000000000000 Binary files a/src/vieneu/__pycache__/standard.cpython-312.pyc and /dev/null differ diff --git a/src/vieneu/__pycache__/turbo.cpython-312.pyc b/src/vieneu/__pycache__/turbo.cpython-312.pyc deleted file mode 100644 index e1d0dfc87ae50d0b49f01c409435304fc5296a38..0000000000000000000000000000000000000000 Binary files a/src/vieneu/__pycache__/turbo.cpython-312.pyc and /dev/null differ diff --git a/src/vieneu/__pycache__/utils.cpython-312.pyc b/src/vieneu/__pycache__/utils.cpython-312.pyc deleted file mode 100644 index 1093c7d6336508fcefcb981f3da15ed055a051ab..0000000000000000000000000000000000000000 Binary files a/src/vieneu/__pycache__/utils.cpython-312.pyc and /dev/null differ diff --git "a/src/vieneu/assets/samples/B\303\254nh (nam mi\341\273\201n B\341\272\257c).pt" "b/src/vieneu/assets/samples/B\303\254nh (nam mi\341\273\201n B\341\272\257c).pt" deleted file mode 100644 index d4daaf33136c91d125d7158fbddc32f962ac5254..0000000000000000000000000000000000000000 --- "a/src/vieneu/assets/samples/B\303\254nh (nam mi\341\273\201n B\341\272\257c).pt" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1f896d618fc46c3e131eda7b4168e25e9c2fb2d7ea0e864bedff2577fbd0bd30 -size 2089 diff --git "a/src/vieneu/assets/samples/B\303\254nh (nam mi\341\273\201n B\341\272\257c).txt" "b/src/vieneu/assets/samples/B\303\254nh (nam mi\341\273\201n B\341\272\257c).txt" deleted file mode 100644 index a8f91a90d0f830c1cd482ac5468ccbc1e1af43ed..0000000000000000000000000000000000000000 --- "a/src/vieneu/assets/samples/B\303\254nh (nam mi\341\273\201n B\341\272\257c).txt" +++ /dev/null @@ -1 +0,0 @@ -Anh chỉ muốn được nhìn nhận như là một huấn luyện viên. \ No newline at end of file diff --git "a/src/vieneu/assets/samples/B\303\254nh (nam mi\341\273\201n B\341\272\257c).wav" "b/src/vieneu/assets/samples/B\303\254nh (nam mi\341\273\201n B\341\272\257c).wav" deleted file mode 100644 index 8219ecdfcac169ade4cc03bbce49372eb0afa766..0000000000000000000000000000000000000000 --- "a/src/vieneu/assets/samples/B\303\254nh (nam mi\341\273\201n B\341\272\257c).wav" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:135f087ced48606c4d406b770a11e344d4d9aa6bd7adfb3e5c26f69cd9cc6df1 -size 127054 diff --git "a/src/vieneu/assets/samples/Ly (n\341\273\257 mi\341\273\201n B\341\272\257c).pt" "b/src/vieneu/assets/samples/Ly (n\341\273\257 mi\341\273\201n B\341\272\257c).pt" deleted file mode 100644 index b55700d20101a48ed3859205342e099ae5965d35..0000000000000000000000000000000000000000 --- "a/src/vieneu/assets/samples/Ly (n\341\273\257 mi\341\273\201n B\341\272\257c).pt" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:69b6bc9bb1062122dc3755be907d87f232fa8be5129b54f6994dead35f4935c6 -size 2153 diff --git "a/src/vieneu/assets/samples/Ly (n\341\273\257 mi\341\273\201n B\341\272\257c).txt" "b/src/vieneu/assets/samples/Ly (n\341\273\257 mi\341\273\201n B\341\272\257c).txt" deleted file mode 100644 index f1f0344fdf00040ad471754f3d78f5f24b96910d..0000000000000000000000000000000000000000 --- "a/src/vieneu/assets/samples/Ly (n\341\273\257 mi\341\273\201n B\341\272\257c).txt" +++ /dev/null @@ -1 +0,0 @@ -Chúng ta có thể áp dụng logic tương tự với người khác. \ No newline at end of file diff --git "a/src/vieneu/assets/samples/Ly (n\341\273\257 mi\341\273\201n B\341\272\257c).wav" "b/src/vieneu/assets/samples/Ly (n\341\273\257 mi\341\273\201n B\341\272\257c).wav" deleted file mode 100644 index 0a2fc9655480d46e8471241313c65a98ce6bf2c3..0000000000000000000000000000000000000000 --- "a/src/vieneu/assets/samples/Ly (n\341\273\257 mi\341\273\201n B\341\272\257c).wav" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0d4e47cfa5ed0b753c2bed07c58e26da89ee2977ca5e941244a6bbafd8869d5e -size 147534 diff --git "a/src/vieneu/assets/samples/Ng\341\273\215c (n\341\273\257 mi\341\273\201n B\341\272\257c).pt" "b/src/vieneu/assets/samples/Ng\341\273\215c (n\341\273\257 mi\341\273\201n B\341\272\257c).pt" deleted file mode 100644 index f341086f4e07f3eacb997cf56bf1d05571e51fea..0000000000000000000000000000000000000000 --- "a/src/vieneu/assets/samples/Ng\341\273\215c (n\341\273\257 mi\341\273\201n B\341\272\257c).pt" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:78ab670f177092dc8586e45536faea20fdb84471dc8d8a8b1b95dd76a4ed3d0d -size 2281 diff --git "a/src/vieneu/assets/samples/Ng\341\273\215c (n\341\273\257 mi\341\273\201n B\341\272\257c).txt" "b/src/vieneu/assets/samples/Ng\341\273\215c (n\341\273\257 mi\341\273\201n B\341\272\257c).txt" deleted file mode 100644 index 0b2fc09b6bb5a79a09282dd877529fbc9d89b044..0000000000000000000000000000000000000000 --- "a/src/vieneu/assets/samples/Ng\341\273\215c (n\341\273\257 mi\341\273\201n B\341\272\257c).txt" +++ /dev/null @@ -1 +0,0 @@ -Trong phòng rất tù mù, nên có thể dễ dàng che dấu nó. \ No newline at end of file diff --git "a/src/vieneu/assets/samples/Ng\341\273\215c (n\341\273\257 mi\341\273\201n B\341\272\257c).wav" "b/src/vieneu/assets/samples/Ng\341\273\215c (n\341\273\257 mi\341\273\201n B\341\272\257c).wav" deleted file mode 100644 index b3c930bbe0f449baef6cca4f77d5176e4d02b2ef..0000000000000000000000000000000000000000 --- "a/src/vieneu/assets/samples/Ng\341\273\215c (n\341\273\257 mi\341\273\201n B\341\272\257c).wav" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:475a73298fbe86e5d92e7fb95c6c26e897e5a2ffbdc3fa9e062df4025767af93 -size 174956 diff --git "a/src/vieneu/assets/samples/Tuy\303\252n (nam mi\341\273\201n B\341\272\257c).pt" "b/src/vieneu/assets/samples/Tuy\303\252n (nam mi\341\273\201n B\341\272\257c).pt" deleted file mode 100644 index de64d407fd538da7e5cac8f9e74266d54df426ca..0000000000000000000000000000000000000000 --- "a/src/vieneu/assets/samples/Tuy\303\252n (nam mi\341\273\201n B\341\272\257c).pt" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:19863fc6652d211159dbabcc4316854b73ad68388ad4eebc3357e0428737ceb1 -size 2473 diff --git "a/src/vieneu/assets/samples/Tuy\303\252n (nam mi\341\273\201n B\341\272\257c).txt" "b/src/vieneu/assets/samples/Tuy\303\252n (nam mi\341\273\201n B\341\272\257c).txt" deleted file mode 100644 index ab1415c3c83309e9bcd0f5dcf327c99fdea7a5a2..0000000000000000000000000000000000000000 --- "a/src/vieneu/assets/samples/Tuy\303\252n (nam mi\341\273\201n B\341\272\257c).txt" +++ /dev/null @@ -1 +0,0 @@ -Bạn cầm khúc cây, và ném vào bãi cỏ xanh tươi rậm rạp ở đằng xa. \ No newline at end of file diff --git "a/src/vieneu/assets/samples/Tuy\303\252n (nam mi\341\273\201n B\341\272\257c).wav" "b/src/vieneu/assets/samples/Tuy\303\252n (nam mi\341\273\201n B\341\272\257c).wav" deleted file mode 100644 index c3f694983a8706a42cc32b21de4e91595cbb7376..0000000000000000000000000000000000000000 --- "a/src/vieneu/assets/samples/Tuy\303\252n (nam mi\341\273\201n B\341\272\257c).wav" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f6b7ac2605db0a2cf634ce0f3a55a87a89f4c2e3bc06f83433e6af583c1f3692 -size 217166 diff --git "a/src/vieneu/assets/samples/V\304\251nh (nam mi\341\273\201n Nam).pt" "b/src/vieneu/assets/samples/V\304\251nh (nam mi\341\273\201n Nam).pt" deleted file mode 100644 index c53b01f37919263d5feea83b9d3a180bb5976903..0000000000000000000000000000000000000000 --- "a/src/vieneu/assets/samples/V\304\251nh (nam mi\341\273\201n Nam).pt" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c87342d3a6a8cbaaf2139c21e7554eea19aba6aa03248e4426238a1c2507e447 -size 2217 diff --git "a/src/vieneu/assets/samples/V\304\251nh (nam mi\341\273\201n Nam).txt" "b/src/vieneu/assets/samples/V\304\251nh (nam mi\341\273\201n Nam).txt" deleted file mode 100644 index e01f4a0fc1874a6ef1bd73bcd2e50b47ed8c685d..0000000000000000000000000000000000000000 --- "a/src/vieneu/assets/samples/V\304\251nh (nam mi\341\273\201n Nam).txt" +++ /dev/null @@ -1 +0,0 @@ -Đến cuối thế kỷ 19, ngành đánh bắt cá được thương mại hóa. \ No newline at end of file diff --git "a/src/vieneu/assets/samples/V\304\251nh (nam mi\341\273\201n Nam).wav" "b/src/vieneu/assets/samples/V\304\251nh (nam mi\341\273\201n Nam).wav" deleted file mode 100644 index 804e1d23449eae11ab055eb50773e9ed5f2b3aac..0000000000000000000000000000000000000000 --- "a/src/vieneu/assets/samples/V\304\251nh (nam mi\341\273\201n Nam).wav" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:632a5c8fa34fe03001cc3c44427b5e0ee70f767377bc788b59a5dc9afa9fba49 -size 164492 diff --git "a/src/vieneu/assets/samples/\304\220oan (n\341\273\257 mi\341\273\201n Nam).pt" "b/src/vieneu/assets/samples/\304\220oan (n\341\273\257 mi\341\273\201n Nam).pt" deleted file mode 100644 index 35e162cd35f122df414839548bd4a33fe7fba5c6..0000000000000000000000000000000000000000 --- "a/src/vieneu/assets/samples/\304\220oan (n\341\273\257 mi\341\273\201n Nam).pt" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:28b48dbae193adc88aa26243086ba3ce862def7035d9793613c2967df29f9afe -size 2793 diff --git "a/src/vieneu/assets/samples/\304\220oan (n\341\273\257 mi\341\273\201n Nam).txt" "b/src/vieneu/assets/samples/\304\220oan (n\341\273\257 mi\341\273\201n Nam).txt" deleted file mode 100644 index f77d56d5069b7d1449aac7c59d167a4006422273..0000000000000000000000000000000000000000 --- "a/src/vieneu/assets/samples/\304\220oan (n\341\273\257 mi\341\273\201n Nam).txt" +++ /dev/null @@ -1 +0,0 @@ -Nuôi con theo phong cách Do Thái, không chỉ tốt cho đứa trẻ, mà còn tốt cho cả các bậc cha mẹ. \ No newline at end of file diff --git "a/src/vieneu/assets/samples/\304\220oan (n\341\273\257 mi\341\273\201n Nam).wav" "b/src/vieneu/assets/samples/\304\220oan (n\341\273\257 mi\341\273\201n Nam).wav" deleted file mode 100644 index 5dcb58561123b31b5f829ef5b0f6ea429a32c4d2..0000000000000000000000000000000000000000 --- "a/src/vieneu/assets/samples/\304\220oan (n\341\273\257 mi\341\273\201n Nam).wav" +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:3e319ed45dd2a1458a52edfe43a83a36eff813f19399ac2e59ee3f93cace74be -size 294830 diff --git a/src/vieneu/assets/voices.json b/src/vieneu/assets/voices.json deleted file mode 100644 index 289e846c8a2a122062999dd9b8d9d32a2131b0af..0000000000000000000000000000000000000000 --- a/src/vieneu/assets/voices.json +++ /dev/null @@ -1,1227 +0,0 @@ -{ - "meta": { - "spec": "vieneu.voice.presets", - "spec_version": "1.0", - "engine": "VieNeu-TTS", - "author": "Phạm Nguyễn Ngọc Bảo (pnnbao-ump)", - "license": "CC BY-NC 4.0", - "homepage": "https://github.com/pnnbao97/VieNeu-TTS", - "notice": "Model and voices are for non-commercial use only. Mention pnnbao-ump when using." - }, - "default_voice": "Binh", - "presets": { - "Binh": { - "codes": [ - 41733, - 30590, - 11238, - 36562, - 17466, - 5163, - 13117, - 12897, - 29227, - 16000, - 48299, - 13043, - 47275, - 29347, - 30114, - 31153, - 47024, - 30384, - 11384, - 1752, - 32654, - 17319, - 27527, - 45027, - 26187, - 13828, - 15407, - 6232, - 19627, - 63554, - 60899, - 36578, - 29883, - 13350, - 12390, - 28718, - 4432, - 17587, - 27299, - 47271, - 44707, - 16571, - 277, - 16457, - 2188, - 12572, - 18900, - 55473, - 63714, - 64163, - 38070, - 8546, - 14662, - 18265, - 9099, - 34666, - 47734, - 47493, - 13336, - 31832, - 1119, - 23964, - 13343, - 3672, - 13847, - 7512, - 46478, - 56550, - 54755, - 39335, - 52455, - 46762, - 60903, - 16811, - 27051, - 25251, - 35193, - 18032, - 18036, - 18044, - 5480, - 11423, - 36306, - 36339, - 60841, - 59881, - 56568, - 60912, - 59825, - 2813, - 33900, - 41052, - 48530, - 7559, - 11194, - 65103, - 48022, - 30603, - 32727, - 9867, - 26448, - 16474, - 9216, - 20674, - 51681, - 59619, - 47578, - 28042, - 52587, - 64807, - 27039, - 41552, - 45423, - 29504, - 1625, - 12808, - 10568, - 10387, - 12391, - 59734, - 58663, - 27878, - 7669, - 2406, - 2283, - 3113, - 5354, - 3110, - 6634, - 3113, - 2282, - 2218, - 3178 - ], - "text": "Anh chỉ muốn được nhìn nhận như là một huấn luyện viên.", - "description": "Bình (nam miền Bắc)" - }, - "Tuyen": { - "codes": [ - 7274, - 18678, - 57063, - 4039, - 10524, - 29196, - 48716, - 16414, - 72, - 31953, - 31158, - 27598, - 54797, - 56551, - 23781, - 2488, - 36291, - 9482, - 776, - 30284, - 30028, - 29980, - 11495, - 30075, - 31195, - 47466, - 52407, - 60517, - 48236, - 22537, - 2942, - 1056, - 6069, - 4002, - 51381, - 40096, - 22945, - 18853, - 36035, - 12298, - 13824, - 2380, - 13834, - 652, - 13860, - 12458, - 1072, - 8570, - 9579, - 33835, - 55538, - 55537, - 54438, - 49398, - 54673, - 49382, - 49301, - 54502, - 53317, - 54405, - 26757, - 26721, - 10342, - 26790, - 40165, - 52386, - 11806, - 29260, - 2060, - 12382, - 18883, - 22458, - 60318, - 34478, - 63987, - 4549, - 8268, - 16552, - 522, - 8633, - 25416, - 49114, - 7847, - 18907, - 55978, - 34470, - 5615, - 28669, - 65011, - 3611, - 4636, - 20060, - 42062, - 44204, - 4200, - 2228, - 44531, - 56826, - 65011, - 11181, - 524, - 26700, - 28750, - 21972, - 16728, - 8764, - 34343, - 51361, - 50340, - 22692, - 3569, - 27722, - 24331, - 31512, - 23977, - 10409, - 30809, - 9289, - 22724, - 59399, - 35891, - 32801, - 36150, - 54321, - 43287, - 13577, - 6244, - 8486, - 30898, - 3686, - 16047, - 36727, - 52343, - 22705, - 2485, - 52707, - 20493, - 31571, - 13395, - 1360, - 13962, - 1872, - 13137, - 942, - 29609, - 598, - 16423, - 25859, - 27703, - 59683, - 39987, - 40247, - 42775, - 1285, - 12548, - 1608, - 13868, - 7627, - 51365, - 51636, - 52450, - 41370, - 34646, - 56627, - 35891, - 59683, - 1801, - 12809, - 18712, - 335, - 36248, - 12622, - 7318, - 56487, - 55525, - 59621, - 59552, - 31977, - 22693, - 6329, - 6229, - 35047, - 37085, - 13518, - 21036, - 11418, - 13990, - 13385, - 8341, - 61155, - 28942, - 12617, - 14424, - 13319, - 19606, - 47867, - 30862, - 64837, - 52243, - 19490, - 18482, - 38953, - 37123, - 14616, - 17166, - 3688, - 46607, - 18520, - 1371, - 14933, - 4623, - 29772, - 55374, - 56801, - 56470, - 39157, - 22613, - 26869, - 39206, - 10469, - 38949, - 5353, - 23722, - 38945, - 22950, - 10425, - 19634 - ], - "text": "Bạn cầm khúc cây, và ném vào bãi cỏ xanh tươi rậm rạp ở đằng xa.", - "description": "Tuyên (nam miền Bắc)" - }, - "Vinh": { - "codes": [ - 20757, - 12321, - 12553, - 13138, - 15011, - 15223, - 12558, - 52330, - 27847, - 12350, - 14969, - 30896, - 11761, - 8191, - 5888, - 5771, - 36083, - 8233, - 14166, - 14086, - 13449, - 13328, - 471, - 19874, - 3321, - 10435, - 16943, - 24612, - 104, - 16507, - 25173, - 48635, - 13219, - 15079, - 15534, - 424, - 20683, - 12392, - 4271, - 33271, - 34999, - 36247, - 62, - 17275, - 45164, - 8302, - 390, - 39347, - 30390, - 9379, - 64430, - 27543, - 36782, - 14179, - 11879, - 36963, - 56682, - 57446, - 6890, - 6566, - 3254, - 6310, - 6234, - 3242, - 6361, - 6313, - 5353, - 3110, - 5270, - 23653, - 5354, - 6297, - 2213, - 5354, - 7206, - 6373, - 6310, - 50294, - 47030, - 12551, - 812, - 29453, - 17548, - 24921, - 27026, - 31398, - 48115, - 25862, - 16920, - 13768, - 12294, - 14658, - 39858, - 39861, - 28659, - 7596, - 12829, - 8104, - 11798, - 52617, - 40358, - 6499, - 26774, - 22629, - 44163, - 13322, - 772, - 14220, - 19084, - 12826, - 19540, - 41563, - 32584, - 861, - 27537, - 49123, - 14023, - 25481, - 26763, - 14384, - 11434, - 59527, - 50522, - 22882, - 36339, - 8238, - 16647, - 28490, - 14690, - 15903, - 2657, - 32354, - 48575, - 29539, - 26747, - 30119, - 14493, - 16557, - 15452, - 20814, - 45065, - 20801, - 25924, - 11518, - 47157, - 54330, - 65213, - 29284, - 15468, - 4254, - 3165, - 734, - 43404, - 29341, - 31117, - 5274, - 9814, - 32473, - 15190, - 10823, - 53515, - 58521, - 23254, - 7394, - 3175, - 2286, - 7269, - 4330, - 3121, - 1254, - 2073, - 7402 - ], - "text": "Đến cuối thế kỷ 19, ngành đánh bắt cá được thương mại hóa.", - "description": "Vĩnh (nam miền Nam)" - }, - "Doan": { - "codes": [ - 2406, - 2546, - 40098, - 56787, - 19382, - 20387, - 26183, - 7057, - 12027, - 15786, - 10594, - 7051, - 5650, - 2707, - 56482, - 56753, - 11395, - 4365, - 19284, - 2876, - 55656, - 1630, - 55577, - 2474, - 60457, - 12074, - 36007, - 20343, - 51387, - 24507, - 43419, - 55662, - 56822, - 31875, - 24601, - 25470, - 17722, - 16645, - 43220, - 27300, - 6233, - 22567, - 60785, - 24586, - 3626, - 2421, - 31416, - 27827, - 2935, - 31398, - 28506, - 56481, - 35010, - 12301, - 13129, - 15212, - 9999, - 52389, - 18661, - 52466, - 39911, - 20451, - 18807, - 16762, - 4384, - 2200, - 31322, - 15001, - 2452, - 11033, - 40097, - 59521, - 12570, - 4683, - 10345, - 30283, - 1623, - 31306, - 5707, - 7836, - 13655, - 21676, - 613, - 14198, - 64382, - 19314, - 1654, - 1576, - 35450, - 52646, - 39161, - 39146, - 22002, - 22757, - 38118, - 17894, - 35297, - 2790, - 2193, - 7670, - 6481, - 7665, - 2386, - 3490, - 2137, - 36282, - 60430, - 14350, - 6158, - 8044, - 24360, - 27576, - 49011, - 47991, - 47963, - 30478, - 56037, - 55385, - 34862, - 13867, - 1336, - 30526, - 41855, - 8488, - 26035, - 55729, - 34038, - 22630, - 36337, - 13387, - 14089, - 16161, - 32620, - 48949, - 43964, - 36520, - 2028, - 19877, - 52470, - 22761, - 23732, - 60834, - 59482, - 45115, - 13861, - 1448, - 797, - 2787, - 36595, - 24922, - 9889, - 14311, - 15190, - 13215, - 34407, - 51378, - 36243, - 5180, - 16731, - 1917, - 9314, - 31397, - 29277, - 12396, - 4264, - 24740, - 27052, - 297, - 16485, - 21932, - 14170, - 12910, - 13067, - 13917, - 25933, - 50341, - 16626, - 35049, - 37298, - 39140, - 37618, - 22757, - 20897, - 7653, - 5281, - 11476, - 4192, - 7313, - 2402, - 7650, - 51302, - 43691, - 20331, - 3271, - 5453, - 30808, - 2847, - 7749, - 40338, - 52464, - 11402, - 28234, - 15416, - 3324, - 16184, - 43958, - 36586, - 32554, - 48355, - 52598, - 36074, - 3300, - 36261, - 56675, - 270, - 2836, - 3684, - 15985, - 16313, - 45052, - 11181, - 42586, - 52722, - 19680, - 18864, - 18656, - 18789, - 36209, - 43094, - 17451, - 810, - 55060, - 34964, - 2345, - 10325, - 52594, - 35041, - 6245, - 51618, - 4106, - 1912, - 13925, - 48045, - 30287, - 12366, - 33933, - 45389, - 45903, - 2653, - 9051, - 362, - 55495, - 51441, - 4104, - 13930, - 29517, - 15213, - 42247, - 51316, - 18658, - 59570, - 20467, - 28319, - 14360, - 6552, - 7969, - 44130, - 53353, - 51574, - 19669, - 20209, - 55365, - 38938, - 37163, - 17980, - 38, - 10349, - 13643, - 25863, - 26732, - 1291, - 36275, - 36727, - 61411, - 24730, - 12322, - 2228, - 30506, - 41833, - 24940, - 32796, - 45420, - 45973, - 17069, - 29518, - 17785, - 4679, - 5514, - 51595, - 51426, - 38134, - 37089, - 21734, - 22757, - 20689, - 22752, - 1188, - 3298 - ], - "text": "Nuôi con theo phong cách Do Thái, không chỉ tốt cho đứa trẻ, mà còn tốt cho cả các bậc cha mẹ.", - "description": "Đoan (nữ miền Nam)" - }, - "Ly": { - "codes": [ - 40987, - 13225, - 20405, - 20135, - 6007, - 12266, - 22379, - 52646, - 56501, - 13443, - 13912, - 13591, - 13916, - 9751, - 13916, - 13655, - 2652, - 14091, - 31048, - 4891, - 852, - 56739, - 27813, - 52707, - 10269, - 20060, - 58685, - 19020, - 20245, - 56563, - 13461, - 45399, - 9913, - 11049, - 373, - 4473, - 12844, - 4533, - 58788, - 58540, - 51378, - 13389, - 8718, - 11037, - 1805, - 56805, - 19702, - 54314, - 39471, - 40759, - 18951, - 34569, - 23336, - 5286, - 15792, - 56531, - 36021, - 63651, - 47783, - 23182, - 23378, - 20679, - 19841, - 18883, - 22917, - 28272, - 11418, - 7840, - 2565, - 27179, - 27938, - 59923, - 54866, - 9569, - 13178, - 14125, - 21055, - 55475, - 44198, - 30955, - 6310, - 2534, - 1058, - 6634, - 21861, - 43493, - 52722, - 12570, - 12880, - 14674, - 16161, - 16149, - 23955, - 31611, - 8091, - 2923, - 52662, - 44274, - 26790, - 36051, - 4633, - 18260, - 51031, - 2337, - 9750, - 10567, - 11616, - 55713, - 59862, - 56997, - 30882, - 14822, - 1126, - 2298, - 6183, - 6374, - 60883, - 47847, - 3695, - 31592, - 30469, - 21280, - 8502, - 45655, - 11691, - 22951, - 35326, - 19367, - 12163, - 7009, - 2950, - 9568, - 45734, - 12664, - 25782, - 57419, - 57434, - 62473, - 54550, - 41641, - 33381, - 51033, - 7005, - 8711, - 30493, - 25358, - 5917, - 40003, - 42470, - 1178, - 2278, - 2150, - 1253, - 23718 - ], - "text": "Chúng ta có thể áp dụng logic tương tự với người khác.", - "description": "Ly (nữ miền Bắc)" - }, - "Ngoc": { - "codes": [ - 37403, - 18205, - 20076, - 19745, - 22870, - 28331, - 14859, - 46095, - 50262, - 54273, - 24589, - 7757, - 1566, - 19737, - 14619, - 7721, - 32505, - 47959, - 46502, - 48362, - 48551, - 47591, - 26331, - 63071, - 38955, - 34833, - 51217, - 45317, - 29528, - 45832, - 13065, - 11078, - 60817, - 54485, - 56418, - 54502, - 40368, - 39107, - 16975, - 20324, - 24254, - 7857, - 35254, - 42983, - 62954, - 64427, - 63207, - 58783, - 23462, - 25908, - 23458, - 27301, - 27238, - 39798, - 47801, - 31671, - 30453, - 31670, - 59637, - 56566, - 56566, - 56570, - 56546, - 60898, - 56550, - 60658, - 40162, - 44262, - 44258, - 23782, - 28130, - 19618, - 11746, - 3234, - 11750, - 3170, - 10727, - 3170, - 6630, - 2165, - 2227, - 3271, - 6214, - 7650, - 19698, - 39895, - 27619, - 22962, - 23459, - 23187, - 20841, - 18038, - 42346, - 26218, - 22070, - 24956, - 25143, - 48614, - 39783, - 55977, - 46678, - 30295, - 30107, - 62511, - 54586, - 52690, - 15436, - 31838, - 39788, - 14729, - 27146, - 56497, - 13449, - 26202, - 9086, - 18366, - 2682, - 9638, - 46709, - 28853, - 24934, - 55339, - 55334, - 50192, - 50272, - 42603, - 5689, - 5502, - 30008, - 61620, - 29108, - 54874, - 55354, - 51221, - 55377, - 57424, - 13993, - 9995, - 1836, - 1547, - 9245, - 26119, - 43589, - 31379, - 30951, - 47767, - 62798, - 58521, - 55395, - 35895, - 57, - 57512, - 17197, - 9785, - 45749, - 30470, - 29782, - 56583, - 35878, - 41219, - 34584, - 47717, - 15984, - 27894, - 28073, - 46998, - 42662, - 60307, - 45209, - 9498, - 47193, - 31007, - 26958, - 60841, - 44639, - 60826, - 24415, - 56681, - 3934, - 7069, - 23119, - 58525, - 36087 - ], - "text": "Trong phòng rất tù mù, nên có thể dễ dàng che dấu nó.", - "description": "Ngọc (nữ miền Bắc)" - } - } -} \ No newline at end of file diff --git a/src/vieneu/base.py b/src/vieneu/base.py deleted file mode 100644 index e44a0ea8ca70ebc1a2e925611c594c1045f9750a..0000000000000000000000000000000000000000 --- a/src/vieneu/base.py +++ /dev/null @@ -1,360 +0,0 @@ -from abc import ABC, abstractmethod -from pathlib import Path -from typing import Optional, Union, List, Dict, Any -import json -import numpy as np -import logging -from huggingface_hub import hf_hub_download -from sea_g2p import Normalizer - -# Configure logging -logger = logging.getLogger("Vieneu") - -class BaseVieneuTTS(ABC): - """ - Abstract base class for VieNeu-TTS implementations. - Provides shared functionality for voice management and common operations. - """ - - def __init__(self): - self.sample_rate = 24_000 - self.max_context = 2048 - self.hop_length = 480 - - self.assets_dir = Path(__file__).parent / "assets" - self._preset_voices: Dict[str, Any] = {} - self._default_voice: Optional[str] = None - self.normalizer = Normalizer() - self._ref_phoneme_cache: Dict[str, str] = {} - - # Watermarker placeholder - self.watermarker = None - self._init_watermarker() - - def _init_watermarker(self) -> None: - """Initialize optional audio watermarker.""" - try: - import perth - self.watermarker = perth.PerthImplicitWatermarker() - logger.info("🔒 Audio watermarking initialized (Perth)") - except (ImportError, AttributeError): - self.watermarker = None - - def _load_voices(self, backbone_repo: Optional[str], hf_token: Optional[str] = None, clear_existing: bool = False) -> None: - """Unified voice loading for Local and Remote paths.""" - if not backbone_repo: - return - - path_obj = Path(backbone_repo) - if path_obj.exists(): - # Local Path (Dir or File) - if path_obj.is_dir(): - json_path = path_obj / "voices.json" - else: - json_path = path_obj.parent / "voices.json" - - if json_path.exists(): - self._load_voices_from_file(json_path, clear_existing=clear_existing) - else: - if clear_existing: - self._preset_voices.clear() - logger.warning(f"Validation Warning: Local path '{backbone_repo}' missing 'voices.json'.") - logger.warning(f"Falling back to Custom Voice Cloning mode.") - else: - # Remote Repo - if clear_existing: - self._preset_voices.clear() - - try: - self._load_voices_from_repo(backbone_repo, hf_token) - except Exception as e: - logger.warning(f"Could not load voices from repo '{backbone_repo}': {e}") - logger.warning(f"Falling back to Custom Voice Cloning mode.") - - def _load_voices_from_file(self, file_path: Path, clear_existing: bool = False) -> None: - """Load voices from a local JSON file.""" - try: - if not file_path.exists(): - logger.error(f"Voice file not found: {file_path}") - return - - with open(file_path, 'r', encoding='utf-8') as f: - try: - data = json.load(f) - except json.JSONDecodeError as e: - logger.error(f"Invalid JSON in voice file {file_path}: {e}") - return - - if "presets" in data: - if clear_existing: - self._preset_voices.clear() - logger.info("🧹 Cleared existing voices for replacement") - - # Merge into existing presets - self._preset_voices.update(data["presets"]) - logger.info(f"📢 Loaded {len(data['presets'])} voices from {file_path.name}") - - # Update default voice if provided - if "default_voice" in data and data["default_voice"]: - self._default_voice = data["default_voice"] - - except Exception as e: - logger.error(f"Failed to load voices from {file_path}: {e}") - - def _load_voices_from_repo(self, repo_id: str, hf_token: Optional[str] = None) -> None: - """Download and load voices.json from a HuggingFace repo.""" - voices_file = None - try: - # 1. Try normal download (checks for updates from server) - voices_file = hf_hub_download( - repo_id=repo_id, - filename="voices.json", - token=hf_token, - repo_type="model" - ) - except Exception: - # 2. Network error? Try to use cached version if available - logger.warning(f"Network check failed for voices.json. Trying local cache...") - try: - voices_file = hf_hub_download( - repo_id=repo_id, - filename="voices.json", - token=hf_token, - repo_type="model", - local_files_only=True - ) - logger.info(f"✅ Using cached voices.json") - except Exception: - # 3. No cache available either - pass - - if voices_file: - self._load_voices_from_file(Path(voices_file)) - else: - logger.warning(f"Repository '{repo_id}' is missing 'voices.json'. Falling back to Custom Voice mode.") - - def list_preset_voices(self) -> List[tuple[str, str]]: - """List available preset voices as (description, id).""" - return [ - (v.get("description", k) if isinstance(v, dict) else str(v), k) - for k, v in self._preset_voices.items() - ] - - def get_preset_voice(self, voice_name: Optional[str] = None) -> Dict[str, Any]: - """ - Get reference codes and text for a preset voice. - - Args: - voice_name: Name of voice. If None, uses default_voice. - - Returns: - dict: { 'codes': Union[np.ndarray, 'torch.Tensor'], 'text': str } - """ - if voice_name is None: - voice_name = self._default_voice - if voice_name is None: - if self._preset_voices: - voice_name = next(iter(self._preset_voices)) - else: - raise ValueError("No voice specified and no preset voices available.") - - if voice_name not in self._preset_voices: - raise ValueError(f"Voice '{voice_name}' not found. Available: {self.list_preset_voices()}") - - voice_data = self._preset_voices[voice_name] - codes = voice_data["codes"] - - # Only convert to torch if explicitly requested or if we're not in turbo mode - if isinstance(codes, list): - if codes and isinstance(codes[0], float): - codes = np.array(codes, dtype=np.float32) - else: - # Là integer token sequence (Standard mode) - try: - import torch - codes = torch.tensor(codes, dtype=torch.long) - except ImportError: - codes = np.array(codes, dtype=np.int64) - - return {"codes": codes, "text": voice_data["text"]} - - def get_ref_phonemes(self, ref_text: str) -> str: - """ - Get phonemized version of reference text, using cache if available. - """ - if ref_text not in self._ref_phoneme_cache: - from vieneu_utils.phonemize_text import phonemize_with_dict - self._ref_phoneme_cache[ref_text] = phonemize_with_dict(ref_text) - return self._ref_phoneme_cache[ref_text] - - def save(self, audio: np.ndarray, output_path: Union[str, Path]) -> None: - """Save audio waveform to a file.""" - import soundfile as sf - sf.write(str(output_path), audio, self.sample_rate) - - def encode_reference(self, ref_audio_path: Union[str, Path]) -> Union[np.ndarray, 'torch.Tensor']: - """ - Encode reference audio to codes. - - Args: - ref_audio_path: Path to the reference audio file. - - Returns: - Union[np.ndarray, torch.Tensor]: Encoded codes. - """ - import librosa - wav, _ = librosa.load(ref_audio_path, sr=16000, mono=True) - - # If we have an ONNX encoder or specialized turbo encoder, handle it here - # For now, default backends still use torch - try: - import torch - wav_tensor = torch.from_numpy(wav).float().unsqueeze(0).unsqueeze(0) # [1, 1, T] - - # Ensure device and dtype compatibility - if hasattr(self.codec, "device"): - wav_tensor = wav_tensor.to(self.codec.device) - - with torch.no_grad(): - ref_codes = self.codec.encode_code(audio_or_path=wav_tensor).squeeze(0).squeeze(0) - return ref_codes - except ImportError: - raise ImportError("Torch is required for encode_reference in the current backend. Please install torch or use a backend that supports standalone encoding.") - - def _decode(self, codes_str: str) -> np.ndarray: - """ - Decode speech tokens to audio waveform. - - Args: - codes_str: String containing speech tokens. - - Returns: - np.ndarray: Decoded audio waveform. - """ - from .utils import extract_speech_ids - speech_ids = extract_speech_ids(codes_str) - - if len(speech_ids) == 0: - raise ValueError("No valid speech tokens found in the output.") - - # Onnx decode - if getattr(self, "_is_onnx_codec", False): - codes = np.array(speech_ids, dtype=np.int32)[np.newaxis, np.newaxis, :] - recon = self.codec.decode_code(codes) - # Torch decode - else: - try: - import torch - with torch.no_grad(): - codes = torch.tensor(speech_ids, dtype=torch.long)[None, None, :] - if hasattr(self.codec, "device"): - codes = codes.to(self.codec.device) - - recon = self.codec.decode_code(codes) - if hasattr(recon, "cpu"): - recon = recon.cpu() - if hasattr(recon, "numpy"): - recon = recon.numpy() - except ImportError: - raise ImportError("Torch is required for the current codec backend. Please install torch or use an ONNX-based codec.") - - - return recon[0, 0, :] - - def _resolve_ref_voice( - self, - voice: Optional[Dict[str, Any]] = None, - ref_audio: Optional[Union[str, Path]] = None, - ref_codes: Optional[Union[np.ndarray, 'torch.Tensor']] = None, - ref_text: Optional[str] = None - ) -> tuple[Union[np.ndarray, 'torch.Tensor'], str]: - """Resolve reference voice codes and text.""" - if voice is not None: - ref_codes = voice.get('codes', ref_codes) - ref_text = voice.get('text', ref_text) - - if ref_audio is not None and ref_codes is None: - ref_codes = self.encode_reference(ref_audio) - elif self._default_voice and (ref_codes is None or ref_text is None): - try: - voice_data = self.get_preset_voice(None) - ref_codes = voice_data['codes'] - ref_text = voice_data['text'] - except Exception: - pass - - if ref_codes is None or ref_text is None: - raise ValueError("Must provide either 'voice' dict or both 'ref_codes' and 'ref_text'.") - - return ref_codes, ref_text - - def _apply_watermark(self, wav: np.ndarray) -> np.ndarray: - """Apply watermark to audio if enabled.""" - if self.watermarker: - return self.watermarker.apply_watermark(wav, sample_rate=self.sample_rate) - return wav - - def _format_prompt( - self, - ref_codes: Union[List[int], 'torch.Tensor', np.ndarray], - ref_text: str, - input_text: str, - ref_phonemes: Optional[str] = None, - input_phonemes: Optional[str] = None - ) -> str: - """ - Format the prompt for the TTS model. - Common implementation for LMDeploy (Fast) and Remote backends. - Standard backend uses a specialized chat template via tokenizer. - """ - if isinstance(ref_codes, (np.ndarray, list)): - ref_codes_list = np.array(ref_codes).flatten().tolist() - else: - # Assume it's a torch tensor if torch is installed - try: - import torch - if isinstance(ref_codes, torch.Tensor): - ref_codes_list = ref_codes.flatten().tolist() - else: - ref_codes_list = ref_codes - except ImportError: - ref_codes_list = ref_codes - - # Import inside method to avoid potential circular dependencies between - # base TTS and phonemization utilities. - from vieneu_utils.phonemize_text import phonemize_with_dict - - ref_text_phones = ref_phonemes if ref_phonemes else self.get_ref_phonemes(ref_text) - input_text_phones = input_phonemes if input_phonemes else phonemize_with_dict(input_text, skip_normalize=True) - codes_str = "".join([f"<|speech_{idx}|>" for idx in ref_codes_list]) - - return ( - f"user: Convert the text to speech:<|TEXT_PROMPT_START|>{ref_text_phones} {input_text_phones}" - f"<|TEXT_PROMPT_END|>\nassistant:<|SPEECH_GENERATION_START|>{codes_str}" - ) - - @abstractmethod - def infer(self, text: str, **kwargs: Any) -> np.ndarray: - """Main inference method for single text.""" - pass - - @abstractmethod - def infer_batch(self, texts: List[str], apply_watermark: bool = True, **kwargs: Any) -> List[np.ndarray]: - """Main inference method for batch processing.""" - pass - - def close(self) -> None: - """Release resources.""" - pass - - def __enter__(self) -> 'BaseVieneuTTS': - return self - - def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: - self.close() - - def __del__(self): - try: - self.close() - except Exception: - pass diff --git a/src/vieneu/core_xpu.py b/src/vieneu/core_xpu.py deleted file mode 100644 index 905f21611bd80dab8f2c694eb0c539bcd71760be..0000000000000000000000000000000000000000 --- a/src/vieneu/core_xpu.py +++ /dev/null @@ -1,211 +0,0 @@ -import torch -import gc -import librosa -import numpy as np -from typing import Optional, Union, List -from pathlib import Path -from transformers import AutoTokenizer, AutoModelForCausalLM -import logging -from neucodec import NeuCodec, DistillNeuCodec -from .standard import VieNeuTTS -from vieneu_utils.phonemize_text import phonemize_batch - -logger = logging.getLogger("Vieneu.XPU") - -class XPUVieNeuTTS(VieNeuTTS): - """ - XPU (Intel Arc GPU) optimized implementation of VieNeu-TTS. - Uses native PyTorch XPU backend with bfloat16 and specialized compilation/warmup. - """ - - def __init__( - self, - backbone_repo="pnnbao-ump/VieNeu-TTS-0.3B-q4-gguf", - backbone_device="xpu", # Forced default - codec_repo="neuphonic/distill-neucodec", - codec_device="xpu", # Forced default - hf_token=None, - ): - # Ensure we are strictly on XPU - if backbone_device != "xpu": - logger.warning("XPUVieNeuTTS initialized with non-xpu device. Forcing 'xpu'.") - backbone_device = "xpu" - if codec_device != "xpu": - codec_device = "xpu" - - super().__init__( - backbone_repo=backbone_repo, - backbone_device=backbone_device, - codec_repo=codec_repo, - codec_device=codec_device, - hf_token=hf_token - ) - - def _load_backbone(self, backbone_repo, backbone_device, hf_token=None): - """XPU (Intel Arc GPU) loading implementation using native PyTorch XPU.""" - logger.info(f"Loading backbone from: {backbone_repo} on {backbone_device} (XPU) ...") - - # Verify XPU is available - if not hasattr(torch, 'xpu') or not torch.xpu.is_available(): - raise RuntimeError("XPU device requested but torch.xpu.is_available() returned False") - - self.tokenizer = AutoTokenizer.from_pretrained(backbone_repo, token=hf_token) - self.tokenizer.padding_side = "left" - - if self.tokenizer.pad_token is None: - self.tokenizer.pad_token = self.tokenizer.eos_token - - # Load in bfloat16 for XPU optimization - self.backbone = AutoModelForCausalLM.from_pretrained( - backbone_repo, - token=hf_token, - dtype=torch.bfloat16 - ).to(device="xpu") - - logger.info(f" ✅ Model loaded on XPU device") - - def _load_codec(self, codec_repo, codec_device): - """XPU (Intel Arc GPU) codec loading implementation.""" - logger.info(f"Loading codec from: {codec_repo} on {codec_device} (XPU) ...") - - if not hasattr(torch, 'xpu') or not torch.xpu.is_available(): - raise RuntimeError("XPU device requested but torch.xpu.is_available() returned False") - - match codec_repo: - case "neuphonic/neucodec": - self.codec = NeuCodec.from_pretrained(codec_repo) - self.codec.eval().to(device="xpu", dtype=torch.float32) - case "neuphonic/distill-neucodec": - self.codec = DistillNeuCodec.from_pretrained(codec_repo) - self.codec.eval().to(device="xpu", dtype=torch.float32) - case "neuphonic/neucodec-onnx-decoder-int8": - raise ValueError("ONNX decoder does not support XPU device. Use CPU codec.") - case _: - raise ValueError(f"Unsupported codec repository: {codec_repo}") - - logger.info(f" ✅ Codec loaded on XPU device") - - - - def _infer_torch(self, prompt_ids: list[int], temperature: float = 1.0, top_k: int = 50) -> str: - """XPU-specific inference using native PyTorch XPU with autocast.""" - prompt_tensor = torch.tensor(prompt_ids).unsqueeze(0).to("xpu") - speech_end_id = self.tokenizer.convert_tokens_to_ids("<|SPEECH_GENERATION_END|>") - - with torch.no_grad(): - # Use XPU autocast for performance - with torch.autocast(device_type="xpu", dtype=torch.bfloat16, enabled=True): - output_tokens = self.backbone.generate( - prompt_tensor, - max_length=self.max_context, - eos_token_id=speech_end_id, - do_sample=True, - temperature=temperature, - top_k=top_k, - use_cache=True, - min_new_tokens=50, - ) - - input_length = prompt_tensor.shape[-1] - output_str = self.tokenizer.decode( - output_tokens[0, input_length:].cpu().numpy().tolist(), add_special_tokens=False - ) - - # Cleanup XPU memory after generation - torch.xpu.synchronize() - torch.xpu.empty_cache() - - return output_str - - def encode_reference(self, ref_audio_path): - """Override to ensure input tensor is on XPU.""" - - wav, _ = librosa.load(ref_audio_path, sr=16000, mono=True) - wav_tensor = torch.from_numpy(wav).float().unsqueeze(0).unsqueeze(0) - - # Move to XPU explicitly - wav_tensor = wav_tensor.to(device="xpu", dtype=torch.float32) - - with torch.no_grad(): - ref_codes = self.codec.encode_code(audio_or_path=wav_tensor).squeeze(0).squeeze(0) - return ref_codes - - def close(self): - """Extended close to handle XPU cache clearing.""" - super().close() - try: - if hasattr(torch, 'xpu') and torch.xpu.is_available(): - torch.xpu.empty_cache() - except Exception: - pass - - def infer_batch( - self, - texts: list[str], - ref_audio: Optional[Union[str, Path]] = None, - ref_codes: Optional[Union[np.ndarray, torch.Tensor]] = None, - ref_text: Optional[str] = None, - voice: Optional[dict] = None, - temperature: float = 1.0, - top_k: int = 50, - skip_normalize: bool = False, - apply_watermark: bool = True - ) -> list[np.ndarray]: - """ - Thực hiện inference theo batch trên XPU sử dụng thuần PyTorch. - """ - ref_codes, ref_text = self._resolve_ref_voice(voice, ref_audio, ref_codes, ref_text) - - if not skip_normalize: - texts = [self.normalizer.normalize(t) for t in texts] - - # Pre-phonemize all inputs for performance - ref_phonemes = self.get_ref_phonemes(ref_text) - chunk_phonemes = phonemize_batch(texts, skip_normalize=True) - - # Prepare prompt for each chunk in batch - batch_prompt_ids = [] - for phonemes in chunk_phonemes: - prompt_ids = self._apply_chat_template(ref_codes, ref_phonemes, phonemes) - batch_prompt_ids.append(torch.tensor(prompt_ids)) - - inputs = self.tokenizer.pad( - {"input_ids": batch_prompt_ids}, - padding=True, - return_tensors="pt" - ).to(device="xpu") - - speech_end_id = self.tokenizer.convert_tokens_to_ids("<|SPEECH_GENERATION_END|>") - - with torch.no_grad(): - # Use XPU autocast for performance - with torch.autocast(device_type="xpu", dtype=torch.bfloat16, enabled=True): - output_tokens = self.backbone.generate( - **inputs, - max_length=self.max_context, - eos_token_id=speech_end_id, - do_sample=True, - temperature=temperature, - top_k=top_k, - use_cache=True, - min_new_tokens=50, - ) - - # Batch Decoding - results = [] - input_length = inputs["input_ids"].shape[-1] - - for i in range(len(texts)): - generated_ids = output_tokens[i, input_length:] - output_str = self.tokenizer.decode(generated_ids, add_special_tokens=False) - wav = self._decode(output_str) - - if apply_watermark: - wav = self._apply_watermark(wav) - - results.append(wav) - - torch.xpu.synchronize() - torch.xpu.empty_cache() - - return results \ No newline at end of file diff --git a/src/vieneu/factory.py b/src/vieneu/factory.py deleted file mode 100644 index 43aa30893602f48144fcbd637ce736c46ec9c080..0000000000000000000000000000000000000000 --- a/src/vieneu/factory.py +++ /dev/null @@ -1,35 +0,0 @@ - - -def Vieneu(mode="turbo", **kwargs): - """ - Factory function for VieNeu-TTS. - - Args: - mode: 'standard' (CPU/GPU-GGUF), 'fast' (GPU-LMDeploy), 'remote' (API), 'xpu' (Intel GPU) - **kwargs: Arguments for chosen class - - Returns: - BaseVieneuTTS: An instance of a VieNeu-TTS implementation. - """ - match mode: - case "remote" | "api": - from .remote import RemoteVieNeuTTS - return RemoteVieNeuTTS(**kwargs) - case "fast" | "gpu": - from .fast import FastVieNeuTTS - return FastVieNeuTTS(**kwargs) - case "turbo": - from .turbo import TurboVieNeuTTS - return TurboVieNeuTTS(**kwargs) - case "turbo_gpu": - from .turbo import TurboGPUVieNeuTTS - return TurboGPUVieNeuTTS(**kwargs) - case "xpu": - try: - from .core_xpu import XPUVieNeuTTS - return XPUVieNeuTTS(**kwargs) - except Exception as e: - raise RuntimeError(f"Failed to load XPU backend. Ensure Intel GPU drivers and torch.xpu are installed: {e}") from e - case "standard": - from .standard import VieNeuTTS - return VieNeuTTS(**kwargs) diff --git a/src/vieneu/fast.py b/src/vieneu/fast.py deleted file mode 100644 index 4e409a1b484374bb7ff73bdf6356a39ef8904656..0000000000000000000000000000000000000000 --- a/src/vieneu/fast.py +++ /dev/null @@ -1,339 +0,0 @@ -from pathlib import Path -from typing import Optional, Union, List, Generator, Any, Dict -import numpy as np -import torch -import gc -import logging -from collections import defaultdict -from .base import BaseVieneuTTS -from .utils import _compile_codec_with_triton, extract_speech_ids, _linear_overlap_add -from vieneu_utils.phonemize_text import phonemize_with_dict, phonemize_batch -from vieneu_utils.core_utils import split_text_into_chunks, join_audio_chunks -from neucodec import NeuCodec, DistillNeuCodec - -logger = logging.getLogger("Vieneu.Fast") - -class FastVieNeuTTS(BaseVieneuTTS): - """ - GPU-optimized VieNeu-TTS using LMDeploy TurbomindEngine. - """ - - def __init__( - self, - backbone_repo: str = "pnnbao-ump/VieNeu-TTS", - backbone_device: str = "cuda", - codec_repo: str = "neuphonic/distill-neucodec", - codec_device: str = "cuda", - memory_util: float = 0.3, - tp: int = 1, - enable_prefix_caching: bool = True, - quant_policy: int = 0, - enable_triton: bool = True, - max_batch_size: int = 4, - hf_token: Optional[str] = None, - ): - super().__init__() - - if backbone_device != "cuda" and not backbone_device.startswith("cuda:"): - raise ValueError("LMDeploy backend requires CUDA device") - - # Streaming configuration - self.streaming_overlap_frames = 1 - self.streaming_frames_per_chunk = 50 - self.streaming_lookforward = 5 - self.streaming_lookback = 50 - self.streaming_stride_samples = self.streaming_frames_per_chunk * self.hop_length - - self._is_turbo = "v2-Turbo" in backbone_repo - self.max_batch_size = max_batch_size - self._ref_cache: Dict[str, Any] = {} - self.stored_dict = defaultdict(dict) - - self._is_onnx_codec = False - self._is_vieneu_codec = False # Flag for VieNeu-Codec special ONNX handling - self._triton_enabled = False - - self._load_backbone_lmdeploy(backbone_repo, memory_util, tp, enable_prefix_caching, quant_policy, hf_token) - self._load_codec(codec_repo, codec_device, enable_triton) - self._load_voices(backbone_repo, hf_token) - self._warmup_model() - - logger.info("✅ FastVieNeuTTS with optimizations loaded successfully!") - logger.info(f" Max batch size: {self.max_batch_size}") - - def _load_backbone_lmdeploy(self, repo, memory_util, tp, enable_prefix_caching, quant_policy, hf_token=None): - logger.info(f"Loading backbone with LMDeploy from: {repo}") - if hf_token: - import os - os.environ["HF_TOKEN"] = hf_token - - try: - from lmdeploy import pipeline, TurbomindEngineConfig, GenerationConfig - except ImportError as e: - raise ImportError( - "Failed to import `lmdeploy`. Install with: pip install vieneu[gpu]" - ) from e - - backend_config = TurbomindEngineConfig( - cache_max_entry_count=memory_util, - tp=tp, - enable_prefix_caching=enable_prefix_caching, - dtype='bfloat16', - quant_policy=quant_policy - ) - self.backbone = pipeline(repo, backend_config=backend_config) - self.gen_config = GenerationConfig( - top_p=0.95, top_k=50, temperature=1.0, max_new_tokens=2048, - do_sample=True, min_new_tokens=40, - ) - - def _load_codec(self, codec_repo, codec_device, enable_triton): - logger.info(f"Loading codec from: {codec_repo} on {codec_device}") - match codec_repo: - case "neuphonic/neucodec": - self.codec = NeuCodec.from_pretrained(codec_repo) - self.codec.eval().to(codec_device) - case "neuphonic/distill-neucodec": - self.codec = DistillNeuCodec.from_pretrained(codec_repo) - self.codec.eval().to(codec_device) - case "neuphonic/neucodec-onnx-decoder-int8": - if codec_device != "cpu": - raise ValueError("ONNX decoder only runs on CPU") - try: - from neucodec import NeuCodecOnnxDecoder - except ImportError as e: - raise ImportError("Failed to import ONNX decoder.") from e - self.codec = NeuCodecOnnxDecoder.from_pretrained(codec_repo) - self._is_onnx_codec = True - case "pnnbao-ump/VieNeu-Codec": - import onnxruntime as ort - from huggingface_hub import hf_hub_download - logger.info(f"🚀 Loading VieNeu-Codec (ONNX) ...") - decoder_path = hf_hub_download(repo_id=codec_repo, filename="vieneu_decoder.onnx") - - providers = ["CPUExecutionProvider"] - if codec_device in ("gpu", "cuda"): - providers = ["CUDAExecutionProvider"] + providers - - self.decoder_sess = ort.InferenceSession(decoder_path, providers=providers) - self._is_onnx_codec = True - self._is_vieneu_codec = True - case _: - raise ValueError(f"Unsupported codec repository: {codec_repo}") - - if enable_triton and not self._is_onnx_codec and codec_device != "cpu": - self._triton_enabled = _compile_codec_with_triton(self.codec) - - def _warmup_model(self): - logger.info("🔥 Warming up model...") - try: - dummy_codes = list(range(10)) - dummy_prompt = self._format_prompt(dummy_codes, "warmup", "test") - _ = self.backbone([dummy_prompt], gen_config=self.gen_config, do_preprocess=False) - logger.info(" ✅ Warmup complete") - except Exception as e: - logger.warning(f" ⚠️ Warmup failed: {e}") - - def _decode(self, codes_str: str, voice: Optional[Dict[str, Any]] = None) -> np.ndarray: - if not getattr(self, "_is_vieneu_codec", False): - speech_ids = extract_speech_ids(codes_str) - if not speech_ids: - raise ValueError("No valid speech tokens found in the output.") - - if self._is_onnx_codec: - codes = np.array(speech_ids, dtype=np.int32)[np.newaxis, np.newaxis, :] - recon = self.codec.decode_code(codes) - else: - with torch.no_grad(): - codes = torch.tensor(speech_ids, dtype=torch.long)[None, None, :].to(self.codec.device) - recon = self.codec.decode_code(codes).cpu().numpy() - return recon[0, 0, :] - - # Handling VieNeu-Codec (v2-Turbo) ONNX decoding - speech_ids = extract_speech_ids(codes_str) - if not speech_ids: - return np.array([], dtype=np.float32) - - # Process inputs - tokens = np.array(speech_ids, dtype=np.int64)[None, :] - v_idx = 16 # Default - embedding = np.zeros((1, 128), dtype=np.float32) - - if voice: - v_idx = voice.get("voice_id", 16) - emb = voice.get("codes") - if isinstance(emb, (np.ndarray, list)): - embedding = np.array(emb, dtype=np.float32) - if embedding.ndim == 1: - embedding = embedding[None, :] - - inputs = { - "content_ids": tokens, - "voice_id": np.array([v_idx], dtype=np.int64), - "evoice_embedding": embedding - } - - audio = self.decoder_sess.run(None, inputs)[0] - if audio.ndim == 3: - return audio[0, 0, :] - elif audio.ndim == 2: - return audio[0, :] - return audio.flatten() - - - def infer(self, text: str, ref_audio: Optional[Union[str, Path]] = None, ref_codes: Optional[Union[np.ndarray, torch.Tensor]] = None, ref_text: Optional[str] = None, max_chars: int = 256, silence_p: float = 0.15, crossfade_p: float = 0.0, voice: Optional[Dict[str, Any]] = None, temperature: float = 1.0, top_k: int = 50, skip_normalize: bool = False, **kwargs) -> np.ndarray: - - ref_codes, ref_text = self._resolve_ref_voice(voice, ref_audio, ref_codes, ref_text) - - if not skip_normalize: - text = self.normalizer.normalize(text) - - self.gen_config.temperature = temperature - self.gen_config.top_k = top_k - - chunks = split_text_into_chunks(text, max_chars=max_chars) - if not chunks: - return np.array([], dtype=np.float32) - - skip_phonemize = kwargs.get("skip_phonemize", False) - - if len(chunks) == 1: - if skip_phonemize: - prompt = self._format_prompt(ref_codes, ref_text, "", input_phonemes=chunks[0]) - else: - prompt = self._format_prompt(ref_codes, ref_text, chunks[0]) - - responses = self.backbone([prompt], gen_config=self.gen_config, do_preprocess=False) - wav = self._decode(responses[0].text, voice=voice) - wav = self._apply_watermark(wav) - else: - all_wavs = self.infer_batch(chunks, ref_codes, ref_text, voice=voice, temperature=temperature, top_k=top_k, skip_normalize=True, **kwargs) - wav = join_audio_chunks(all_wavs, self.sample_rate, silence_p, crossfade_p) - - return wav - - def infer_batch(self, texts: List[str], ref_audio: Optional[Union[str, Path]] = None, ref_codes: Optional[Union[np.ndarray, torch.Tensor]] = None, ref_text: Optional[str] = None, voice: Optional[Dict[str, Any]] = None, temperature: float = 1.0, top_k: int = 50, skip_normalize: bool = False, apply_watermark: bool = True, max_batch_size: Optional[int] = None, **kwargs) -> List[np.ndarray]: - - if not skip_normalize: - texts = [self.normalizer.normalize(t) for t in texts] - - max_batch_size = max_batch_size or self.max_batch_size - - ref_codes, ref_text = self._resolve_ref_voice(voice, ref_audio, ref_codes, ref_text) - - # Pre-phonemize all for performance - ref_phonemes = self.get_ref_phonemes(ref_text) - - skip_phonemize = kwargs.get("skip_phonemize", False) - if skip_phonemize: - chunk_phonemes = texts - else: - chunk_phonemes = phonemize_batch(texts, skip_normalize=True) - - self.gen_config.temperature = temperature - self.gen_config.top_k = top_k - - all_wavs = [] - for i in range(0, len(texts), max_batch_size): - batch_texts = texts[i : i + max_batch_size] - batch_phonemes = chunk_phonemes[i : i + max_batch_size] - prompts = [self._format_prompt(ref_codes, ref_text, text, ref_phonemes=ref_phonemes, input_phonemes=ph) - for text, ph in zip(batch_texts, batch_phonemes)] - responses = self.backbone(prompts, gen_config=self.gen_config, do_preprocess=False) - batch_codes = [response.text for response in responses] - batch_wavs = [self._decode(codes, voice=voice) for codes in batch_codes] - if apply_watermark: - batch_wavs = [self._apply_watermark(w) for w in batch_wavs] - all_wavs.extend(batch_wavs) - return all_wavs - - def infer_stream(self, text: str, ref_audio: Optional[Union[str, Path]] = None, ref_codes: Optional[Union[np.ndarray, torch.Tensor]] = None, ref_text: Optional[str] = None, max_chars: int = 256, voice: Optional[Dict[str, Any]] = None, temperature: float = 1.0, top_k: int = 50, skip_normalize: bool = False, **kwargs) -> Generator[np.ndarray, None, None]: - - ref_codes, ref_text = self._resolve_ref_voice(voice, ref_audio, ref_codes, ref_text) - - if not skip_normalize: - text = self.normalizer.normalize(text) - - self.gen_config.temperature = temperature - self.gen_config.top_k = top_k - - chunks = split_text_into_chunks(text, max_chars=max_chars) - for chunk in chunks: - yield from self._infer_stream_single(chunk, ref_codes, ref_text, voice=voice) - - def _infer_stream_single(self, text: str, ref_codes: Union[np.ndarray, torch.Tensor, List[int]], ref_text: str, voice: Optional[Dict[str, Any]] = None) -> Generator[np.ndarray, None, None]: - if isinstance(ref_codes, (torch.Tensor, np.ndarray)): - ref_codes_list = ref_codes.flatten().tolist() - else: - ref_codes_list = ref_codes - - prompt = self._format_prompt(ref_codes_list, ref_text, text) - audio_cache = [] - token_cache = [f"<|speech_{idx}|>" for idx in ref_codes_list] - n_decoded_samples = 0 - n_decoded_tokens = len(ref_codes_list) - - for response in self.backbone.stream_infer([prompt], gen_config=self.gen_config, do_preprocess=False): - output_str = response.text - new_tokens = output_str[len("".join(token_cache[len(ref_codes_list):])):] if len(token_cache) > len(ref_codes_list) else output_str - if new_tokens: - token_cache.append(new_tokens) - - if len(token_cache[n_decoded_tokens:]) >= self.streaming_frames_per_chunk + self.streaming_lookforward: - tokens_start = max(n_decoded_tokens - self.streaming_lookback - self.streaming_overlap_frames, 0) - tokens_end = n_decoded_tokens + self.streaming_frames_per_chunk + self.streaming_lookforward + self.streaming_overlap_frames - sample_start = (n_decoded_tokens - tokens_start) * self.hop_length - sample_end = sample_start + (self.streaming_frames_per_chunk + 2 * self.streaming_overlap_frames) * self.hop_length - curr_codes = token_cache[tokens_start:tokens_end] - recon = self._decode("".join(curr_codes), voice=voice) - recon = self._apply_watermark(recon) - recon = recon[sample_start:sample_end] - audio_cache.append(recon) - - processed_recon = _linear_overlap_add(audio_cache, stride=self.streaming_stride_samples) - new_samples_end = len(audio_cache) * self.streaming_stride_samples - processed_recon = processed_recon[n_decoded_samples:new_samples_end] - n_decoded_samples = new_samples_end - n_decoded_tokens += self.streaming_frames_per_chunk - yield processed_recon - - remaining_tokens = len(token_cache) - n_decoded_tokens - if remaining_tokens > 0: - tokens_start = max(len(token_cache) - (self.streaming_lookback + self.streaming_overlap_frames + remaining_tokens), 0) - sample_start = (len(token_cache) - tokens_start - remaining_tokens - self.streaming_overlap_frames) * self.hop_length - curr_codes = token_cache[tokens_start:] - recon = self._decode("".join(curr_codes), voice=voice) - recon = self._apply_watermark(recon) - recon = recon[sample_start:] - audio_cache.append(recon) - processed_recon = _linear_overlap_add(audio_cache, stride=self.streaming_stride_samples) - processed_recon = processed_recon[n_decoded_samples:] - yield processed_recon - - def cleanup_memory(self): - if torch.cuda.is_available(): - torch.cuda.empty_cache() - gc.collect() - - def _format_prompt(self, ref_codes, ref_text, input_text, ref_phonemes=None, input_phonemes=None): - if not self._is_turbo: - return super()._format_prompt(ref_codes, ref_text, input_text, ref_phonemes, input_phonemes) - - # Turbo v2 format (ALWAYS uses speaker_16 in prompt) - from vieneu_utils.phonemize_text import phonemize_with_dict - phonemes = input_phonemes if input_phonemes else phonemize_with_dict(input_text, skip_normalize=True) - - return ( - f"<|speaker_16|>" - f"<|TEXT_PROMPT_START|>{phonemes}<|TEXT_PROMPT_END|>" - f"<|SPEECH_GENERATION_START|>" - ) - - def get_optimization_stats(self) -> Dict[str, Any]: - return { - 'triton_enabled': self._triton_enabled, - 'max_batch_size': self.max_batch_size, - 'cached_references': len(self._ref_cache), - 'active_sessions': len(self.stored_dict), - 'prefix_caching': True, - } diff --git a/src/vieneu/remote.py b/src/vieneu/remote.py deleted file mode 100644 index 6ab0a2a28030dd4426c2c8f40965afa7d823b5cb..0000000000000000000000000000000000000000 --- a/src/vieneu/remote.py +++ /dev/null @@ -1,279 +0,0 @@ -from pathlib import Path -from typing import Optional, Union, List, Generator, Any, Dict -import numpy as np -import torch -import requests -import json -import asyncio -import logging -from .standard import VieNeuTTS -from .utils import _linear_overlap_add -from vieneu_utils.phonemize_text import phonemize_with_dict -from vieneu_utils.core_utils import split_text_into_chunks, join_audio_chunks - -logger = logging.getLogger("Vieneu.Remote") - -class RemoteVieNeuTTS(VieNeuTTS): - """ - Client for VieNeu-TTS running on a remote LMDeploy server. - """ - - def __init__( - self, - api_base: str = "http://localhost:23333/v1", - model_name: str = "pnnbao-ump/VieNeu-TTS", - codec_repo: str = "neuphonic/distill-neucodec", - codec_device: str = "cpu", - hf_token: Optional[str] = None - ): - self.api_base = api_base.rstrip('/') - self.model_name = model_name - - super().__init__( - backbone_repo=None, - codec_repo=codec_repo, - codec_device=codec_device, - hf_token=hf_token - ) - - self.streaming_frames_per_chunk = 10 - self.streaming_lookforward = 5 - self.streaming_lookback = 50 - self.streaming_stride_samples = self.streaming_frames_per_chunk * self.hop_length - self._load_voices_from_repo(model_name, hf_token) - - def _load_backbone(self, backbone_repo, backbone_device, hf_token=None): - pass - - - def infer(self, text: str, ref_audio: Optional[Union[str, Path]] = None, ref_codes: Optional[Union[np.ndarray, torch.Tensor]] = None, ref_text: Optional[str] = None, max_chars: int = 256, silence_p: float = 0.15, crossfade_p: float = 0.0, voice: Optional[Dict[str, Any]] = None, temperature: float = 1.0, top_k: int = 50, skip_normalize: bool = False) -> np.ndarray: - - ref_codes, ref_text = self._resolve_ref_voice(voice, ref_audio, ref_codes, ref_text) - - if not skip_normalize: - text = self.normalizer.normalize(text) - - chunks = split_text_into_chunks(text, max_chars=max_chars) - if not chunks: - return np.array([], dtype=np.float32) - - if len(chunks) == 1: - prompt = self._format_prompt(ref_codes, ref_text, chunks[0]) - payload = { - "model": self.model_name, - "messages": [{"role": "user", "content": prompt}], - "max_tokens": 2048, - "temperature": temperature, - "top_k": top_k, - "stop": ["<|SPEECH_GENERATION_END|>"], - "stream": False - } - try: - response = requests.post(f"{self.api_base}/chat/completions", json=payload, timeout=60) - response.raise_for_status() - output_str = response.json()["choices"][0]["message"]["content"] - wav = self._decode(output_str) - return self._apply_watermark(wav) - except Exception as e: - logger.error(f"Error during remote inference: {e}") - return np.array([], dtype=np.float32) - - # For multiple chunks, use async for parallel processing - return asyncio.run(self.infer_async( - text, ref_codes=ref_codes, ref_text=ref_text, - max_chars=max_chars, silence_p=silence_p, crossfade_p=crossfade_p, - temperature=temperature, top_k=top_k, skip_normalize=True, apply_watermark=True - )) - - def infer_stream(self, text: str, ref_audio: Optional[Union[str, Path]] = None, ref_codes: Optional[Union[np.ndarray, torch.Tensor]] = None, ref_text: Optional[str] = None, max_chars: int = 256, voice: Optional[Dict[str, Any]] = None, temperature: float = 1.0, top_k: int = 50, skip_normalize: bool = False) -> Generator[np.ndarray, None, None]: - - ref_codes, ref_text = self._resolve_ref_voice(voice, ref_audio, ref_codes, ref_text) - - if not skip_normalize: - text = self.normalizer.normalize(text) - - chunks = split_text_into_chunks(text, max_chars=max_chars) - for chunk in chunks: - yield from self._infer_stream_chunk(chunk, ref_codes, ref_text, temperature, top_k) - - def _infer_stream_chunk(self, chunk, ref_codes, ref_text, temperature, top_k): - prompt = self._format_prompt(ref_codes, ref_text, chunk) - payload = { - "model": self.model_name, - "messages": [{"role": "user", "content": prompt}], - "max_tokens": 2048, - "temperature": temperature, - "top_k": top_k, - "stop": ["<|SPEECH_GENERATION_END|>"], - "stream": True - } - - if isinstance(ref_codes, (torch.Tensor, np.ndarray)): - ref_codes_list = ref_codes.flatten().tolist() - else: - ref_codes_list = ref_codes - - audio_cache: List[np.ndarray] = [] - token_cache: List[str] = [f"<|speech_{idx}|>" for idx in ref_codes_list] - n_decoded_samples: int = 0 - n_decoded_tokens: int = len(ref_codes_list) - - try: - with requests.post(f"{self.api_base}/chat/completions", json=payload, stream=True, timeout=60) as r: - r.raise_for_status() - for line in r.iter_lines(): - if not line: continue - line_str = line.decode('utf-8') - if not line_str.startswith('data: '): continue - data_str = line_str[6:] - if data_str == '[DONE]': break - try: - content = json.loads(data_str)["choices"][0]["delta"].get("content", "") - if content: - token_cache.append(content) - if len(token_cache[n_decoded_tokens:]) >= self.streaming_frames_per_chunk + self.streaming_lookforward: - tokens_start = max(n_decoded_tokens - self.streaming_lookback - self.streaming_overlap_frames, 0) - tokens_end = n_decoded_tokens + self.streaming_frames_per_chunk + self.streaming_lookforward + self.streaming_overlap_frames - sample_start = (n_decoded_tokens - tokens_start) * self.hop_length - sample_end = sample_start + (self.streaming_frames_per_chunk + 2 * self.streaming_overlap_frames) * self.hop_length - curr_codes = token_cache[tokens_start:tokens_end] - recon = self._decode("".join(curr_codes)) - recon = self._apply_watermark(recon) - recon = recon[sample_start:sample_end] - audio_cache.append(recon) - processed_recon = _linear_overlap_add(audio_cache, stride=self.streaming_stride_samples) - new_samples_end = len(audio_cache) * self.streaming_stride_samples - processed_recon = processed_recon[n_decoded_samples:new_samples_end] - n_decoded_samples = new_samples_end - n_decoded_tokens += self.streaming_frames_per_chunk - yield processed_recon - except json.JSONDecodeError: continue - except Exception as e: - logger.error(f"Error streaming chunk: {e}") - return - - remaining_tokens = len(token_cache) - n_decoded_tokens - if remaining_tokens > 0: - tokens_start = max(len(token_cache) - (self.streaming_lookback + self.streaming_overlap_frames + remaining_tokens), 0) - sample_start = (len(token_cache) - tokens_start - remaining_tokens - self.streaming_overlap_frames) * self.hop_length - curr_codes = token_cache[tokens_start:] - recon = self._decode("".join(curr_codes)) - recon = self._apply_watermark(recon) - recon = recon[sample_start:] - audio_cache.append(recon) - processed_recon = _linear_overlap_add(audio_cache, stride=self.streaming_stride_samples) - processed_recon = processed_recon[n_decoded_samples:] - yield processed_recon - - async def infer_async(self, text: str, ref_audio: Optional[Union[str, Path]] = None, ref_codes: Optional[Union[np.ndarray, torch.Tensor]] = None, ref_text: Optional[str] = None, max_chars: int = 256, silence_p: float = 0.15, crossfade_p: float = 0.0, voice: Optional[Dict[str, Any]] = None, temperature: float = 1.0, top_k: int = 50, session=None, skip_normalize: bool = False, apply_watermark: bool = True) -> np.ndarray: - try: - import aiohttp - except ImportError: - raise ImportError("Async requires 'aiohttp'.") - - ref_codes, ref_text = self._resolve_ref_voice(voice, ref_audio, ref_codes, ref_text) - - if not skip_normalize: - text = self.normalizer.normalize(text) - - chunks = split_text_into_chunks(text, max_chars=max_chars) - if not chunks: - return np.array([], dtype=np.float32) - - should_close_session = False - if session is None: - session = aiohttp.ClientSession() - should_close_session = True - - try: - tasks = [self._infer_chunk_async(session, chunk, ref_codes, ref_text, temperature, top_k) for chunk in chunks] - wavs = await asyncio.gather(*tasks) - final_wav = join_audio_chunks(wavs, self.sample_rate, silence_p, crossfade_p) - if apply_watermark: - final_wav = self._apply_watermark(final_wav) - return final_wav - finally: - if should_close_session: - await session.close() - - def infer_batch(self, texts: List[str], ref_audio: Optional[Union[str, Path]] = None, ref_codes: Optional[Union[np.ndarray, torch.Tensor]] = None, ref_text: Optional[str] = None, voice: Optional[Dict[str, Any]] = None, temperature: float = 1.0, top_k: int = 50, skip_normalize: bool = False, apply_watermark: bool = True) -> List[np.ndarray]: - """Synchronous wrapper for async batch inference.""" - return asyncio.run(self.infer_batch_async( - texts, ref_audio=ref_audio, ref_codes=ref_codes, ref_text=ref_text, - voice=voice, temperature=temperature, top_k=top_k, skip_normalize=skip_normalize, - apply_watermark=apply_watermark - )) - - async def _infer_chunk_async( - self, - session, - chunk: str, - ref_codes: Union[List[int], torch.Tensor, np.ndarray], - ref_text: str, - temperature: float, - top_k: int, - ref_phonemes: Optional[str] = None, - chunk_phonemes: Optional[str] = None - ) -> np.ndarray: - """Internal helper for asynchronous chunk inference.""" - prompt = self._format_prompt(ref_codes, ref_text, chunk, ref_phonemes=ref_phonemes, input_phonemes=chunk_phonemes) - payload = { - "model": self.model_name, - "messages": [{"role": "user", "content": prompt}], - "max_tokens": 2048, - "temperature": temperature, - "top_k": top_k, - "stop": ["<|SPEECH_GENERATION_END|>"], - "stream": False - } - try: - async with session.post(f"{self.api_base}/chat/completions", json=payload, timeout=60) as resp: - resp.raise_for_status() - data = await resp.json() - output_str = data["choices"][0]["message"]["content"] - return self._decode(output_str) - except Exception as e: - logger.error(f"Error in async chunk: {e}") - return np.array([], dtype=np.float32) - - async def infer_batch_async(self, texts: List[str], ref_audio: Optional[Union[str, Path]] = None, ref_codes: Optional[Union[np.ndarray, torch.Tensor]] = None, ref_text: Optional[str] = None, max_chars: int = 256, silence_p: float = 0.15, crossfade_p: float = 0.0, voice: Optional[Dict[str, Any]] = None, temperature: float = 1.0, top_k: int = 50, concurrency_limit: int = 50, skip_normalize: bool = False, apply_watermark: bool = True) -> List[np.ndarray]: - try: - import aiohttp - except ImportError: - raise ImportError("Async requires 'aiohttp'.") - - if not skip_normalize: - texts = [self.normalizer.normalize(t) for t in texts] - - ref_codes, ref_text = self._resolve_ref_voice(voice, ref_audio, ref_codes, ref_text) - - # Pre-phonemize all for performance - ref_phonemes = self.get_ref_phonemes(ref_text) - all_phonemes = phonemize_batch(texts, skip_normalize=True) - - sem = asyncio.Semaphore(concurrency_limit) - async with aiohttp.ClientSession() as session: - async def bounded_infer(text, ph): - async with sem: - # Split into chunks internally if needed - chunks = split_text_into_chunks(text, max_chars=max_chars) - if not chunks: return np.array([], dtype=np.float32) - - if len(chunks) == 1: - wav = await self._infer_chunk_async(session, chunks[0], ref_codes, ref_text, temperature, top_k, ref_phonemes=ref_phonemes, chunk_phonemes=ph) - if apply_watermark: wav = self._apply_watermark(wav) - return wav - - # Re-phonemize chunks if splitting happened - chunk_phonemes = phonemize_batch(chunks, skip_normalize=True) - tasks = [self._infer_chunk_async(session, c, ref_codes, ref_text, temperature, top_k, ref_phonemes=ref_phonemes, chunk_phonemes=c_ph) - for c, c_ph in zip(chunks, chunk_phonemes)] - wavs = await asyncio.gather(*tasks) - final_wav = join_audio_chunks(wavs, self.sample_rate, silence_p, crossfade_p) - if apply_watermark: final_wav = self._apply_watermark(final_wav) - return final_wav - - tasks = [bounded_infer(text, ph) for text, ph in zip(texts, all_phonemes)] - results = await asyncio.gather(*tasks) - - return results diff --git a/src/vieneu/serve.py b/src/vieneu/serve.py deleted file mode 100644 index a9edcc03174ef1bf10bbfc272133eac225391d55..0000000000000000000000000000000000000000 --- a/src/vieneu/serve.py +++ /dev/null @@ -1,118 +0,0 @@ -import argparse -import os -import subprocess -import sys -import time -import requests -import logging - -# Configure logging -logging.basicConfig(level=logging.INFO, format='%(message)s') -logger = logging.getLogger("Vieneu.Serve") - -def check_command(cmd: str) -> bool: - try: - subprocess.run([cmd, "--version"], capture_output=True, check=False) - return True - except FileNotFoundError: - return False - -def get_public_ip() -> str: - try: - return requests.get("https://api.ipify.org").text - except Exception: - return "your-server-ip" - -def run_server(args: argparse.Namespace) -> None: - """ - Starts the LMDeploy API server. - """ - logger.info(f"🚀 Starting VieNeu-TTS Remote Server...") - logger.info(f"📦 Model: {args.model}") - - cmd = [ - "lmdeploy", "serve", "api_server", - args.model, - "--server-name", "0.0.0.0", - "--server-port", str(args.port), - "--tp", str(args.tp), - "--cache-max-entry-count", str(args.memory_util), - "--model-name", args.model_name - ] - - if args.quant_policy: - cmd.extend(["--quant-policy", str(args.quant_policy)]) - - logger.info(f"🛠️ Command: {' '.join(cmd)}") - - # Start the server in a subprocess - server_process = subprocess.Popen(cmd) - - # Wait for server to start - logger.info(f"⏳ Waiting for server to initialize on port {args.port}...") - - # Optional Tunneling - tunnel_process = None - public_url = None - - if args.tunnel: - if check_command("bore"): - logger.info("🌐 Starting tunnel via 'bore'...") - tunnel_cmd = ["bore", "local", str(args.port), "--to", "bore.pub"] - tunnel_process = subprocess.Popen(tunnel_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) - - # Try to catch the public URL from bore output - start_time = time.time() - while time.time() - start_time < 10: - line = tunnel_process.stdout.readline() - if "listening at" in line: - public_url = line.split("listening at")[-1].strip() - logger.info(f"✅ Public URL: http://{public_url}") - break - else: - logger.warning("⚠️ 'bore' not found. Please install it to use --tunnel (https://github.com/ekzhang/bore)") - logger.info(f"📍 Using local address: http://{get_public_ip()}:{args.port}") - else: - logger.info(f"✅ Server running locally at: http://0.0.0.0:{args.port}") - logger.info(f"📍 Public access (if enabled): http://{get_public_ip()}:{args.port}") - - logger.info("\n💡 To use this server in your SDK:") - sdk_url = f"http://{public_url}" if public_url else f"http://{get_public_ip()}:{args.port}" - logger.info(f" from vieneu import Vieneu") - logger.info(f" tts = Vieneu(mode='remote', api_base='{sdk_url}/v1', model_name='{args.model_name}')") - logger.info("") - - try: - server_process.wait() - except KeyboardInterrupt: - logger.info("\n🛑 Stopping server...") - server_process.terminate() - if tunnel_process: - tunnel_process.terminate() - -def main() -> None: - parser = argparse.ArgumentParser(description="VieNeu-TTS Remote Server CLI") - parser.add_argument("--model", type=str, default="pnnbao-ump/VieNeu-TTS", help="HuggingFace model ID or local path") - parser.add_argument("--model-name", type=str, default="pnnbao-ump/VieNeu-TTS", help="Model name for API mapping") - parser.add_argument("--port", type=int, default=23333, help="Server port") - parser.add_argument("--tp", type=int, default=1, help="Tensor parallel size") - parser.add_argument("--memory-util", type=float, default=0.3, help="GPU memory utilization (0.0-1.0)") - parser.add_argument("--quant-policy", type=int, default=0, help="KV cache quantization (0, 4, 8)") - parser.add_argument("--tunnel", action="store_true", help="Automatically expose the server via bore.pub") - - args = parser.parse_args() - - # Sync model_name with model if model is provided but model_name is default - if args.model != "pnnbao-ump/VieNeu-TTS" and args.model_name == "pnnbao-ump/VieNeu-TTS": - args.model_name = args.model - - # Check if lmdeploy is installed - if not check_command("lmdeploy"): - logger.error("❌ 'lmdeploy' not found!") - logger.error(" Please install it using: pip install vieneu[gpu]") - sys.exit(1) - - run_server(args) - -if __name__ == "__main__": - main() diff --git a/src/vieneu/standard.py b/src/vieneu/standard.py deleted file mode 100644 index 59f9728784635b3e1a3fd4b0b4d8e67505a61568..0000000000000000000000000000000000000000 --- a/src/vieneu/standard.py +++ /dev/null @@ -1,587 +0,0 @@ -import os -import platform -from pathlib import Path -from typing import Optional, Union, List, Generator, Any, Dict -import numpy as np -import torch -import gc -import logging -from .base import BaseVieneuTTS -from .utils import extract_speech_ids, _linear_overlap_add -from vieneu_utils.phonemize_text import phonemize_with_dict, phonemize_batch -from vieneu_utils.core_utils import split_text_into_chunks, join_audio_chunks - -logger = logging.getLogger("Vieneu.Standard") - -class VieNeuTTS(BaseVieneuTTS): - """ - Standard VieNeu-TTS implementation. - Supports PyTorch + Transformers backend and GGUF quantized models. - """ - - def __init__( - self, - backbone_repo: str = "pnnbao-ump/VieNeu-TTS-0.3B-q4-gguf", - backbone_device: str = "cpu", - codec_repo: str = "neuphonic/distill-neucodec", - codec_device: str = "cpu", - hf_token: Optional[str] = None, - ): - super().__init__() - - # Streaming configuration - self.streaming_overlap_frames = 1 - self.streaming_frames_per_chunk = 25 - self.streaming_lookforward = 10 - self.streaming_lookback = 100 - self.streaming_stride_samples = self.streaming_frames_per_chunk * self.hop_length - - self._is_quantized_model = False - self._is_onnx_codec = False - self.tokenizer = None - self.backbone = None - self.codec = None - self._is_turbo = backbone_repo and "v2-Turbo" in backbone_repo - self._is_vieneu_codec = False # Flag for VieNeu-Codec special ONNX handling - - if backbone_repo: - self._load_backbone(backbone_repo, backbone_device, hf_token) - self._load_codec(codec_repo, codec_device) - self._load_voices(backbone_repo, hf_token) - # Warmup is skipped by default to prevent startup hungs on CPU/Spaces. - - def _warmup_model(self) -> None: - """Warm up the model to initialize CUDA/XPU kernels and KV cache.""" - try: - logger.info("🔥 Warming up standard model...") - dummy_text = "Xin chào" - # Using very short dummy ref to speed up - dummy_ref_codes = torch.zeros(10, dtype=torch.long) - dummy_ref_text = "Chào" - _ = self.infer(dummy_text, ref_codes=dummy_ref_codes, ref_text=dummy_ref_text, max_chars=16) - logger.info(" ✅ Warmup complete") - except Exception as e: - logger.warning(f" ⚠️ Warmup failed: {e}") - - def close(self) -> None: - """Explicitly release model resources.""" - try: - if self.backbone is not None: - if self._is_quantized_model: - close_fn = getattr(self.backbone, "close", None) - if callable(close_fn): - close_fn() - self.backbone = None - - if self.codec is not None: - self.codec = None - - gc.collect() - except Exception as e: - # During shutdown, globals like 'logger' might be None. - if logger is not None: - logger.error(f"Error during VieNeuTTS closure: {e}") - - def _load_backbone(self, backbone_repo: str, backbone_device: str, hf_token: Optional[str] = None) -> None: - if backbone_device == "mps" and not torch.backends.mps.is_available(): - logger.warning("MPS not available, falling back to CPU") - backbone_device = "cpu" - - logger.info(f"Loading backbone from: {backbone_repo} on {backbone_device} ...") - - if backbone_repo.lower().endswith("gguf") or "gguf" in backbone_repo.lower(): - try: - from llama_cpp import Llama - except ImportError as e: - raise ImportError( - "Failed to import `llama_cpp`. Please install llama-cpp-python version >= 0.3.16." - ) from e - self.backbone = Llama.from_pretrained( - repo_id=backbone_repo, - filename="*.gguf", - verbose=False, - n_gpu_layers=-1 if backbone_device in ("gpu", "cuda") else 0, - n_ctx=self.max_context, - mlock=True, - flash_attn=True if backbone_device in ("gpu", "cuda") else False, - token=hf_token, - ) - self._is_quantized_model = True - else: - from transformers import AutoTokenizer, AutoModelForCausalLM - self.tokenizer = AutoTokenizer.from_pretrained(backbone_repo, token=hf_token) - - # Configure tokenizer for batching - self.tokenizer.padding_side = "left" - if self.tokenizer.pad_token is None: - self.tokenizer.pad_token = self.tokenizer.eos_token - - self.backbone = AutoModelForCausalLM.from_pretrained(backbone_repo, token=hf_token).to( - torch.device(backbone_device) - ) - - # Optional torch.compile for non-Windows/non-Mac platforms if desired - if os.getenv("VIENEU_COMPILE") == "1" and platform.system() == "Linux": - try: - logger.info("🚀 Compiling backbone model with torch.compile...") - self.backbone = torch.compile(self.backbone, mode="reduce-overhead") - except Exception as e: - logger.warning(f"Failed to compile backbone: {e}") - - def _load_codec(self, codec_repo: str, codec_device: str) -> None: - if codec_device == "mps" and not torch.backends.mps.is_available(): - logger.warning("Warning: MPS not available for codec, falling back to CPU") - codec_device = "cpu" - - logger.info(f"Loading codec from: {codec_repo} on {codec_device} ...") - - if codec_repo == "neuphonic/neucodec": - from neucodec import NeuCodec - self.codec = NeuCodec.from_pretrained(codec_repo) - elif codec_repo == "neuphonic/distill-neucodec": - from neucodec import DistillNeuCodec - self.codec = DistillNeuCodec.from_pretrained(codec_repo) - elif codec_repo == "neuphonic/neucodec-onnx-decoder-int8": - if codec_device != "cpu": - raise ValueError("Onnx decoder only currently runs on CPU.") - try: - from neucodec import NeuCodecOnnxDecoder - except ImportError as e: - raise ImportError( - "Failed to import the onnx decoder. Ensure onnxruntime and neucodec >= 0.0.4 are installed." - ) from e - self.codec = NeuCodecOnnxDecoder.from_pretrained(codec_repo) - self._is_onnx_codec = True - elif codec_repo == "pnnbao-ump/VieNeu-Codec": - import onnxruntime as ort - from huggingface_hub import hf_hub_download - - logger.info("🚀 Loading VieNeu-Codec (ONNX) ...") - decoder_path = hf_hub_download(repo_id=codec_repo, filename="vieneu_decoder.onnx") - - providers = ["CPUExecutionProvider"] - if codec_device in ("gpu", "cuda"): - providers = ["CUDAExecutionProvider"] + providers - - self.decoder_sess = ort.InferenceSession(decoder_path, providers=providers) - self._is_onnx_codec = True - self._is_vieneu_codec = True - else: - raise ValueError(f"Unsupported codec repository: {codec_repo}") - - if not self._is_onnx_codec: - self.codec.eval().to(codec_device) - - def load_lora_adapter(self, lora_repo_id: str, hf_token: Optional[str] = None) -> bool: - if self._is_quantized_model: - raise NotImplementedError("LoRA not supported for GGUF quantized models. Use PyTorch backbone.") - - try: - from peft import PeftModel - except ImportError as e: - raise ImportError("PEFT library required for LoRA. Install with: pip install peft") - - logger.info(f"🎯 Loading LoRA adapter from: {lora_repo_id}") - - if not hasattr(self, '_lora_loaded') or not self._lora_loaded: - self._current_lora_repo = None - self._lora_loaded = False - - if self._lora_loaded: - self.unload_lora_adapter() - - try: - self.backbone = PeftModel.from_pretrained(self.backbone, lora_repo_id, token=hf_token) - self._lora_loaded = True - self._current_lora_repo = lora_repo_id - self._load_voices(lora_repo_id, hf_token, clear_existing=True) - logger.info(f" ✅ LoRA adapter loaded: {lora_repo_id}") - return True - except Exception as e: - raise RuntimeError(f"Failed to load LoRA adapter: {str(e)}") from e - - def unload_lora_adapter(self) -> bool: - if not getattr(self, '_lora_loaded', False): - return False - - logger.info(f" 🔄 Unloading LoRA adapter: {self._current_lora_repo}") - try: - self.backbone = self.backbone.unload() - self._lora_loaded = False - self._current_lora_repo = None - gc.collect() - if torch.cuda.is_available(): - torch.cuda.empty_cache() - logger.info(" ✅ LoRA adapter unloaded, original weights restored") - return True - except Exception as e: - logger.error(f" ⚠️ Error during unload: {e}") - return False - - def infer(self, text: str, ref_audio: Optional[Union[str, Path]] = None, ref_codes: Optional[Union[np.ndarray, torch.Tensor]] = None, ref_text: Optional[str] = None, max_chars: int = 256, silence_p: float = 0.15, crossfade_p: float = 0.0, voice: Optional[Dict[str, Any]] = None, temperature: float = 1.0, top_k: int = 50, skip_normalize: bool = False, **kwargs) -> np.ndarray: - - ref_codes, ref_text = self._resolve_ref_voice(voice, ref_audio, ref_codes, ref_text) - - if not skip_normalize: - text = self.normalizer.normalize(text) - - # Sync chunking logic for v2-Turbo - skip_phonemize = kwargs.get("skip_phonemize", False) - if self._is_turbo: - # 1. Phonemize everything first for Turbo - if skip_phonemize: - full_phonemes = text - else: - full_phonemes = phonemize_with_dict(text, skip_normalize=True) - - # 2. Split phonemes using the v2 logic - from vieneu_utils.core_utils import split_into_chunks_v2, get_silence_duration_v2, join_audio_chunks - chunks = split_into_chunks_v2(full_phonemes, max_chunk_size=max_chars) - - if not chunks: - return np.array([], dtype=np.float32) - - all_wavs = [] - for i, chunk_phones in enumerate(chunks): - # Synthesis - prompt_ids = self._format_turbo_prompt_standard(chunk_phones, voice=voice) - output_str = self._infer_torch(prompt_ids, temperature, top_k) - wav = self._decode(output_str, ref_codes=ref_codes, voice=voice) - all_wavs.append(wav) - - # Add intelligent silence duration for Turbo - if i < len(chunks) - 1: - silence_dur = get_silence_duration_v2(chunk_phones) - if silence_dur > 0: - silence = np.zeros(int(self.sample_rate * silence_dur), dtype=np.float32) - all_wavs.append(silence) - - final_wav = np.concatenate(all_wavs) if len(all_wavs) > 1 else all_wavs[0] - return self._apply_watermark(final_wav) - - # Legacy logic for non-turbo models (v1/v0.3) - chunks = split_text_into_chunks(text, max_chars=max_chars) - if not chunks: - return np.array([], dtype=np.float32) - - if len(chunks) == 1: - ref_phonemes = self.get_ref_phonemes(ref_text) - if skip_phonemize: - phonemes = chunks[0] - else: - phonemes = phonemize_with_dict(chunks[0], skip_normalize=True) - - if self._is_quantized_model: - output_str = self._infer_ggml(ref_codes, ref_phonemes, phonemes, temperature, top_k) - else: - prompt_ids = self._apply_chat_template(ref_codes, ref_phonemes, phonemes) - output_str = self._infer_torch(prompt_ids, temperature, top_k) - wav = self._decode(output_str, ref_codes=ref_codes, voice=voice) - return self._apply_watermark(wav) - - all_wavs = self.infer_batch( - chunks, - ref_codes=ref_codes, - ref_text=ref_text, - temperature=temperature, - top_k=top_k, - skip_normalize=True, - apply_watermark=False - ) - final_wav = join_audio_chunks(all_wavs, self.sample_rate, silence_p, crossfade_p) - return self._apply_watermark(final_wav) - - def infer_batch(self, texts: List[str], ref_audio: Optional[Union[str, Path]] = None, ref_codes: Optional[Union[np.ndarray, torch.Tensor]] = None, ref_text: Optional[str] = None, voice: Optional[Dict[str, Any]] = None, temperature: float = 1.0, top_k: int = 50, skip_normalize: bool = False, apply_watermark: bool = True, **kwargs) -> List[np.ndarray]: - ref_codes, ref_text = self._resolve_ref_voice(voice, ref_audio, ref_codes, ref_text) - - if not skip_normalize: - texts = [self.normalizer.normalize(t) for t in texts] - - ref_phonemes = self.get_ref_phonemes(ref_text) - - skip_phonemize = kwargs.get("skip_phonemize", False) - if skip_phonemize: - chunk_phonemes = texts - else: - chunk_phonemes = phonemize_batch(texts, skip_normalize=True) - - all_wavs = [] - # If model is GGUF, we still process sequentially for now as llama-cpp-python batching for TTS is complex - if self._is_quantized_model: - for phonemes in chunk_phonemes: - if self._is_turbo: - prompt_ids = self._format_turbo_prompt_standard(phonemes, voice=voice) - else: - prompt_ids = self._apply_chat_template(ref_codes, ref_phonemes, phonemes) - output_str = self._infer_torch(prompt_ids, temperature, top_k) - wav = self._decode(output_str, ref_codes=ref_codes, voice=voice) - if apply_watermark: - wav = self._apply_watermark(wav) - all_wavs.append(wav) - # If model is Torch, we can leverage true batch generation - else: - batch_prompt_ids = [] - for phonemes in chunk_phonemes: - if self._is_turbo: - prompt_ids = self._format_turbo_prompt_standard(phonemes, voice=voice) - else: - prompt_ids = self._apply_chat_template(ref_codes, ref_phonemes, phonemes) - batch_prompt_ids.append(torch.tensor(prompt_ids)) - - inputs = self.tokenizer.pad( - {"input_ids": batch_prompt_ids}, - padding=True, - return_tensors="pt" - ) - # Move all tensors to device - inputs = {k: v.to(self.backbone.device) for k, v in inputs.items()} - - speech_end_id = self.tokenizer.convert_tokens_to_ids("<|SPEECH_GENERATION_END|>") - with torch.no_grad(): - output_tokens = self.backbone.generate( - **inputs, - max_length=self.max_context, - eos_token_id=speech_end_id, - do_sample=True, - temperature=temperature, - top_k=top_k, - use_cache=True, - min_new_tokens=50, - ) - - input_length = inputs["input_ids"].shape[-1] - for i in range(len(texts)): - generated_ids = output_tokens[i, input_length:] - output_str = self.tokenizer.decode(generated_ids, add_special_tokens=False) - wav = self._decode(output_str, ref_codes=ref_codes, voice=voice) - if apply_watermark: - wav = self._apply_watermark(wav) - all_wavs.append(wav) - - return all_wavs - - def infer_stream(self, text: str, ref_audio: Optional[Union[str, Path]] = None, ref_codes: Optional[Union[np.ndarray, torch.Tensor]] = None, ref_text: Optional[str] = None, max_chars: int = 256, voice: Optional[Dict[str, Any]] = None, temperature: float = 1.0, top_k: int = 50, skip_normalize: bool = False) -> Generator[np.ndarray, None, None]: - - ref_codes, ref_text = self._resolve_ref_voice(voice, ref_audio, ref_codes, ref_text) - - if not skip_normalize: - text = self.normalizer.normalize(text) - - chunks = split_text_into_chunks(text, max_chars=max_chars) - if not chunks: - return - - # Pre-phonemize all inputs for performance - ref_phonemes = self.get_ref_phonemes(ref_text) - - skip_phonemize = kwargs.get("skip_phonemize", False) - if skip_phonemize: - chunk_phonemes = chunks - else: - chunk_phonemes = phonemize_batch(chunks, skip_normalize=True) - - for phonemes in chunk_phonemes: - if self._is_turbo: - prompt_ids = self._format_turbo_prompt_standard(phonemes, ref_codes=ref_codes) - output_str = self._infer_torch(prompt_ids, temperature, top_k) - wav = self._decode(output_str, voice=voice) - yield self._apply_watermark(wav) - elif self._is_quantized_model: - yield from self._infer_stream_ggml(ref_codes, ref_phonemes, phonemes, temperature, top_k) - else: - prompt_ids = self._apply_chat_template(ref_codes, ref_phonemes, phonemes) - output_str = self._infer_torch(prompt_ids, temperature, top_k) - wav = self._decode(output_str, voice=voice) - yield self._apply_watermark(wav) - - def _apply_chat_template(self, ref_codes: Union[List[int], torch.Tensor, np.ndarray], ref_phonemes: str, chunk_phonemes: str) -> List[int]: - if isinstance(ref_codes, (torch.Tensor, np.ndarray)): - ref_codes_list = ref_codes.flatten().tolist() - else: - ref_codes_list = ref_codes - - full_phonemes = f"{ref_phonemes} {chunk_phonemes}" - - speech_replace = self.tokenizer.convert_tokens_to_ids("<|SPEECH_REPLACE|>") - speech_gen_start = self.tokenizer.convert_tokens_to_ids("<|SPEECH_GENERATION_START|>") - text_replace = self.tokenizer.convert_tokens_to_ids("<|TEXT_REPLACE|>") - text_prompt_start = self.tokenizer.convert_tokens_to_ids("<|TEXT_PROMPT_START|>") - text_prompt_end = self.tokenizer.convert_tokens_to_ids("<|TEXT_PROMPT_END|>") - - input_ids = self.tokenizer.encode(full_phonemes, add_special_tokens=False) - chat = "user: Convert the text to speech:<|TEXT_REPLACE|>\nassistant:<|SPEECH_REPLACE|>" - ids = self.tokenizer.encode(chat) - - text_replace_idx = ids.index(text_replace) - ids = ids[:text_replace_idx] + [text_prompt_start] + input_ids + [text_prompt_end] + ids[text_replace_idx + 1:] - - speech_replace_idx = ids.index(speech_replace) - codes_str = "".join([f"<|speech_{i}|>" for i in ref_codes_list]) - codes = self.tokenizer.encode(codes_str, add_special_tokens=False) - ids = ids[:speech_replace_idx] + [speech_gen_start] + list(codes) - return ids - - def _get_voice_params(self, ref_codes: Any) -> tuple: - """Trích xuất v_id và embedding (Robustly sync-fixed for ONNX).""" - v_id = -1 - emb = np.zeros((1, 128), dtype=np.float32) - - if ref_codes is None: - return v_id, emb - - # 1. Handle Dictionary (Presets) - if isinstance(ref_codes, dict): - v_id = ref_codes.get("voice_id", -1) - codes = ref_codes.get("codes") - if codes is not None: - if hasattr(codes, "detach"): codes = codes.detach().cpu().numpy() - codes = np.array(codes) - if codes.size == 128: - emb = codes.reshape(1, 128).astype(np.float32) - - # 2. Handle Numeric ID - elif isinstance(ref_codes, int): - v_id = ref_codes - - # 3. Handle Raw Embedding (Cloning/Arrays) - elif isinstance(ref_codes, (np.ndarray, torch.Tensor)): - if hasattr(ref_codes, "detach"): codes = ref_codes.detach().cpu().numpy() - else: codes = np.array(ref_codes) - if codes.size == 128: - emb = codes.reshape(1, 128).astype(np.float32) - v_id = -1 - - return v_id, emb - - def _decode(self, codes_str: str, ref_codes: Optional[Any] = None, voice: Optional[Dict[str, Any]] = None) -> np.ndarray: - """Custom decode for VieNeu-Codec compatibility.""" - if not getattr(self, "_is_vieneu_codec", False): - return super()._decode(codes_str) - - # Handling VieNeu-Codec (v2-Turbo) ONNX decoding - from .utils import extract_speech_ids - speech_ids = extract_speech_ids(codes_str) - if not speech_ids: - return np.array([], dtype=np.float32) - - # Logic cực kỳ chính xác: - # Nếu có 'voice' (nhân vật chọn trong list), ta dùng nó để lấy ID và Embedding. - # Nếu không có 'voice' (chế độ Cloning), ta dùng 'ref_codes' (đã chứa Embedding từ file audio). - v_id, embedding = self._get_voice_params(voice) if (voice is not None) else self._get_voice_params(ref_codes) - - # Chốt ID: Nếu vẫn là -1 (chưa xác định), mặc định là 16 (Xuân Vĩnh). - # Nếu đã có v_id chuẩn (ví dụ 0 cho Đoan Trang), nó sẽ được sử dụng ngay. - v_idx = v_id if v_id >= 0 else 1 - - # Debug nhỏ (Log cho dev) - # logger.info(f"🎤 Decoding with Voice ID: {v_idx}") - - inputs = { - "content_ids": np.array(speech_ids, dtype=np.int64)[None, :], - "voice_id": np.array([v_idx], dtype=np.int64), - "evoice_embedding": embedding.astype(np.float32) - } - - audio = self.decoder_sess.run(None, inputs)[0] - # Flatten and correct shapes - if audio.ndim == 3: - return audio[0, 0, :] - elif audio.ndim == 2: - return audio[0, :] - return audio.flatten() - - def _format_turbo_prompt_standard(self, chunk_phonemes: str, voice: Optional[Dict[str, Any]] = None) -> List[int]: - """Apply v2-Turbo template for Standard backbone with dynamic speaker.""" - - # Turbo v2 format with dynamic speaker - prompt = ( - f"<|speaker_16|>" - f"<|TEXT_PROMPT_START|>{chunk_phonemes}<|TEXT_PROMPT_END|>" - f"<|SPEECH_GENERATION_START|>" - ) - return self.tokenizer.encode(prompt, add_special_tokens=False) - - def _infer_torch(self, prompt_ids: List[int], temperature: float = 1.0, top_k: int = 50) -> str: - prompt_tensor = torch.tensor(prompt_ids).unsqueeze(0).to(self.backbone.device) - speech_end_id = self.tokenizer.convert_tokens_to_ids("<|SPEECH_GENERATION_END|>") - with torch.no_grad(): - output_tokens = self.backbone.generate( - prompt_tensor, - max_length=self.max_context, - eos_token_id=speech_end_id, - do_sample=True, - temperature=temperature, - top_k=top_k, - use_cache=True, - min_new_tokens=50, - ) - input_length = prompt_tensor.shape[-1] - output_str = self.tokenizer.decode(output_tokens[0, input_length:].cpu().numpy().tolist(), add_special_tokens=False) - return output_str - - def _infer_ggml(self, ref_codes: Union[List[int], torch.Tensor, np.ndarray], ref_phonemes: str, chunk_phonemes: str, temperature: float = 1.0, top_k: int = 50) -> str: - if isinstance(ref_codes, (torch.Tensor, np.ndarray)): - ref_codes_list = ref_codes.flatten().tolist() - else: - ref_codes_list = ref_codes - - codes_str = "".join([f"<|speech_{idx}|>" for idx in ref_codes_list]) - prompt = ( - f"user: Convert the text to speech:<|TEXT_PROMPT_START|>{ref_phonemes} {chunk_phonemes}" - f"<|TEXT_PROMPT_END|>\nassistant:<|SPEECH_GENERATION_START|>{codes_str}" - ) - output = self.backbone(prompt, max_tokens=self.max_context, temperature=temperature, top_k=top_k, stop=["<|SPEECH_GENERATION_END|>"]) - return output["choices"][0]["text"] - - def _infer_stream_ggml(self, ref_codes: Union[np.ndarray, torch.Tensor, List[int]], ref_phonemes: str, chunk_phonemes: str, temperature: float = 1.0, top_k: int = 50) -> Generator[np.ndarray, None, None]: - - if isinstance(ref_codes, (torch.Tensor, np.ndarray)): - ref_codes_list = ref_codes.flatten().tolist() - else: - ref_codes_list = ref_codes - - codes_str = "".join([f"<|speech_{idx}|>" for idx in ref_codes_list]) - prompt = ( - f"user: Convert the text to speech:<|TEXT_PROMPT_START|>{ref_phonemes} {chunk_phonemes}" - f"<|TEXT_PROMPT_END|>\nassistant:<|SPEECH_GENERATION_START|>{codes_str}" - ) - - audio_cache: List[np.ndarray] = [] - token_cache: List[str] = [f"<|speech_{idx}|>" for idx in ref_codes_list] - n_decoded_samples: int = 0 - n_decoded_tokens: int = len(ref_codes_list) - - for item in self.backbone(prompt, max_tokens=self.max_context, temperature=temperature, top_k=top_k, stop=["<|SPEECH_GENERATION_END|>"], stream=True): - output_str = item["choices"][0]["text"] - token_cache.append(output_str) - - if len(token_cache[n_decoded_tokens:]) >= self.streaming_frames_per_chunk + self.streaming_lookforward: - tokens_start = max(n_decoded_tokens - self.streaming_lookback - self.streaming_overlap_frames, 0) - tokens_end = n_decoded_tokens + self.streaming_frames_per_chunk + self.streaming_lookforward + self.streaming_overlap_frames - sample_start = (n_decoded_tokens - tokens_start) * self.hop_length - sample_end = sample_start + (self.streaming_frames_per_chunk + 2 * self.streaming_overlap_frames) * self.hop_length - curr_codes = token_cache[tokens_start:tokens_end] - recon = self._decode("".join(curr_codes)) - recon = self._apply_watermark(recon) - recon = recon[sample_start:sample_end] - audio_cache.append(recon) - - processed_recon = _linear_overlap_add(audio_cache, stride=self.streaming_stride_samples) - new_samples_end = len(audio_cache) * self.streaming_stride_samples - processed_recon = processed_recon[n_decoded_samples:new_samples_end] - n_decoded_samples = new_samples_end - n_decoded_tokens += self.streaming_frames_per_chunk - yield processed_recon - - remaining_tokens = len(token_cache) - n_decoded_tokens - if remaining_tokens > 0: - tokens_start = max(len(token_cache) - (self.streaming_lookback + self.streaming_overlap_frames + remaining_tokens), 0) - sample_start = (len(token_cache) - tokens_start - remaining_tokens - self.streaming_overlap_frames) * self.hop_length - curr_codes = token_cache[tokens_start:] - recon = self._decode("".join(curr_codes)) - recon = self._apply_watermark(recon) - recon = recon[sample_start:] - audio_cache.append(recon) - processed_recon = _linear_overlap_add(audio_cache, stride=self.streaming_stride_samples) - processed_recon = processed_recon[n_decoded_samples:] - yield processed_recon \ No newline at end of file diff --git a/src/vieneu/turbo.py b/src/vieneu/turbo.py deleted file mode 100644 index ba44d6bc7b88677d2050bb6dea2ba58823cec15b..0000000000000000000000000000000000000000 --- a/src/vieneu/turbo.py +++ /dev/null @@ -1,538 +0,0 @@ -import os -import numpy as np -import logging -from typing import Optional, List, Any, Generator -from pathlib import Path -from .base import BaseVieneuTTS -from vieneu_utils.phonemize_text import phonemize_text -from vieneu_utils.core_utils import split_into_chunks_v2, get_silence_duration_v2 - -logger = logging.getLogger("Vieneu.Turbo") - -class TurboGPUVieNeuTTS(BaseVieneuTTS): - def __init__( - self, - backbone_repo: str = "pnnbao-ump/VieNeu-TTS-v2-Turbo", - decoder_repo: str = "pnnbao-ump/VieNeu-Codec", - decoder_filename: str = "vieneu_decoder.onnx", - encoder_repo: str = "pnnbao-ump/VieNeu-Codec", - encoder_filename: str = "vieneu_encoder.onnx", - device: str = "cuda", - backend: str = "standard", # 'standard' (transformers) or 'lmdeploy' - hf_token: Optional[str] = None, - **kwargs - ): - super().__init__() - # Normalize device: PyTorch expects 'cuda', Gradio might pass 'gpu' - self.device = "cuda" if device.lower() in ["cuda", "gpu"] else "cpu" - self.backend = backend.lower() - self.backbone = None - self.tokenizer = None - self.decoder_sess = None - self.encoder_sess = None - self._is_onnx_codec = True - - # Load components - self._load_backbone(backbone_repo, device, hf_token, **kwargs) - self._load_decoder(decoder_repo, decoder_filename, device, hf_token) - self._load_encoder(encoder_repo, encoder_filename, device, hf_token) - - # Load voices - self._load_voices(backbone_repo, hf_token) - - def _load_backbone(self, repo, device, hf_token=None, **kwargs): - if self.backend == "lmdeploy": - try: - from lmdeploy import pipeline, TurbomindEngineConfig, GenerationConfig - logger.info(f"Loading Turbo GPU (LMDeploy) from: {repo}") - - engine_config = TurbomindEngineConfig( - cache_max_entry_count=kwargs.get("memory_util", 0.3), - tp=kwargs.get("tp", 1), - enable_prefix_caching=kwargs.get("enable_prefix_caching", True), - dtype='bfloat16' if self.device == "cuda" else 'float16' - ) - self.backbone = pipeline(repo, backend_config=engine_config) - self.gen_config = GenerationConfig( - top_p=0.95, top_k=50, temperature=0.4, max_new_tokens=2048, - do_sample=True, stop_words=["<|SPEECH_GENERATION_END|>"] - ) - except ImportError: - logger.warning("LMDeploy not found. Falling back to Standard (Transformers).") - self.backend = "standard" - - if self.backend == "standard": - from transformers import AutoTokenizer, AutoModelForCausalLM - import torch - - logger.info(f"Loading Turbo GPU (Standard) from: {repo} on {self.device}") - self.tokenizer = AutoTokenizer.from_pretrained(repo, token=hf_token) - self.backbone = AutoModelForCausalLM.from_pretrained( - repo, - torch_dtype=torch.bfloat16 if self.device == "cuda" else torch.float32, - token=hf_token - ).to(self.device) - self.backbone.eval() - - def _load_decoder(self, decoder_repo, decoder_filename, device, hf_token=None): - import onnxruntime as ort - if os.path.exists(decoder_repo) and not os.path.isdir(decoder_repo): - decoder_path = decoder_repo - else: - from huggingface_hub import hf_hub_download - decoder_path = hf_hub_download( - repo_id=decoder_repo, filename=decoder_filename, token=hf_token - ) - - # Using CPU for codec is recommended by the user for stability in certain environments - codec_providers = ["CPUExecutionProvider"] - self.decoder_sess = ort.InferenceSession(decoder_path, providers=codec_providers) - - def _load_encoder(self, encoder_repo, encoder_filename, device, hf_token=None): - import onnxruntime as ort - if os.path.exists(encoder_repo) and not os.path.isdir(encoder_repo): - encoder_path = encoder_repo - else: - from huggingface_hub import hf_hub_download - try: - encoder_path = hf_hub_download( - repo_id=encoder_repo, filename=encoder_filename, token=hf_token - ) - except Exception: - logger.warning("Speaker encoder not found for Turbo GPU.") - return - - # Codec components (Encoder/Decoder) are small ONNX models. - # Running them on CPU is often more stable in shared GPU environments (HF ZeroGPU). - codec_providers = ["CPUExecutionProvider"] - self.encoder_sess = ort.InferenceSession(encoder_path, providers=codec_providers) - - def _get_voice_params(self, ref_codes: Any) -> np.ndarray: - if isinstance(ref_codes, dict): - ref_codes = ref_codes.get("codes") - if isinstance(ref_codes, (np.ndarray, list)): - emb = np.array(ref_codes, dtype=np.float32) - if emb.ndim == 1: - emb = emb[None, :] - return emb - return np.zeros((1, 128), dtype=np.float32) - - def encode_reference(self, ref_audio: Any) -> np.ndarray: - """Standalone ONNX implementation for speaker encoding in Turbo GPU mode.""" - if self.encoder_sess is None: - raise RuntimeError("Speaker encoder model not loaded for Turbo GPU.") - - import librosa - if isinstance(ref_audio, (str, Path)): - wav, _ = librosa.load(ref_audio, sr=24000) - else: - wav = ref_audio - - if wav.ndim == 1: - wav = wav[None, :] - - inputs = {"waveform": wav.astype(np.float32)} - embedding = self.encoder_sess.run(None, inputs)[0] - return embedding - - def _format_turbo_prompt(self, phonemes: str) -> str: - return ( - f"<|speaker_16|>" - f"<|TEXT_PROMPT_START|>{phonemes}<|TEXT_PROMPT_END|>" - f"<|SPEECH_GENERATION_START|>" - ) - - def infer(self, text: str, ref_codes: Optional[Any] = None, temperature: float = 0.4, top_k: int = 50, max_chars: int = 256, skip_normalize: bool = False, skip_phonemize: bool = False, **kwargs) -> np.ndarray: - from vieneu_utils.phonemize_text import phonemize_text - from vieneu_utils.core_utils import split_into_chunks_v2, get_silence_duration_v2 - - phonemes = phonemize_text(text) if not skip_phonemize else text - chunks = split_into_chunks_v2(phonemes, max_chunk_size=max_chars) - - if ref_codes is None: - ref_codes = self.get_preset_voice() - voice_embedding = self._get_voice_params(ref_codes) - - all_wavs = [] - for i, chunk in enumerate(chunks): - prompt = self._format_turbo_prompt(chunk.text) - - if self.backend == "lmdeploy": - self.gen_config.temperature = temperature - self.gen_config.top_k = top_k - responses = self.backbone([prompt], gen_config=self.gen_config, do_preprocess=False) - generated_text = responses[0].text - else: - import torch - inputs = self.tokenizer(prompt, return_tensors="pt").to(self.device) - with torch.no_grad(): - output_tokens = self.backbone.generate( - **inputs, - max_new_tokens=2048, - temperature=temperature, - top_k=top_k, - do_sample=True, - pad_token_id=self.tokenizer.eos_token_id, - # stop_strings=... is transformers >= 4.41.0 - # For compatibility, we trim the output if eos is hit - ) - generated_text = self.tokenizer.decode(output_tokens[0][inputs.input_ids.shape[-1]:], skip_special_tokens=True) - - wav = self._decode(generated_text, voice_embedding) - all_wavs.append(wav) - - if i < len(chunks) - 1: - silence_dur = get_silence_duration_v2(chunk) - if silence_dur > 0: - all_wavs.append(np.zeros(int(self.sample_rate * silence_dur), dtype=np.float32)) - - final_wav = np.concatenate(all_wavs) if len(all_wavs) > 1 else all_wavs[0] - return self._apply_watermark(final_wav) - - def infer_batch( - self, - texts: List[str], - ref_codes: Optional[Any] = None, - temperature: float = 0.4, - top_k: int = 50, - max_batch_size: int = 4, - skip_normalize: bool = False, - skip_phonemize: bool = False, - apply_watermark: bool = True, - **kwargs - ) -> List[np.ndarray]: - """Batch inference for Turbo GPU.""" - from vieneu_utils.phonemize_text import phonemize_batch - - if ref_codes is None: - ref_codes = self.get_preset_voice() - voice_embedding = self._get_voice_params(ref_codes) - - chunk_phonemes = phonemize_batch(texts, skip_normalize=True) if not skip_phonemize else texts - - all_wavs = [] - for i in range(0, len(texts), max_batch_size): - batch_texts = texts[i : i + max_batch_size] - batch_ph = chunk_phonemes[i : i + max_batch_size] - - if self.backend == "lmdeploy": - prompts = [self._format_turbo_prompt(ph) for ph in batch_ph] - self.gen_config.temperature = temperature - self.gen_config.top_k = top_k - responses = self.backbone(prompts, gen_config=self.gen_config, do_preprocess=False) - batch_codes = [r.text for r in responses] - batch_wavs = [self._decode(c, voice_embedding) for c in batch_codes] - else: - # Standard sequential fallback for transformers - batch_wavs = [ - self.infer( - t, - ref_codes=ref_codes, - temperature=temperature, - top_k=top_k, - skip_normalize=True, - skip_phonemize=True - ) for t in batch_texts - ] - - if apply_watermark: - batch_wavs = [self._apply_watermark(w) for w in batch_wavs] - all_wavs.extend(batch_wavs) - - return all_wavs - - def _decode(self, codes_str: str, voice_embedding: np.ndarray) -> np.ndarray: - from .utils import extract_speech_ids - speech_ids = extract_speech_ids(codes_str) - if not speech_ids: - return np.array([], dtype=np.float32) - - tokens = np.array(speech_ids, dtype=np.int64)[None, :] - inputs = { - "content_ids": tokens, - "voice_embedding": voice_embedding - } - audio = self.decoder_sess.run(None, inputs)[0] - return audio.flatten() - - def close(self): - self.backbone = None - self.decoder_sess = None - self.encoder_sess = None - - - -class TurboVieNeuTTS(BaseVieneuTTS): - def __init__( - self, - backbone_repo: str = "pnnbao-ump/VieNeu-TTS-v2-Turbo-GGUF", - backbone_filename: str = "vieneu-tts-v2-turbo.gguf", - decoder_repo: str = "pnnbao-ump/VieNeu-Codec", - decoder_filename: str = "vieneu_decoder.onnx", - encoder_repo: str = "pnnbao-ump/VieNeu-Codec", - encoder_filename: str = "vieneu_encoder.onnx", - device: str = "cpu", - hf_token: Optional[str] = None, - **kwargs - ): - super().__init__() - self.backbone = None - self.decoder_sess = None - self.encoder_sess = None - self._is_onnx_codec = True - self.max_context = 4096 - - # Load components - self._load_backbone(backbone_repo, backbone_filename, device, hf_token) - self._load_decoder(decoder_repo, decoder_filename, device, hf_token) - self._load_encoder(encoder_repo, encoder_filename, device, hf_token) - - # Load voices from the repository/directory (uses voices.json) - self._load_voices(backbone_repo, hf_token) - - def _load_backbone(self, backbone_repo, backbone_filename, device, hf_token=None): - try: - from llama_cpp import Llama - except ImportError: - raise ImportError("llama-cpp-python is required for Turbo mode.") - - if os.path.exists(backbone_repo): - model_path = backbone_repo - else: - from huggingface_hub import hf_hub_download - try: - model_path = hf_hub_download( - repo_id=backbone_repo, filename=backbone_filename, token=hf_token - ) - except Exception: - if os.path.exists(backbone_filename): - model_path = backbone_filename - else: - raise FileNotFoundError(f"Neither repo '{backbone_repo}' nor '{backbone_filename}' found.") - - self.backbone = Llama( - model_path=model_path, - n_ctx=self.max_context, - n_gpu_layers=-1 if device in ("gpu", "cuda") else 0, - mlock=True, - flash_attn=device in ("gpu", "cuda"), - verbose=False, - ) - - def _load_decoder(self, decoder_repo, decoder_filename, device, hf_token=None): - try: - import onnxruntime as ort - except ImportError: - raise ImportError("onnxruntime is required for Turbo mode.") - - if os.path.exists(decoder_repo): - decoder_path = decoder_repo - else: - from huggingface_hub import hf_hub_download - try: - decoder_path = hf_hub_download( - repo_id=decoder_repo, filename=decoder_filename, token=hf_token - ) - except Exception: - if os.path.exists(decoder_filename): - decoder_path = decoder_filename - else: - raise FileNotFoundError(f"Neither repo '{decoder_repo}' nor '{decoder_filename}' found.") - - providers = ["CUDAExecutionProvider", "CPUExecutionProvider"] if device in ("gpu", "cuda") else ["CPUExecutionProvider"] - self.decoder_sess = ort.InferenceSession(decoder_path, providers=providers) - - def _load_encoder(self, encoder_repo, encoder_filename, device, hf_token=None): - try: - import onnxruntime as ort - except ImportError: - return - - if os.path.exists(encoder_repo): - encoder_path = encoder_repo - else: - from huggingface_hub import hf_hub_download - try: - encoder_path = hf_hub_download( - repo_id=encoder_repo, filename=encoder_filename, token=hf_token - ) - except Exception: - if os.path.exists(encoder_filename): - encoder_path = encoder_filename - else: - logger.warning("Speaker encoder not found, voice cloning might be limited in Turbo mode.") - return - - providers = ["CUDAExecutionProvider", "CPUExecutionProvider"] if device in ("gpu", "cuda") else ["CPUExecutionProvider"] - self.encoder_sess = ort.InferenceSession(encoder_path, providers=providers) - - def encode_reference(self, ref_audio: Any) -> np.ndarray: - """Standalone ONNX implementation for speaker encoding in Turbo mode.""" - if self.encoder_sess is None: - raise RuntimeError("Speaker encoder model not loaded. Please ensure vieneu_encoder.onnx is available.") - - import librosa - if isinstance(ref_audio, (str, Path)): - wav, _ = librosa.load(ref_audio, sr=24000) - else: - wav = ref_audio - - if wav.ndim == 1: - wav = wav[None, :] - - inputs = {"waveform": wav.astype(np.float32)} - embedding = self.encoder_sess.run(None, inputs)[0] - return embedding - - def _get_voice_params(self, ref_codes: Any) -> np.ndarray: - """Extract the 128-dim voice embedding for the new decoder logic.""" - # Handle dict input (from get_preset_voice returned dict) - if isinstance(ref_codes, dict): - ref_codes = ref_codes.get("codes") - - # Ensure it is a float32 numpy array with shape (1, D) - if isinstance(ref_codes, (np.ndarray, list)): - emb = np.array(ref_codes, dtype=np.float32) - if emb.ndim == 1: - emb = emb[None, :] - if emb.shape[-1] in [128]: - return emb - - # Fallback to zeros (128-dim) - return np.zeros((1, 128), dtype=np.float32) - - def _decode(self, codes_str: str, voice_embedding: Optional[np.ndarray] = None) -> np.ndarray: - from .utils import extract_speech_ids - speech_ids = extract_speech_ids(codes_str) - if not speech_ids: - return np.array([], dtype=np.float32) - - tokens = np.array(speech_ids, dtype=np.int64)[None, :] - - if voice_embedding is None: - voice_embedding = np.zeros((1, 128), dtype=np.float32) - - inputs = { - "content_ids": tokens, - "voice_embedding": voice_embedding - } - audio = self.decoder_sess.run(None, inputs)[0] - - if audio.ndim == 3: - return audio[0, 0, :] - elif audio.ndim == 2: - return audio[0, :] - return audio.flatten() - - def infer( - self, - text: str, - ref_codes: Optional[Any] = None, - temperature: float = 0.4, - top_k: int = 50, - max_chars: int = 256, - skip_normalize: bool = False, - skip_phonemize: bool = False, - **kwargs - ) -> np.ndarray: - phonemes = phonemize_text(text) if not skip_phonemize else text - - chunks = split_into_chunks_v2(phonemes, max_chunk_size=max_chars) - if not chunks: - return np.array([], dtype=np.float32) - - # Use default voice if none provided - if ref_codes is None: - ref_codes = self.get_preset_voice() - - voice_embedding = self._get_voice_params(ref_codes) - - all_wavs = [] - for i, chunk in enumerate(chunks): - prompt = self._format_turbo_prompt(chunk.text) - - self.backbone.reset() - result = self.backbone( - prompt, - max_tokens=2048, - temperature=temperature, - top_k=top_k, - top_p=0.95, - min_p=0.05, - stop=["<|SPEECH_GENERATION_END|>"], - repeat_penalty=1.15, - echo=False, - ) - wav = self._decode(result["choices"][0]["text"], voice_embedding) - all_wavs.append(wav) - - if i < len(chunks) - 1: - silence_dur = get_silence_duration_v2(chunk) - if silence_dur > 0: - all_wavs.append(np.zeros(int(self.sample_rate * silence_dur), dtype=np.float32)) - - final_wav = np.concatenate(all_wavs) if len(all_wavs) > 1 else all_wavs[0] - return self._apply_watermark(final_wav) - - def _format_turbo_prompt(self, phonemes: str) -> str: - return ( - f"<|speaker_16|>" - f"<|TEXT_PROMPT_START|>{phonemes}<|TEXT_PROMPT_END|>" - f"<|SPEECH_GENERATION_START|>" - ) - - def infer_stream( - self, - text: str, - ref_codes: Optional[Any] = None, - temperature: float = 0.4, - top_k: int = 50, - max_chars: int = 256, - skip_normalize: bool = False, - skip_phonemize: bool = False, - **kwargs - ) -> Generator[np.ndarray, None, None]: - phonemes = phonemize_text(text) if not skip_phonemize else text - - chunks = split_into_chunks_v2(phonemes, max_chunk_size=max_chars) - - if ref_codes is None: - ref_codes = self.get_preset_voice() - - voice_embedding = self._get_voice_params(ref_codes) - - for i, chunk in enumerate(chunks): - prompt = self._format_turbo_prompt(chunk.text) - - self.backbone.reset() - result = self.backbone( - prompt, - max_tokens=2048, - temperature=temperature, - top_k=top_k, - top_p=0.95, - min_p=0.05, - stop=["<|SPEECH_GENERATION_END|>"], - repeat_penalty=1.15, - echo=False, - ) - wav = self._decode(result["choices"][0]["text"], voice_embedding) - yield self._apply_watermark(wav) - - if i < len(chunks) - 1: - silence_dur = get_silence_duration_v2(chunk) - if silence_dur > 0: - yield np.zeros(int(self.sample_rate * silence_dur), dtype=np.float32) - - def infer_batch(self, texts: List[str], apply_watermark: bool = True, **kwargs) -> List[np.ndarray]: - results = [self.infer(t, **kwargs) for t in texts] - if apply_watermark: - results = [self._apply_watermark(r) for r in results] - return results - - def close(self): - if self.backbone: - self.backbone.close() - self.backbone = None - self.decoder_sess = None - self.encoder_sess = None diff --git a/src/vieneu/utils.py b/src/vieneu/utils.py deleted file mode 100644 index c31c6c8cd56cf3eb0e993e0563cd0ad65e18f05c..0000000000000000000000000000000000000000 --- a/src/vieneu/utils.py +++ /dev/null @@ -1,125 +0,0 @@ -import numpy as np -import re -import logging -from typing import List, Dict, Optional, Any - -# Configure logging -logger = logging.getLogger("Vieneu.Utils") - -# Persistent cache for weights to avoid recomputing if frame_length is constant -_WEIGHT_CACHE: Dict[int, np.ndarray] = {} - -def _linear_overlap_add(frames: List[np.ndarray], stride: int) -> np.ndarray: - """ - Perform linear overlap-add on a list of audio frames. - - Original implementation inspired by: - https://github.com/facebookresearch/encodec/blob/main/encodec/utils.py - - Args: - frames: List of audio frames to join. - stride: Stride between frames in samples. - - Returns: - Joined audio waveform. - """ - if not frames: - return np.array([], dtype=np.float32) - - dtype = frames[0].dtype - shape = frames[0].shape[:-1] - - total_size = 0 - for i, frame in enumerate(frames): - frame_end = stride * i + frame.shape[-1] - total_size = max(total_size, frame_end) - - sum_weight = np.zeros(total_size, dtype=dtype) - out = np.zeros((*shape, total_size), dtype=dtype) - - offset: int = 0 - for frame in frames: - frame_length = frame.shape[-1] - - if frame_length not in _WEIGHT_CACHE or _WEIGHT_CACHE[frame_length].dtype != dtype: - # Recompute weight if not in cache or dtype mismatch - t = np.linspace(0, 1, frame_length + 2, dtype=dtype)[1:-1] - weight = np.abs(0.5 - (t - 0.5)) - _WEIGHT_CACHE[frame_length] = weight - else: - weight = _WEIGHT_CACHE[frame_length] - - out[..., offset : offset + frame_length] += weight * frame - sum_weight[offset : offset + frame_length] += weight - offset += stride - - # Ensure no division by zero; use small epsilon if needed - safe_sum_weight = np.where(sum_weight > 0, sum_weight, 1.0) - return out / safe_sum_weight - -def _compile_codec_with_triton(codec: Any) -> bool: - """ - Compile codec with Triton for faster decoding (Windows/Linux compatible). - - Args: - codec: The codec model to compile. - - Returns: - True if compilation was successful, False otherwise. - """ - try: - import triton - import torch - - if hasattr(codec, 'dec') and hasattr(codec.dec, 'resblocks'): - if len(codec.dec.resblocks) > 2: - # Use torch.compile with triton-friendly backend - codec.dec.resblocks[2].forward = torch.compile( - codec.dec.resblocks[2].forward, - mode="reduce-overhead", - dynamic=True - ) - logger.info(" ✅ Triton compilation enabled for codec") - return True - - except ImportError: - # Silently fail for optional triton optimization - return False - except Exception as e: - logger.error(f" ⚠️ Triton compilation failed: {e}") - return False - -# Pre-compile regex for speech token extraction -RE_SPEECH_TOKEN = re.compile(r"<\|speech_(\d+)\|>") - -def extract_speech_ids(codes_str: str) -> List[int]: - """Extract speech token IDs from a string using regex.""" - return [int(num) for num in RE_SPEECH_TOKEN.findall(codes_str)] - -# Shared Built-in voices for Turbo v2 -TURBO_VOICE_PRESETS = { - "Xuân Vĩnh (Nam - Miền Nam)": { - "description": "Xuân Vĩnh (Nam - Miền Nam)", - "voice_id": 3, - "codes": [3], - "text": "Chào mừng bạn đến với VieNeu-TTS phiên bản Turbo v2." - }, - "Đoan Trang (Nữ - Miền Bắc)": { - "description": "Đoan Trang (Nữ - Miền Bắc)", - "voice_id": 0, - "codes": [0], - "text": "Chào mừng bạn đến với VieNeu-TTS phiên bản Turbo v2." - }, - "Thục Đoan (Nữ - Miền Nam)": { - "description": "Thục Đoan (Nữ - Miền Nam)", - "voice_id": 1, - "codes": [1], - "text": "Chào mừng bạn đến với VieNeu-TTS phiên bản Turbo v2." - }, - "Phạm Tuyên (Nam - Miền Bắc)": { - "description": "Phạm Tuyên (Nam - Miền Bắc)", - "voice_id": 2, - "codes": [2], - "text": "Chào mừng bạn đến với VieNeu-TTS phiên bản Turbo v2." - } -} diff --git a/src/vieneu_utils/__init__.py b/src/vieneu_utils/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/src/vieneu_utils/__pycache__/__init__.cpython-312.pyc b/src/vieneu_utils/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 122038382f80f7b9c56849978007f1d51c54d2a8..0000000000000000000000000000000000000000 Binary files a/src/vieneu_utils/__pycache__/__init__.cpython-312.pyc and /dev/null differ diff --git a/src/vieneu_utils/__pycache__/core_utils.cpython-312.pyc b/src/vieneu_utils/__pycache__/core_utils.cpython-312.pyc deleted file mode 100644 index ff4ed6da9d197db96f99732e9b4cdf0058c7aea4..0000000000000000000000000000000000000000 Binary files a/src/vieneu_utils/__pycache__/core_utils.cpython-312.pyc and /dev/null differ diff --git a/src/vieneu_utils/__pycache__/phonemize_text.cpython-312.pyc b/src/vieneu_utils/__pycache__/phonemize_text.cpython-312.pyc deleted file mode 100644 index f2d444c661cf30f141be2b4ab5e487ea4dfbbd38..0000000000000000000000000000000000000000 Binary files a/src/vieneu_utils/__pycache__/phonemize_text.cpython-312.pyc and /dev/null differ diff --git a/src/vieneu_utils/core_utils.py b/src/vieneu_utils/core_utils.py deleted file mode 100644 index c404cc6a408b480dbef81423b8441f78736df70f..0000000000000000000000000000000000000000 --- a/src/vieneu_utils/core_utils.py +++ /dev/null @@ -1,301 +0,0 @@ -import re -import os -from dataclasses import dataclass -from typing import List, Tuple, Optional - -import numpy as np - -# ─── Regex ─────────────────────────────────────────────────────────────────── - -RE_NEWLINE = re.compile(r'[\r\n]+') # dùng chung cho cả v1 và v2 -RE_SENTENCE_FINDALL = re.compile(r'[^.!?]*[.!?]|[^.!?]+') - -# v1 only -RE_SENTENCE_END = re.compile(r'(?<=[\.\!\?\…])\s+') -RE_MINOR_PUNCT = re.compile(r'(?<=[\,\;\:\-\–\—])\s+') - -# v2 noise cleanup -_NOISE_RULES: List[Tuple[re.Pattern, str]] = [ - (re.compile(r'([.!?])[.,;:]+'), r'\1'), - (re.compile(r'[.,;:]+([.!?])'), r'\1'), - (re.compile(r'\s+[,;]\s+'), ' '), - (re.compile(r' {2,}'), ' '), -] -_MULTI_PUNCT = re.compile(r'([.!?])\s*[.!?]+') - -# ─── Data class ────────────────────────────────────────────────────────────── - -@dataclass -class PhoneChunk: - text: str - is_sentence_end: bool # True = kết thúc câu thật | False = cắt nhân tạo - -# ─── Audio utils ───────────────────────────────────────────────────────────── - -def join_audio_chunks( - chunks: List[np.ndarray], - sr: int, - silence_p: float = 0.0, - crossfade_p: float = 0.0, -) -> np.ndarray: - if not chunks: - return np.array([], dtype=np.float32) - if len(chunks) == 1: - return chunks[0] - - silence_samples = int(sr * silence_p) - crossfade_samples = int(sr * crossfade_p) - final_wav = chunks[0] - - for i in range(1, len(chunks)): - next_chunk = chunks[i] - if silence_samples > 0: - silence = np.zeros(silence_samples, dtype=np.float32) - final_wav = np.concatenate([final_wav, silence, next_chunk]) - elif crossfade_samples > 0: - overlap = min(len(final_wav), len(next_chunk), crossfade_samples) - if overlap > 0: - fade_out = np.linspace(1.0, 0.0, overlap, dtype=np.float32) - fade_in = np.linspace(0.0, 1.0, overlap, dtype=np.float32) - blended = final_wav[-overlap:] * fade_out + next_chunk[:overlap] * fade_in - final_wav = np.concatenate([final_wav[:-overlap], blended, next_chunk[overlap:]]) - else: - final_wav = np.concatenate([final_wav, next_chunk]) - else: - final_wav = np.concatenate([final_wav, next_chunk]) - - return final_wav - -# ─── v1: split raw text ────────────────────────────────────────────────────── - -def split_text_into_chunks(text: str, max_chars: int = 256) -> List[str]: - """Split raw text (chưa phonemize) thành chunks <= max_chars.""" - if not text: - return [] - - paragraphs = RE_NEWLINE.split(text.strip()) - final_chunks: List[str] = [] - - for para in paragraphs: - para = para.strip() - if not para: - continue - - sentences = RE_SENTENCE_END.split(para) - buffer = "" - - for sentence in sentences: - sentence = sentence.strip() - if not sentence: - continue - - if len(sentence) > max_chars: - if buffer: - final_chunks.append(buffer) - buffer = "" - - sub_parts = RE_MINOR_PUNCT.split(sentence) - for part in sub_parts: - part = part.strip() - if not part: - continue - if len(buffer) + 1 + len(part) <= max_chars: - buffer = (buffer + ' ' + part) if buffer else part - else: - if buffer: - final_chunks.append(buffer) - buffer = part - if len(buffer) > max_chars: - words, current = buffer.split(), "" - for word in words: - if current and len(current) + 1 + len(word) > max_chars: - final_chunks.append(current) - current = word - else: - current = (current + ' ' + word) if current else word - buffer = current - else: - if buffer and len(buffer) + 1 + len(sentence) > max_chars: - final_chunks.append(buffer) - buffer = sentence - else: - buffer = (buffer + ' ' + sentence) if buffer else sentence - - if buffer: - final_chunks.append(buffer) - - return [c.strip() for c in final_chunks if c.strip()] - -# ─── v2 helpers ────────────────────────────────────────────────────────────── - -def _pick_strongest(m: re.Match) -> str: - s = m.group(0) - return '!' if '!' in s else '?' if '?' in s else '.' - - -def _clean_phoneme_noise(text: str) -> str: - for pattern, repl in _NOISE_RULES: - text = pattern.sub(repl, text) - return _MULTI_PUNCT.sub(_pick_strongest, text).strip() - - -def _find_best_split(text: str, max_size: int) -> Tuple[int, bool]: - mid = max_size // 2 - best_comma_pos, best_comma_dist = -1, max_size - best_space_pos, best_space_dist = -1, max_size - - for i in range(min(max_size, len(text))): - ch = text[i] - if ch == ',': - d = abs(i - mid) - if d < best_comma_dist: - best_comma_dist, best_comma_pos = d, i - elif ch == ' ': - d = abs(i - mid) - if d < best_space_dist: - best_space_dist, best_space_pos = d, i - - if best_comma_pos != -1: - return best_comma_pos, True - if best_space_pos != -1: - return best_space_pos, False - return -1, False - - -def _smart_split_body(text: str, max_chunk_size: int) -> List[str]: - result: List[str] = [] - stack = [text.strip()] - - while stack: - seg = stack.pop() - if not seg: - continue - if len(seg) <= max_chunk_size: - result.append(seg) - continue - - pos, _ = _find_best_split(seg, max_chunk_size) - if pos != -1: - left = seg[:pos].rstrip() - right = seg[pos + 1:].lstrip() - else: - cut = max_chunk_size - while cut > 0 and seg[cut - 1] != ' ': - cut -= 1 - if cut == 0: - cut = max_chunk_size - left = seg[:cut].rstrip() - right = seg[cut:].lstrip() - - if right: - stack.append(right) - if left: - stack.append(left) - - return result - - -def _split_sentence(sent: str, max_chunk_size: int) -> List[PhoneChunk]: - sent = sent.strip() - if not sent: - return [] - - if sent[-1] in '.!?': - body, punct = sent[:-1].rstrip(), sent[-1] - else: - body, punct = sent, '.' - - if not body: - return [] - - if len(sent) <= max_chunk_size: - return [PhoneChunk(text=body + punct, is_sentence_end=True)] - - sub_chunks = _smart_split_body(body, max_chunk_size) - if not sub_chunks: - return [PhoneChunk(text=punct, is_sentence_end=True)] - - last_idx = len(sub_chunks) - 1 - return [ - PhoneChunk( - text=chunk + (punct if i == last_idx else '.'), - is_sentence_end=(i == last_idx), - ) - for i, chunk in enumerate(sub_chunks) - if chunk - ] - -# ─── v2: split phoneme string ──────────────────────────────────────────────── - -def split_into_chunks_v2( - full_phones: str, - max_chunk_size: int = 256, - min_chunk_size: int = 10, -) -> List[PhoneChunk]: - """ - Phân đoạn chuỗi phoneme thành các PhoneChunk. - is_sentence_end=True → kết thúc câu thật → cần silence - is_sentence_end=False → cắt nhân tạo → không cần silence - """ - if not full_phones: - return [] - - full_phones = _clean_phoneme_noise(full_phones) - - raw_parts: List[PhoneChunk] = [] - for para in RE_NEWLINE.split(full_phones): - para = para.strip() - if not para: - continue - for sent in RE_SENTENCE_FINDALL.findall(para): - sent = sent.strip() - if sent: - raw_parts.extend(_split_sentence(sent, max_chunk_size)) - - if not raw_parts: - return [] - - merged: List[PhoneChunk] = [] - i, n = 0, len(raw_parts) - while i < n: - cur = raw_parts[i] - while len(cur.text) < min_chunk_size and i + 1 < n: - nxt = raw_parts[i + 1] - candidate = cur.text.rstrip('.!?').rstrip() + ' ' + nxt.text - if len(candidate) <= max_chunk_size: - cur = PhoneChunk(text=candidate, is_sentence_end=nxt.is_sentence_end) - i += 1 - else: - break - merged.append(cur) - i += 1 - - if len(merged) >= 2 and len(merged[-1].text) < min_chunk_size: - last = merged.pop() - candidate = merged[-1].text.rstrip('.!?').rstrip() + ' ' + last.text - if len(candidate) <= max_chunk_size: - merged[-1] = PhoneChunk(text=candidate, is_sentence_end=last.is_sentence_end) - else: - merged.append(last) - - return merged - - -def get_silence_duration_v2(chunk: PhoneChunk) -> float: - """ - Silence sau chunk (giây). - is_sentence_end=False → 0.0s - kết thúc '!'/'?' → 0.4s - kết thúc '.' → 0.3s - """ - if not chunk.is_sentence_end: - return 0.0 - return 0.4 if chunk.text.strip()[-1] in '!?' else 0.3 - -# ─── Misc ──────────────────────────────────────────────────────────────────── - -def env_bool(name: str, default: bool = False) -> bool: - v = os.getenv(name) - if v is None: - return default - return v.strip().lower() in ('1', 'true', 'yes', 'y', 'on') \ No newline at end of file diff --git a/src/vieneu_utils/phonemize_text.py b/src/vieneu_utils/phonemize_text.py deleted file mode 100644 index e7d08efa068d75459bbdbbe40b48f883127a53d7..0000000000000000000000000000000000000000 --- a/src/vieneu_utils/phonemize_text.py +++ /dev/null @@ -1,114 +0,0 @@ -""" -Phonemization module for VieNeu-TTS. -Delegates all normalization and G2P logic to the sea-g2p library, -which provides a unified, tested, and maintained Vietnamese G2P pipeline. -""" -import functools -import logging -from sea_g2p import SEAPipeline, G2P, Normalizer - -logger = logging.getLogger("Vieneu.Phonemizer") - -# --------------------------------------------------------------------------- -# Shared singletons (instantiation is lazy-safe and thread-safe via GIL) -# --------------------------------------------------------------------------- -_pipeline: SEAPipeline = None -_g2p: G2P = None -_normalizer: Normalizer = None - -def _get_pipeline() -> SEAPipeline: - global _pipeline - if _pipeline is None: - _pipeline = SEAPipeline(lang="vi") - return _pipeline - -def _get_g2p() -> G2P: - global _g2p - if _g2p is None: - _g2p = G2P(lang="vi") - return _g2p - -def _get_normalizer() -> Normalizer: - global _normalizer - if _normalizer is None: - _normalizer = Normalizer() - return _normalizer - -# --------------------------------------------------------------------------- -# Public API (same signatures as before — callers don't need to change) -# --------------------------------------------------------------------------- - -@functools.lru_cache(maxsize=1024) -def _phonemize_cached(text: str) -> str: - """Cached single-text phonemization (normalize + G2P).""" - return _get_pipeline().run(text) - - -def phonemize_text(text: str) -> str: - """Normalize and phonemize a single Vietnamese/bilingual text string.""" - return _phonemize_cached(text) - - -def phonemize_batch( - texts: list[str], - skip_normalize: bool = False, - phoneme_dict: dict = None, - **kwargs, -) -> list[str]: - """ - Phonemize multiple texts with bilingual support. - - Args: - texts: List of input strings. - skip_normalize: If True, assume the texts are already normalized - (i.e. only run G2P, not the normalizer). - phoneme_dict: Optional custom {word: phoneme} dict that overrides - the built-in dictionary for specific words. - """ - if not texts: - return [] - - g2p = _get_g2p() - - if skip_normalize: - # Texts are pre-normalized — only run the G2P layer - return g2p.phonemize_batch(texts, phoneme_dict=phoneme_dict) - else: - # Full pipeline: normalize then G2P - normalizer = _get_normalizer() - normalized = [normalizer.normalize(t) for t in texts] - return g2p.phonemize_batch(normalized, phoneme_dict=phoneme_dict) - - -def phonemize_with_dict( - text: str, - phoneme_dict: dict = None, - skip_normalize: bool = False, -) -> str: - """ - Phonemize a single text, optionally with a custom word→phoneme mapping. - - When phoneme_dict is None and skip_normalize is False, the result is - cached via lru_cache for performance. - """ - if phoneme_dict is not None: - # Custom dict supplied — skip cache to avoid cross-contamination - return phonemize_batch( - [text], skip_normalize=skip_normalize, phoneme_dict=phoneme_dict - )[0] - if skip_normalize: - return _get_g2p().phonemize_batch([text])[0] - return _phonemize_cached(text) - - -# --------------------------------------------------------------------------- -# CLI helper (python -m vieneu_utils.phonemize_text "some text") -# --------------------------------------------------------------------------- -if __name__ == "__main__": - import sys - test_text = ( - " ".join(sys.argv[1:]) - if len(sys.argv) > 1 - else "Giá SP500 hôm nay là 4.200,5 điểm." - ) - print(f"Output: {phonemize_text(test_text)}") \ No newline at end of file diff --git a/src/vieneu_utils/url_extract.py b/src/vieneu_utils/url_extract.py deleted file mode 100644 index 2f2089e2a91f54333f9689acc2466dbb6e61dc0d..0000000000000000000000000000000000000000 --- a/src/vieneu_utils/url_extract.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Extract article text from URLs using trafilatura.""" - -from urllib.parse import urlparse - -import trafilatura - - -def extract_text_from_url(url: str, max_chars: int = 5000) -> dict: - """ - Extract article text from a URL. - - Returns dict with keys: title, text, char_count, truncated, error - """ - # Validate URL - try: - parsed = urlparse(url) - if parsed.scheme not in ("http", "https"): - return {"title": None, "text": None, "char_count": 0, "truncated": False, "error": "URL must start with http:// or https://"} - if not parsed.netloc: - return {"title": None, "text": None, "char_count": 0, "truncated": False, "error": "Invalid URL"} - except Exception: - return {"title": None, "text": None, "char_count": 0, "truncated": False, "error": "Invalid URL format"} - - # Fetch and extract - try: - downloaded = trafilatura.fetch_url(url) - except Exception as e: - return {"title": None, "text": None, "char_count": 0, "truncated": False, "error": f"Failed to fetch URL: {e}"} - - if not downloaded: - return {"title": None, "text": None, "char_count": 0, "truncated": False, "error": "Could not fetch the URL (may be unreachable or require login)"} - - text = trafilatura.extract(downloaded, include_comments=False, include_tables=False) - if not text or len(text.strip()) < 20: - return {"title": None, "text": None, "char_count": 0, "truncated": False, "error": "Could not extract meaningful text from this URL"} - - # Try to get title - metadata = trafilatura.extract(downloaded, output_format="json", include_comments=False) - title = None - if metadata: - import json - try: - meta_dict = json.loads(metadata) - title = meta_dict.get("title") - except (json.JSONDecodeError, TypeError): - pass - - text = text.strip() - truncated = len(text) > max_chars - if truncated: - text = text[:max_chars] - - return { - "title": title, - "text": text, - "char_count": len(text), - "truncated": truncated, - "error": None, - } diff --git a/utils/__init__.py b/utils/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/utils/__pycache__/__init__.cpython-312.pyc b/utils/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index b1c0d0d5c1b1abc22aab9d99870e61949bc7d6ad..0000000000000000000000000000000000000000 Binary files a/utils/__pycache__/__init__.cpython-312.pyc and /dev/null differ diff --git a/utils/__pycache__/core_utils.cpython-312.pyc b/utils/__pycache__/core_utils.cpython-312.pyc deleted file mode 100644 index a8ab178665cf872ddb8b8a30121d28e9289a766f..0000000000000000000000000000000000000000 Binary files a/utils/__pycache__/core_utils.cpython-312.pyc and /dev/null differ diff --git a/utils/__pycache__/normalize_text.cpython-312.pyc b/utils/__pycache__/normalize_text.cpython-312.pyc deleted file mode 100644 index bd9d2213b77366d3da9b49ea7f06d5e6ac03dcbf..0000000000000000000000000000000000000000 Binary files a/utils/__pycache__/normalize_text.cpython-312.pyc and /dev/null differ diff --git a/utils/__pycache__/phonemize_text.cpython-312.pyc b/utils/__pycache__/phonemize_text.cpython-312.pyc deleted file mode 100644 index 141b5d41ca41d9a22f78ddbd89747d702cf4ee4a..0000000000000000000000000000000000000000 Binary files a/utils/__pycache__/phonemize_text.cpython-312.pyc and /dev/null differ diff --git a/utils/core_utils.py b/utils/core_utils.py deleted file mode 100644 index a5009cac3793ecf270f7f62b6b4481df50d37e23..0000000000000000000000000000000000000000 --- a/utils/core_utils.py +++ /dev/null @@ -1,76 +0,0 @@ -import re -import os -from typing import List - -def split_text_into_chunks(text: str, max_chars: int = 256) -> List[str]: - """ - Split raw text into chunks no longer than max_chars. - """ - # 1. First split by newlines - each line/paragraph is handled independently - paragraphs = re.split(r"[\r\n]+", text.strip()) - final_chunks = [] - - for para in paragraphs: - para = para.strip() - if not para: - continue - - # 2. Split current paragraph into sentences - sentences = re.split(r"(?<=[\.\!\?\…])\s+", para) - - buffer = "" - for sentence in sentences: - sentence = sentence.strip() - if not sentence: - continue - - # If sentence itself is longer than max_chars, we must split it by minor punctuation or words - if len(sentence) > max_chars: - # Flush buffer before handling a giant sentence - if buffer: - final_chunks.append(buffer) - buffer = "" - - # Split giant sentence by minor punctuation (, ; : -) - sub_parts = re.split(r"(?<=[\,\;\:\-\–\—])\s+", sentence) - for part in sub_parts: - part = part.strip() - if not part: continue - - if len(buffer) + 1 + len(part) <= max_chars: - buffer = (buffer + " " + part) if buffer else part - else: - if buffer: final_chunks.append(buffer) - buffer = part - - # If even a sub-part is too long, split by spaces (words) - if len(buffer) > max_chars: - words = buffer.split() - current = "" - for word in words: - if current and len(current) + 1 + len(word) > max_chars: - final_chunks.append(current) - current = word - else: - current = (current + " " + word) if current else word - buffer = current - else: - # Normal sentence: check if it fits in current buffer - if buffer and len(buffer) + 1 + len(sentence) > max_chars: - final_chunks.append(buffer) - buffer = sentence - else: - buffer = (buffer + " " + sentence) if buffer else sentence - - # End of paragraph: flush whatever is in buffer - if buffer: - final_chunks.append(buffer) - buffer = "" - - return [c.strip() for c in final_chunks if c.strip()] - -def env_bool(name: str, default: bool = False) -> bool: - v = os.getenv(name) - if v is None: - return default - return v.strip().lower() in ("1", "true", "yes", "y", "on") \ No newline at end of file diff --git a/utils/normalize_text.py b/utils/normalize_text.py deleted file mode 100644 index 48f10ef808bf75dcf9076f7f01343563bc977383..0000000000000000000000000000000000000000 --- a/utils/normalize_text.py +++ /dev/null @@ -1,428 +0,0 @@ -import re - -class VietnameseTTSNormalizer: - """ - A text normalizer for Vietnamese Text-to-Speech systems. - Converts numbers, dates, units, and special characters into readable Vietnamese text. - """ - - def __init__(self): - self.units = { - 'km': 'ki lô mét', 'dm': 'đê xi mét', 'cm': 'xen ti mét', - 'mm': 'mi li mét', 'nm': 'na nô mét', 'µm': 'mic rô mét', - 'μm': 'mic rô mét', 'm': 'mét', - - 'kg': 'ki lô gam', 'g': 'gam', 'mg': 'mi li gam', - - 'km²': 'ki lô mét vuông', 'km2': 'ki lô mét vuông', - 'm²': 'mét vuông', 'm2': 'mét vuông', - 'cm²': 'xen ti mét vuông', 'cm2': 'xen ti mét vuông', - 'mm²': 'mi li mét vuông', 'mm2': 'mi li mét vuông', - 'ha': 'héc ta', - - 'km³': 'ki lô mét khối', 'km3': 'ki lô mét khối', - 'm³': 'mét khối', 'm3': 'mét khối', - 'cm³': 'xen ti mét khối', 'cm3': 'xen ti mét khối', - 'mm³': 'mi li mét khối', 'mm3': 'mi li mét khối', - 'l': 'lít', 'dl': 'đê xi lít', 'ml': 'mi li lít', 'hl': 'héc tô lít', - - 'v': 'vôn', 'kv': 'ki lô vôn', 'mv': 'mi li vôn', - 'a': 'am pe', 'ma': 'mi li am pe', 'ka': 'ki lô am pe', - 'w': 'oát', 'kw': 'ki lô oát', 'mw': 'mê ga oát', 'gw': 'gi ga oát', - 'kwh': 'ki lô oát giờ', 'mwh': 'mê ga oát giờ', 'wh': 'oát giờ', - 'ω': 'ôm', 'ohm': 'ôm', 'kω': 'ki lô ôm', 'mω': 'mê ga ôm', - - 'hz': 'héc', 'khz': 'ki lô héc', 'mhz': 'mê ga héc', 'ghz': 'gi ga héc', - - 'pa': 'pát cal', 'kpa': 'ki lô pát cal', 'mpa': 'mê ga pát cal', - 'bar': 'ba', 'mbar': 'mi li ba', 'atm': 'át mốt phia', 'psi': 'pi ét xai', - - 'j': 'giun', 'kj': 'ki lô giun', - 'cal': 'ca lo', 'kcal': 'ki lô ca lo', - } - - self.digits = ['không', 'một', 'hai', 'ba', 'bốn', - 'năm', 'sáu', 'bảy', 'tám', 'chín'] - - def normalize(self, text): - """Main normalization pipeline with EN tag protection.""" - # Step 1: Extract and protect EN tags - en_contents = [] - placeholder_pattern = "___EN_PLACEHOLDER_{}___ " - - def extract_en(match): - en_contents.append(match.group(0)) - return placeholder_pattern.format(len(en_contents) - 1) - - text = re.sub(r'.*?', extract_en, text, flags=re.IGNORECASE) - - # Step 2: Normal normalization pipeline - text = text.lower() - text = self._normalize_temperature(text) - text = self._normalize_currency(text) - text = self._normalize_percentage(text) - text = self._normalize_units(text) - text = self._normalize_time(text) - text = self._normalize_date(text) - text = self._normalize_phone(text) - text = self._normalize_numbers(text) - text = self._number_to_words(text) - text = self._normalize_special_chars(text) - text = self._normalize_whitespace(text) - - # Step 3: Restore EN tags - for idx, en_content in enumerate(en_contents): - text = text.replace(placeholder_pattern.format(idx).lower(), en_content + ' ') - - # Final whitespace cleanup - text = self._normalize_whitespace(text) - - return text - - def _normalize_temperature(self, text): - """Convert temperature notation to words.""" - text = re.sub(r'-(\d+(?:[.,]\d+)?)\s*°\s*c\b', r'âm \1 độ xê', text, flags=re.IGNORECASE) - text = re.sub(r'-(\d+(?:[.,]\d+)?)\s*°\s*f\b', r'âm \1 độ ép', text, flags=re.IGNORECASE) - text = re.sub(r'(\d+(?:[.,]\d+)?)\s*°\s*c\b', r'\1 độ xê', text, flags=re.IGNORECASE) - text = re.sub(r'(\d+(?:[.,]\d+)?)\s*°\s*f\b', r'\1 độ ép', text, flags=re.IGNORECASE) - text = re.sub(r'°', ' độ ', text) - return text - - def _normalize_currency(self, text): - """Convert currency notation to words.""" - def decimal_currency(match): - whole = match.group(1) - decimal = match.group(2) - unit = match.group(3) - decimal_words = ' '.join([self.digits[int(d)] for d in decimal]) - unit_map = {'k': 'nghìn', 'm': 'triệu', 'b': 'tỷ'} - unit_word = unit_map.get(unit.lower(), unit) - return f"{whole} phẩy {decimal_words} {unit_word}" - - text = re.sub(r'(\d+)[.,](\d+)\s*([kmb])\b', decimal_currency, text, flags=re.IGNORECASE) - text = re.sub(r'(\d+)\s*k\b', r'\1 nghìn', text, flags=re.IGNORECASE) - text = re.sub(r'(\d+)\s*m\b', r'\1 triệu', text, flags=re.IGNORECASE) - text = re.sub(r'(\d+)\s*b\b', r'\1 tỷ', text, flags=re.IGNORECASE) - text = re.sub(r'(\d+(?:[.,]\d+)?)\s*đ\b', r'\1 đồng', text) - text = re.sub(r'(\d+(?:[.,]\d+)?)\s*vnd\b', r'\1 đồng', text, flags=re.IGNORECASE) - text = re.sub(r'\$\s*(\d+(?:[.,]\d+)?)', r'\1 đô la', text) - text = re.sub(r'(\d+(?:[.,]\d+)?)\s*\$', r'\1 đô la', text) - return text - - def _normalize_percentage(self, text): - """Convert percentage to words.""" - text = re.sub(r'(\d+(?:[.,]\d+)?)\s*%', r'\1 phần trăm', text) - return text - - def _normalize_units(self, text): - """Convert measurement units to words.""" - def expand_compound_with_number(match): - number = match.group(1) - unit1 = match.group(2).lower() - unit2 = match.group(3).lower() - full_unit1 = self.units.get(unit1, unit1) - full_unit2 = self.units.get(unit2, unit2) - return f"{number} {full_unit1} trên {full_unit2}" - - def expand_compound_without_number(match): - unit1 = match.group(1).lower() - unit2 = match.group(2).lower() - full_unit1 = self.units.get(unit1, unit1) - full_unit2 = self.units.get(unit2, unit2) - return f"{full_unit1} trên {full_unit2}" - - text = re.sub(r'(\d+(?:[.,]\d+)?)\s*([a-zA-Zμµ²³°]+)/([a-zA-Zμµ²³°0-9]+)\b', - expand_compound_with_number, text) - text = re.sub(r'\b([a-zA-Zμµ²³°]+)/([a-zA-Zμµ²³°0-9]+)\b', - expand_compound_without_number, text) - - sorted_units = sorted(self.units.items(), key=lambda x: len(x[0]), reverse=True) - for unit, full_name in sorted_units: - pattern = r'(\d+(?:[.,]\d+)?)\s*' + re.escape(unit) + r'\b' - text = re.sub(pattern, rf'\1 {full_name}', text, flags=re.IGNORECASE) - - for unit, full_name in sorted_units: - if any(c in unit for c in '²³°'): - pattern = r'\b' + re.escape(unit) + r'\b' - text = re.sub(pattern, full_name, text, flags=re.IGNORECASE) - - return text - - def _normalize_time(self, text): - """Convert time notation to words with validation.""" - - def validate_and_convert_time(match): - """Validate time components before converting.""" - groups = match.groups() - - # HH:MM:SS format - if len(groups) == 3: - hour, minute, second = groups - hour_int, minute_int, second_int = int(hour), int(minute), int(second) - - if not (0 <= hour_int <= 23): - return match.group(0) - if not (0 <= minute_int <= 59): - return match.group(0) - if not (0 <= second_int <= 59): - return match.group(0) - - return f"{hour} giờ {minute} phút {second} giây" - - # HH:MM or HHhMM format - elif len(groups) == 2: - hour, minute = groups - hour_int, minute_int = int(hour), int(minute) - - if not (0 <= hour_int <= 23): - return match.group(0) - if not (0 <= minute_int <= 59): - return match.group(0) - - return f"{hour} giờ {minute} phút" - - # HHh format - else: - hour = groups[0] - hour_int = int(hour) - - if not (0 <= hour_int <= 23): - return match.group(0) - - return f"{hour} giờ" - - text = re.sub(r'(\d{1,2}):(\d{2}):(\d{2})', validate_and_convert_time, text) - text = re.sub(r'(\d{1,2}):(\d{2})', validate_and_convert_time, text) - text = re.sub(r'(\d{1,2})h(\d{2})', validate_and_convert_time, text) - text = re.sub(r'(\d{1,2})h\b', validate_and_convert_time, text) - - return text - - def _normalize_date(self, text): - """Convert date notation to words with validation.""" - - def is_valid_date(day, month, year): - """Check if date components are valid.""" - day, month, year = int(day), int(month), int(year) - - if not (1 <= day <= 31): - return False - if not (1 <= month <= 12): - return False - - return True - - def date_to_text(match): - day, month, year = match.groups() - if is_valid_date(day, month, year): - return f"ngày {day} tháng {month} năm {year}" - return match.group(0) - - def date_iso_to_text(match): - year, month, day = match.groups() - if is_valid_date(day, month, year): - return f"ngày {day} tháng {month} năm {year}" - return match.group(0) - - def date_short_year(match): - day, month, year = match.groups() - full_year = f"20{year}" if int(year) < 50 else f"19{year}" - if is_valid_date(day, month, full_year): - return f"ngày {day} tháng {month} năm {full_year}" - return match.group(0) - - text = re.sub(r'\bngày\s+(\d{1,2})[/\-](\d{1,2})[/\-](\d{4})\b', - lambda m: date_to_text(m).replace('ngày ngày', 'ngày'), text) - text = re.sub(r'\bngày\s+(\d{1,2})[/\-](\d{1,2})[/\-](\d{2})\b', - lambda m: date_short_year(m).replace('ngày ngày', 'ngày'), text) - text = re.sub(r'\b(\d{4})-(\d{1,2})-(\d{1,2})\b', date_iso_to_text, text) - text = re.sub(r'\b(\d{1,2})[/\-](\d{1,2})[/\-](\d{4})\b', date_to_text, text) - text = re.sub(r'\b(\d{1,2})[/\-](\d{1,2})[/\-](\d{2})\b', date_short_year, text) - - return text - - def _normalize_phone(self, text): - """Convert phone numbers to digit-by-digit reading.""" - def phone_to_text(match): - phone = match.group(0) - phone = re.sub(r'[^\d]', '', phone) - - if phone.startswith('84') and len(phone) >= 10: - phone = '0' + phone[2:] - - if 10 <= len(phone) <= 11: - words = [self.digits[int(d)] for d in phone] - return ' '.join(words) + ' ' - - return match.group(0) - - text = re.sub(r'(\+84|84)[\s\-\.]?\d[\d\s\-\.]{7,}', phone_to_text, text) - text = re.sub(r'\b0\d[\d\s\-\.]{8,}', phone_to_text, text) - return text - - def _normalize_numbers(self, text): - text = re.sub(r'(\d+(?:[,.]\d+)?)%', lambda m: f'{m.group(1)} phần trăm', text) - text = re.sub(r'(\d{1,3})(?:\.(\d{3}))+', lambda m: m.group(0).replace('.', ''), text) - - def decimal_to_words(match): - whole = match.group(1) - decimal = match.group(2) - decimal_words = ' '.join([self.digits[int(d)] for d in decimal]) - separator = 'phẩy' if ',' in match.group(0) else 'chấm' - return f"{whole} {separator} {decimal_words}" - - text = re.sub(r'(\d+),(\d+)', decimal_to_words, text) - text = re.sub(r'(\d+)\.(\d{1,2})\b', decimal_to_words, text) - - return text - - def _read_two_digits(self, n): - """Read two-digit numbers in Vietnamese.""" - if n < 10: - return self.digits[n] - elif n == 10: - return "mười" - elif n < 20: - if n == 15: - return "mười lăm" - return f"mười {self.digits[n % 10]}" - else: - tens = n // 10 - ones = n % 10 - if ones == 0: - return f"{self.digits[tens]} mươi" - elif ones == 1: - return f"{self.digits[tens]} mươi mốt" - elif ones == 5: - return f"{self.digits[tens]} mươi lăm" - else: - return f"{self.digits[tens]} mươi {self.digits[ones]}" - - def _read_three_digits(self, n): - """Read three-digit numbers in Vietnamese.""" - if n < 100: - return self._read_two_digits(n) - - hundreds = n // 100 - remainder = n % 100 - result = f"{self.digits[hundreds]} trăm" - - if remainder == 0: - return result - elif remainder < 10: - result += f" lẻ {self.digits[remainder]}" - else: - result += f" {self._read_two_digits(remainder)}" - - return result - - def _convert_number_to_words(self, num): - """Convert a number to Vietnamese words.""" - if num == 0: - return "không" - - if num < 0: - return f"âm {self._convert_number_to_words(-num)}" - - if num >= 1000000000: - billion = num // 1000000000 - remainder = num % 1000000000 - result = f"{self._read_three_digits(billion)} tỷ" - if remainder > 0: - result += f" {self._convert_number_to_words(remainder)}" - return result - - elif num >= 1000000: - million = num // 1000000 - remainder = num % 1000000 - result = f"{self._read_three_digits(million)} triệu" - if remainder > 0: - result += f" {self._convert_number_to_words(remainder)}" - return result - - elif num >= 1000: - thousand = num // 1000 - remainder = num % 1000 - result = f"{self._read_three_digits(thousand)} nghìn" - if remainder > 0: - if remainder < 10: - result += f" không trăm lẻ {self.digits[remainder]}" - elif remainder < 100: - result += f" không trăm {self._read_two_digits(remainder)}" - else: - result += f" {self._read_three_digits(remainder)}" - return result - - else: - return self._read_three_digits(num) - - def _number_to_words(self, text): - """Convert all remaining numbers to words.""" - def convert_number(match): - num = int(match.group(0)) - return self._convert_number_to_words(num) - - text = re.sub(r'\b\d+\b', convert_number, text) - return text - - def _normalize_special_chars(self, text): - """Handle special characters.""" - # Remove quotes first to avoid creating spaces before commas - text = text.replace('"', '') - text = text.replace("'", '') - text = text.replace(''', '') - text = text.replace(''', '') - text = text.replace('"', '') - text = text.replace('"', '') - - text = text.replace('&', ' và ') - text = text.replace('+', ' cộng ') - text = text.replace('=', ' bằng ') - text = text.replace('#', ' thăng ') - # Handle parentheses/brackets as natural pauses: (text) -> , text , - text = re.sub(r'[\(\[\{]\s*(.*?)\s*[\)\]\}]', r', \1, ', text) - - # Remaining individual brackets or parens - text = re.sub(r'[\[\]\(\)\{\}]', ' ', text) - - # Paired dashes (like parentheses): - text - -> , text , - text = re.sub(r'(?:\s+|^)[-–—]\s*(.*?)\s*[-–—](?:\s+|$)', r', \1 , ', text) - - # Single dashes used as punctuation (with spaces) -> comma - text = re.sub(r'\s+[-–—]+\s+', ', ', text) - - # Dashes at the start of a line (bullet points) -> comma - text = re.sub(r'^[-–—]+\s+', ', ', text) - - # Collapse multiple commas and surrounding spaces (remove spaces before AND after commas) - text = re.sub(r'\s*,\s*', ', ', text) - text = re.sub(r',\s*,+', ',', text) # Remove duplicate commas - - text = re.sub(r'\.{2,}', ' ', text) - text = re.sub(r'\s+\.\s+', ' ', text) - text = re.sub(r'[^\w\sàáảãạăắằẳẵặâấầẩẫậèéẻẽẹêếềểễệìíỉĩịòóỏõọôốồổỗộơớờởỡợùúủũụưứừửữựỳýỷỹỵđ.,!?;:@%_]', ' ', text) - return text - - def _normalize_whitespace(self, text): - """Normalize whitespace.""" - text = re.sub(r'\s+', ' ', text) - text = text.strip() - return text - - -if __name__ == "__main__": - normalizer = VietnameseTTSNormalizer() - - test_texts = [ - "Chỉ cần thay đổi một dấu thanh, ý nghĩa của từ đã hoàn toàn khác biệt. Ví dụ như \"ma\", \"má\", \"mà\", \"mả\", \"mã\", \"mạ\" – đây chính là \"bài toán khó\" mà các kỹ sư công nghệ phải giải quyết để tạo ra một giọng đọc tự nhiên như người bản xứ." - ] - - print("=" * 80) - print("VIETNAMESE TTS NORMALIZATION TEST (WITH EN TAG)") - print("=" * 80) - - for text in test_texts: - print(f"\n📝 Input: {text}") - normalized = normalizer.normalize(text) - print(f"🎵 Output: {normalized}") - print("-" * 80) \ No newline at end of file diff --git a/utils/phoneme_dict.json b/utils/phoneme_dict.json deleted file mode 100644 index fc30e510ac6df36ac9dc69343a1b94f5a6b2e344..0000000000000000000000000000000000000000 --- a/utils/phoneme_dict.json +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:331f9583a0ac0c795000b569e141e0c9d50c3005d02c49b631fb30edb4b407d4 -size 18078190 diff --git a/utils/phonemize_text.py b/utils/phonemize_text.py deleted file mode 100644 index 844e6d75ca43cd179db8e3f6a8df87953a744707..0000000000000000000000000000000000000000 --- a/utils/phonemize_text.py +++ /dev/null @@ -1,346 +0,0 @@ -import os -import json -import platform -import glob -import re -from phonemizer import phonemize -from phonemizer.backend.espeak.espeak import EspeakWrapper -from utils.normalize_text import VietnameseTTSNormalizer - -# Configuration -PHONEME_DICT_PATH = os.getenv( - 'PHONEME_DICT_PATH', - os.path.join(os.path.dirname(__file__), "phoneme_dict.json") -) - -def load_phoneme_dict(path=PHONEME_DICT_PATH): - """Load phoneme dictionary from JSON file.""" - try: - with open(path, "r", encoding="utf-8") as f: - return json.load(f) - except FileNotFoundError: - raise FileNotFoundError( - f"Phoneme dictionary not found at {path}. " - "Please create it or set PHONEME_DICT_PATH environment variable." - ) - -def setup_espeak_library(): - """Configure eSpeak library path based on operating system.""" - system = platform.system() - - if system == "Windows": - _setup_windows_espeak() - elif system == "Linux": - _setup_linux_espeak() - elif system == "Darwin": - _setup_macos_espeak() - else: - raise OSError( - f"Unsupported OS: {system}. " - "Only Windows, Linux, and macOS are supported." - ) - -def _setup_windows_espeak(): - """Setup eSpeak for Windows.""" - default_path = r"C:\Program Files\eSpeak NG\libespeak-ng.dll" - if os.path.exists(default_path): - EspeakWrapper.set_library(default_path) - else: - raise FileNotFoundError( - f"eSpeak library not found at {default_path}. " - "Please install eSpeak NG from: https://github.com/espeak-ng/espeak-ng/releases" - ) - -def _setup_linux_espeak(): - """Setup eSpeak for Linux.""" - search_patterns = [ - "/usr/lib/x86_64-linux-gnu/libespeak-ng.so*", - "/usr/lib/x86_64-linux-gnu/libespeak.so*", - "/usr/lib/libespeak-ng.so*", - "/usr/lib64/libespeak-ng.so*", - "/usr/local/lib/libespeak-ng.so*", - ] - - for pattern in search_patterns: - matches = glob.glob(pattern) - if matches: - EspeakWrapper.set_library(sorted(matches, key=len)[0]) - return - - raise RuntimeError( - "eSpeak NG library not found. Install with:\n" - " Ubuntu/Debian: sudo apt-get install espeak-ng\n" - " Fedora: sudo dnf install espeak-ng\n" - " Arch: sudo pacman -S espeak-ng\n" - "See: https://github.com/pnnbao97/VieNeu-TTS/issues/5" - ) - -def _setup_macos_espeak(): - """Setup eSpeak for macOS.""" - espeak_lib = os.environ.get('PHONEMIZER_ESPEAK_LIBRARY') - - paths_to_check = [ - espeak_lib, - "/opt/homebrew/lib/libespeak-ng.dylib", # Apple Silicon - "/usr/local/lib/libespeak-ng.dylib", # Intel - "/opt/local/lib/libespeak-ng.dylib", # MacPorts - ] - - for path in paths_to_check: - if path and os.path.exists(path): - EspeakWrapper.set_library(path) - return - - raise FileNotFoundError( - "eSpeak library not found. Install with:\n" - " brew install espeak-ng\n" - "Or set: export PHONEMIZER_ESPEAK_LIBRARY=/path/to/libespeak-ng.dylib" - ) - -# Initialize -try: - setup_espeak_library() - phoneme_dict = load_phoneme_dict() - normalizer = VietnameseTTSNormalizer() -except Exception as e: - print(f"Initialization error: {e}") - raise - -def phonemize_text(text: str) -> str: - """ - Convert text to phonemes (simple version without dict, without EN tag). - Kept for backward compatibility. - """ - text = normalizer.normalize(text) - return phonemize( - text, - language="vi", - backend="espeak", - preserve_punctuation=True, - with_stress=True, - language_switch="remove-flags" - ) - - -def phonemize_with_dict(text: str, phoneme_dict=phoneme_dict) -> str: - """ - Phonemize single text with dictionary lookup and EN tag support. - """ - text = normalizer.normalize(text) - - # Split by EN tags - parts = re.split(r'(.*?)', text, flags=re.IGNORECASE) - - en_texts = [] - en_indices = [] - vi_texts = [] - vi_indices = [] - vi_word_maps = [] - - processed_parts = [] - - for part_idx, part in enumerate(parts): - if re.match(r'.*', part, re.IGNORECASE): - # English part - en_content = re.sub(r'', '', part, flags=re.IGNORECASE).strip() - en_texts.append(en_content) - en_indices.append(part_idx) - processed_parts.append(None) - else: - # Vietnamese part - words = part.split() - processed_words = [] - - for word_idx, word in enumerate(words): - match = re.match(r'^(\W*)(.*?)(\W*)$', word) - pre, core, suf = match.groups() if match else ("", word, "") - - if not core: - processed_words.append(word) - elif core in phoneme_dict: - processed_words.append(f"{pre}{phoneme_dict[core]}{suf}") - else: - vi_texts.append(word) - vi_indices.append(part_idx) - vi_word_maps.append((part_idx, len(processed_words))) - processed_words.append(None) - - processed_parts.append(processed_words) - - if en_texts: - try: - en_phonemes = phonemize( - en_texts, - language='en-us', - backend='espeak', - preserve_punctuation=True, - with_stress=True, - language_switch="remove-flags" - ) - - if isinstance(en_phonemes, str): - en_phonemes = [en_phonemes] - - for idx, (part_idx, phoneme) in enumerate(zip(en_indices, en_phonemes)): - processed_parts[part_idx] = phoneme.strip() - except Exception as e: - print(f"Warning: Could not phonemize EN texts: {e}") - for part_idx in en_indices: - processed_parts[part_idx] = en_texts[en_indices.index(part_idx)] - - if vi_texts: - try: - vi_phonemes = phonemize( - vi_texts, - language='vi', - backend='espeak', - preserve_punctuation=True, - with_stress=True, - language_switch='remove-flags' - ) - - if isinstance(vi_phonemes, str): - vi_phonemes = [vi_phonemes] - - for idx, (part_idx, word_idx) in enumerate(vi_word_maps): - phoneme = vi_phonemes[idx].strip() - - original_word = vi_texts[idx] - if original_word.lower().startswith('r'): - phoneme = 'ɹ' + phoneme[1:] if len(phoneme) > 0 else phoneme - - phoneme_dict[original_word] = phoneme - - if processed_parts[part_idx] is not None: - processed_parts[part_idx][word_idx] = phoneme - except Exception as e: - print(f"Warning: Could not phonemize VI texts: {e}") - for idx, (part_idx, word_idx) in enumerate(vi_word_maps): - if processed_parts[part_idx] is not None: - processed_parts[part_idx][word_idx] = vi_texts[idx] - - final_parts = [] - for part in processed_parts: - if isinstance(part, list): - final_parts.append(' '.join(str(w) for w in part if w is not None)) - elif part is not None: - final_parts.append(part) - - result = ' '.join(final_parts) - - result = re.sub(r'\s+([.,!?;:])', r'\1', result) - - return result - - -def phonemize_batch(texts: list, phoneme_dict=phoneme_dict) -> list: - """ - Phonemize multiple texts with optimal batching. - - Args: - texts: List of text strings to phonemize - phoneme_dict: Phoneme dictionary for lookup - - Returns: - List of phonemized texts - """ - normalized_texts = [normalizer.normalize(text) for text in texts] - - all_en_texts = [] - all_en_maps = [] - - all_vi_texts = [] - all_vi_maps = [] - - results = [] - - for text_idx, text in enumerate(normalized_texts): - parts = re.split(r'(.*?)', text, flags=re.IGNORECASE) - processed_parts = [] - - for part_idx, part in enumerate(parts): - if re.match(r'.*', part, re.IGNORECASE): - en_content = re.sub(r'', '', part, flags=re.IGNORECASE).strip() - all_en_texts.append(en_content) - all_en_maps.append((text_idx, part_idx)) - processed_parts.append(None) - else: - words = part.split() - processed_words = [] - - for word in words: - match = re.match(r'^(\W*)(.*?)(\W*)$', word) - pre, core, suf = match.groups() if match else ("", word, "") - - if not core: - processed_words.append(word) - elif core in phoneme_dict: - processed_words.append(f"{pre}{phoneme_dict[core]}{suf}") - else: - all_vi_texts.append(word) - all_vi_maps.append((text_idx, part_idx, len(processed_words))) - processed_words.append(None) - - processed_parts.append(processed_words) - - results.append(processed_parts) - - if all_en_texts: - try: - en_phonemes = phonemize( - all_en_texts, - language='en-us', - backend='espeak', - preserve_punctuation=True, - with_stress=True, - language_switch="remove-flags" - ) - - if isinstance(en_phonemes, str): - en_phonemes = [en_phonemes] - - for (text_idx, part_idx), phoneme in zip(all_en_maps, en_phonemes): - results[text_idx][part_idx] = phoneme.strip() - except Exception as e: - print(f"Warning: Batch EN phonemization failed: {e}") - - if all_vi_texts: - try: - vi_phonemes = phonemize( - all_vi_texts, - language='vi', - backend='espeak', - preserve_punctuation=True, - with_stress=True, - language_switch='remove-flags' - ) - - if isinstance(vi_phonemes, str): - vi_phonemes = [vi_phonemes] - - for idx, (text_idx, part_idx, word_idx) in enumerate(all_vi_maps): - phoneme = vi_phonemes[idx].strip() - - original_word = all_vi_texts[idx] - if original_word.lower().startswith('r'): - phoneme = 'ɹ' + phoneme[1:] if len(phoneme) > 0 else phoneme - - phoneme_dict[original_word] = phoneme - results[text_idx][part_idx][word_idx] = phoneme - except Exception as e: - print(f"Warning: Batch VI phonemization failed: {e}") - - final_results = [] - for processed_parts in results: - final_parts = [] - for part in processed_parts: - if isinstance(part, list): - final_parts.append(' '.join(str(w) for w in part if w is not None)) - elif part is not None: - final_parts.append(part) - - result = ' '.join(final_parts) - result = re.sub(r'\s+([.,!?;:])', r'\1', result) - final_results.append(result) - - return final_results \ No newline at end of file diff --git a/vieneu_tts.py b/vieneu_tts.py deleted file mode 100644 index a9c95b6c4620188dd839d5a0ef7e3ffdc82d5de8..0000000000000000000000000000000000000000 --- a/vieneu_tts.py +++ /dev/null @@ -1,859 +0,0 @@ -from pathlib import Path -from typing import Generator -import librosa -import numpy as np -import torch -from neucodec import NeuCodec, DistillNeuCodec -from utils.phonemize_text import phonemize_with_dict -from collections import defaultdict -from concurrent.futures import ThreadPoolExecutor -import re -import gc - -# ============================================================================ -# Shared Utilities -# ============================================================================ - -def _linear_overlap_add(frames: list[np.ndarray], stride: int) -> np.ndarray: - """Linear overlap-add for smooth audio concatenation""" - assert len(frames) - dtype = frames[0].dtype - shape = frames[0].shape[:-1] - - total_size = 0 - for i, frame in enumerate(frames): - frame_end = stride * i + frame.shape[-1] - total_size = max(total_size, frame_end) - - sum_weight = np.zeros(total_size, dtype=dtype) - out = np.zeros(*shape, total_size, dtype=dtype) - - offset: int = 0 - for frame in frames: - frame_length = frame.shape[-1] - t = np.linspace(0, 1, frame_length + 2, dtype=dtype)[1:-1] - weight = np.abs(0.5 - (t - 0.5)) - - out[..., offset : offset + frame_length] += weight * frame - sum_weight[offset : offset + frame_length] += weight - offset += stride - assert sum_weight.min() > 0 - return out / sum_weight - - -def _compile_codec_with_triton(codec): - """Compile codec with Triton for faster decoding (Windows/Linux compatible)""" - try: - import triton - - if hasattr(codec, 'dec') and hasattr(codec.dec, 'resblocks'): - if len(codec.dec.resblocks) > 2: - codec.dec.resblocks[2].forward = torch.compile( - codec.dec.resblocks[2].forward, - mode="reduce-overhead", - dynamic=True - ) - print(" ✅ Triton compilation enabled for codec") - return True - - except ImportError: - print(" ⚠️ Triton not found. Install for faster speed:") - print(" • Linux: pip install triton") - print(" • Windows: pip install triton-windows") - print(" (Optional but recommended)") - return False - - -# ============================================================================ -# VieNeuTTS - Standard implementation (CPU/GPU compatible) -# Supports: PyTorch Transformers, GGUF/GGML quantized models -# ============================================================================ - -class VieNeuTTS: - """ - Standard VieNeu-TTS implementation. - - Supports: - - PyTorch + Transformers backend (CPU/GPU) - - GGUF quantized models via llama-cpp-python (CPU optimized) - - Use this for: - - CPU-only environments - - Standard PyTorch workflows - - GGUF quantized models - """ - - def __init__( - self, - backbone_repo="pnnbao-ump/VieNeu-TTS", - backbone_device="cpu", - codec_repo="neuphonic/neucodec", - codec_device="cpu", - ): - """ - Initialize VieNeu-TTS. - - Args: - backbone_repo: Model repository or path to GGUF file - backbone_device: Device for backbone ('cpu', 'cuda', 'gpu') - codec_repo: Codec repository - codec_device: Device for codec - """ - - # Constants - self.sample_rate = 24_000 - self.max_context = 2048 - self.hop_length = 480 - self.streaming_overlap_frames = 1 - self.streaming_frames_per_chunk = 25 - self.streaming_lookforward = 5 - self.streaming_lookback = 50 - self.streaming_stride_samples = self.streaming_frames_per_chunk * self.hop_length - - # Flags - self._is_quantized_model = False - self._is_onnx_codec = False - - # HF tokenizer - self.tokenizer = None - - # Load models - self._load_backbone(backbone_repo, backbone_device) - self._load_codec(codec_repo, codec_device) - - def _load_backbone(self, backbone_repo, backbone_device): - # MPS device validation - if backbone_device == "mps": - if not torch.backends.mps.is_available(): - print("Warning: MPS not available, falling back to CPU") - backbone_device = "cpu" - - print(f"Loading backbone from: {backbone_repo} on {backbone_device} ...") - - if backbone_repo.lower().endswith("gguf") or "gguf" in backbone_repo.lower(): - try: - from llama_cpp import Llama - except ImportError as e: - raise ImportError( - "Failed to import `llama_cpp`. " - "Xem hướng dẫn cài đặt llama_cpp_python phiên bản tối thiểu 0.3.16 tại: https://llama-cpp-python.readthedocs.io/en/latest/" - ) from e - self.backbone = Llama.from_pretrained( - repo_id=backbone_repo, - filename="*.gguf", - verbose=False, - n_gpu_layers=-1 if backbone_device == "gpu" else 0, - n_ctx=self.max_context, - mlock=True, - flash_attn=True if backbone_device == "gpu" else False, - ) - self._is_quantized_model = True - - else: - from transformers import AutoTokenizer, AutoModelForCausalLM - self.tokenizer = AutoTokenizer.from_pretrained(backbone_repo) - self.backbone = AutoModelForCausalLM.from_pretrained(backbone_repo).to( - torch.device(backbone_device) - ) - - def _load_codec(self, codec_repo, codec_device): - # MPS device validation - if codec_device == "mps": - if not torch.backends.mps.is_available(): - print("Warning: MPS not available for codec, falling back to CPU") - codec_device = "cpu" - - print(f"Loading codec from: {codec_repo} on {codec_device} ...") - match codec_repo: - case "neuphonic/neucodec": - self.codec = NeuCodec.from_pretrained(codec_repo) - self.codec.eval().to(codec_device) - case "neuphonic/distill-neucodec": - self.codec = DistillNeuCodec.from_pretrained(codec_repo) - self.codec.eval().to(codec_device) - case "neuphonic/neucodec-onnx-decoder-int8": - if codec_device != "cpu": - raise ValueError("Onnx decoder only currently runs on CPU.") - try: - from neucodec import NeuCodecOnnxDecoder - except ImportError as e: - raise ImportError( - "Failed to import the onnx decoder." - "Ensure you have onnxruntime installed as well as neucodec >= 0.0.4." - ) from e - self.codec = NeuCodecOnnxDecoder.from_pretrained(codec_repo) - self._is_onnx_codec = True - case _: - raise ValueError(f"Unsupported codec repository: {codec_repo}") - - def encode_reference(self, ref_audio_path: str | Path): - """Encode reference audio to codes""" - wav, _ = librosa.load(ref_audio_path, sr=16000, mono=True) - wav_tensor = torch.from_numpy(wav).float().unsqueeze(0).unsqueeze(0) # [1, 1, T] - with torch.no_grad(): - ref_codes = self.codec.encode_code(audio_or_path=wav_tensor).squeeze(0).squeeze(0) - return ref_codes - - def infer(self, text: str, ref_codes: np.ndarray | torch.Tensor, ref_text: str) -> np.ndarray: - """ - Perform inference to generate speech from text using the TTS model and reference audio. - - Args: - text (str): Input text to be converted to speech. - ref_codes (np.ndarray | torch.tensor): Encoded reference. - ref_text (str): Reference text for reference audio. - Returns: - np.ndarray: Generated speech waveform. - """ - - # Generate tokens - if self._is_quantized_model: - output_str = self._infer_ggml(ref_codes, ref_text, text) - else: - prompt_ids = self._apply_chat_template(ref_codes, ref_text, text) - output_str = self._infer_torch(prompt_ids) - - # Decode - wav = self._decode(output_str) - - return wav - - def infer_stream(self, text: str, ref_codes: np.ndarray | torch.Tensor, ref_text: str) -> Generator[np.ndarray, None, None]: - """ - Perform streaming inference to generate speech from text using the TTS model and reference audio. - - Args: - text (str): Input text to be converted to speech. - ref_codes (np.ndarray | torch.tensor): Encoded reference. - ref_text (str): Reference text for reference audio. - Yields: - np.ndarray: Generated speech waveform. - """ - - if self._is_quantized_model: - return self._infer_stream_ggml(ref_codes, ref_text, text) - else: - raise NotImplementedError("Streaming is not implemented for the torch backend!") - - def _decode(self, codes: str): - """Decode speech tokens to audio waveform.""" - # Extract speech token IDs using regex - speech_ids = [int(num) for num in re.findall(r"<\|speech_(\d+)\|>", codes)] - - if len(speech_ids) == 0: - raise ValueError( - "No valid speech tokens found in the output. " - "The model may not have generated proper speech tokens." - ) - - # Onnx decode - if self._is_onnx_codec: - codes = np.array(speech_ids, dtype=np.int32)[np.newaxis, np.newaxis, :] - recon = self.codec.decode_code(codes) - # Torch decode - else: - with torch.no_grad(): - codes = torch.tensor(speech_ids, dtype=torch.long)[None, None, :].to( - self.codec.device - ) - recon = self.codec.decode_code(codes).cpu().numpy() - - return recon[0, 0, :] - - def _apply_chat_template(self, ref_codes: list[int], ref_text: str, input_text: str) -> list[int]: - input_text = phonemize_with_dict(ref_text) + " " + phonemize_with_dict(input_text) - - speech_replace = self.tokenizer.convert_tokens_to_ids("<|SPEECH_REPLACE|>") - speech_gen_start = self.tokenizer.convert_tokens_to_ids("<|SPEECH_GENERATION_START|>") - text_replace = self.tokenizer.convert_tokens_to_ids("<|TEXT_REPLACE|>") - text_prompt_start = self.tokenizer.convert_tokens_to_ids("<|TEXT_PROMPT_START|>") - text_prompt_end = self.tokenizer.convert_tokens_to_ids("<|TEXT_PROMPT_END|>") - - input_ids = self.tokenizer.encode(input_text, add_special_tokens=False) - chat = """user: Convert the text to speech:<|TEXT_REPLACE|>\nassistant:<|SPEECH_REPLACE|>""" - ids = self.tokenizer.encode(chat) - - text_replace_idx = ids.index(text_replace) - ids = ( - ids[:text_replace_idx] - + [text_prompt_start] - + input_ids - + [text_prompt_end] - + ids[text_replace_idx + 1 :] # noqa - ) - - speech_replace_idx = ids.index(speech_replace) - codes_str = "".join([f"<|speech_{i}|>" for i in ref_codes]) - codes = self.tokenizer.encode(codes_str, add_special_tokens=False) - ids = ids[:speech_replace_idx] + [speech_gen_start] + list(codes) - - return ids - - def _infer_torch(self, prompt_ids: list[int]) -> str: - prompt_tensor = torch.tensor(prompt_ids).unsqueeze(0).to(self.backbone.device) - speech_end_id = self.tokenizer.convert_tokens_to_ids("<|SPEECH_GENERATION_END|>") - with torch.no_grad(): - output_tokens = self.backbone.generate( - prompt_tensor, - max_length=self.max_context, - eos_token_id=speech_end_id, - do_sample=True, - temperature=1.0, - top_k=50, - use_cache=True, - min_new_tokens=50, - ) - input_length = prompt_tensor.shape[-1] - output_str = self.tokenizer.decode( - output_tokens[0, input_length:].cpu().numpy().tolist(), add_special_tokens=False - ) - return output_str - - def _infer_ggml(self, ref_codes: list[int], ref_text: str, input_text: str) -> str: - ref_text = phonemize_with_dict(ref_text) - input_text = phonemize_with_dict(input_text) - - codes_str = "".join([f"<|speech_{idx}|>" for idx in ref_codes]) - prompt = ( - f"user: Convert the text to speech:<|TEXT_PROMPT_START|>{ref_text} {input_text}" - f"<|TEXT_PROMPT_END|>\nassistant:<|SPEECH_GENERATION_START|>{codes_str}" - ) - output = self.backbone( - prompt, - max_tokens=self.max_context, - temperature=0.7, - top_k=50, - stop=["<|SPEECH_GENERATION_END|>"], - ) - output_str = output["choices"][0]["text"] - return output_str - - def _infer_stream_ggml(self, ref_codes: torch.Tensor, ref_text: str, input_text: str) -> Generator[np.ndarray, None, None]: - ref_text = phonemize_with_dict(ref_text) - input_text = phonemize_with_dict(input_text) - - codes_str = "".join([f"<|speech_{idx}|>" for idx in ref_codes]) - prompt = ( - f"user: Convert the text to speech:<|TEXT_PROMPT_START|>{ref_text} {input_text}" - f"<|TEXT_PROMPT_END|>\nassistant:<|SPEECH_GENERATION_START|>{codes_str}" - ) - - audio_cache: list[np.ndarray] = [] - token_cache: list[str] = [f"<|speech_{idx}|>" for idx in ref_codes] - n_decoded_samples: int = 0 - n_decoded_tokens: int = len(ref_codes) - - for item in self.backbone( - prompt, - max_tokens=self.max_context, - temperature=0.7, - top_k=50, - stop=["<|SPEECH_GENERATION_END|>"], - stream=True - ): - output_str = item["choices"][0]["text"] - token_cache.append(output_str) - - if len(token_cache[n_decoded_tokens:]) >= self.streaming_frames_per_chunk + self.streaming_lookforward: - - # decode chunk - tokens_start = max( - n_decoded_tokens - - self.streaming_lookback - - self.streaming_overlap_frames, - 0 - ) - tokens_end = ( - n_decoded_tokens - + self.streaming_frames_per_chunk - + self.streaming_lookforward - + self.streaming_overlap_frames - ) - sample_start = ( - n_decoded_tokens - tokens_start - ) * self.hop_length - sample_end = ( - sample_start - + (self.streaming_frames_per_chunk + 2 * self.streaming_overlap_frames) * self.hop_length - ) - curr_codes = token_cache[tokens_start:tokens_end] - recon = self._decode("".join(curr_codes)) - recon = recon[sample_start:sample_end] - audio_cache.append(recon) - - # postprocess - processed_recon = _linear_overlap_add( - audio_cache, stride=self.streaming_stride_samples - ) - new_samples_end = len(audio_cache) * self.streaming_stride_samples - processed_recon = processed_recon[ - n_decoded_samples:new_samples_end - ] - n_decoded_samples = new_samples_end - n_decoded_tokens += self.streaming_frames_per_chunk - yield processed_recon - - # final decoding handled separately as non-constant chunk size - remaining_tokens = len(token_cache) - n_decoded_tokens - if len(token_cache) > n_decoded_tokens: - tokens_start = max( - len(token_cache) - - (self.streaming_lookback + self.streaming_overlap_frames + remaining_tokens), - 0 - ) - sample_start = ( - len(token_cache) - - tokens_start - - remaining_tokens - - self.streaming_overlap_frames - ) * self.hop_length - curr_codes = token_cache[tokens_start:] - recon = self._decode("".join(curr_codes)) - recon = recon[sample_start:] - audio_cache.append(recon) - - processed_recon = _linear_overlap_add(audio_cache, stride=self.streaming_stride_samples) - processed_recon = processed_recon[n_decoded_samples:] - yield processed_recon - - -# ============================================================================ -# FastVieNeuTTS - GPU-optimized implementation -# Requires: LMDeploy with CUDA -# ============================================================================ - -class FastVieNeuTTS: - """ - GPU-optimized VieNeu-TTS using LMDeploy TurbomindEngine. - """ - - def __init__( - self, - backbone_repo="pnnbao-ump/VieNeu-TTS", - backbone_device="cuda", - codec_repo="neuphonic/neucodec", - codec_device="cuda", - memory_util=0.3, - tp=1, - enable_prefix_caching=True, - quant_policy=0, - enable_triton=True, - max_batch_size=8, - ): - """ - Initialize FastVieNeuTTS with LMDeploy backend and optimizations. - - Args: - backbone_repo: Model repository - backbone_device: Device for backbone (must be CUDA) - codec_repo: Codec repository - codec_device: Device for codec - memory_util: GPU memory utilization (0.0-1.0) - tp: Tensor parallel size for multi-GPU - enable_prefix_caching: Enable prefix caching for faster batch processing - quant_policy: KV cache quantization (0=off, 8=int8, 4=int4) - enable_triton: Enable Triton compilation for codec - max_batch_size: Maximum batch size for inference (prevent GPU overload) - """ - - if backbone_device != "cuda" and not backbone_device.startswith("cuda:"): - raise ValueError("LMDeploy backend requires CUDA device") - - # Constants - self.sample_rate = 24_000 - self.max_context = 2048 - self.hop_length = 480 - self.streaming_overlap_frames = 1 - self.streaming_frames_per_chunk = 50 - self.streaming_lookforward = 5 - self.streaming_lookback = 50 - self.streaming_stride_samples = self.streaming_frames_per_chunk * self.hop_length - - self.max_batch_size = max_batch_size - - self._ref_cache = {} - - self.stored_dict = defaultdict(dict) - - # Flags - self._is_onnx_codec = False - self._triton_enabled = False - - # Load models - self._load_backbone_lmdeploy(backbone_repo, memory_util, tp, enable_prefix_caching, quant_policy) - self._load_codec(codec_repo, codec_device, enable_triton) - - self._warmup_model() - - print("✅ FastVieNeuTTS with optimizations loaded successfully!") - print(f" Max batch size: {self.max_batch_size} (adjustable to prevent GPU overload)") - - def _load_backbone_lmdeploy(self, repo, memory_util, tp, enable_prefix_caching, quant_policy): - """Load backbone using LMDeploy's TurbomindEngine""" - print(f"Loading backbone with LMDeploy from: {repo}") - - try: - from lmdeploy import pipeline, TurbomindEngineConfig, GenerationConfig - except ImportError as e: - raise ImportError( - "Failed to import `lmdeploy`. " - "Xem hướng dẫn cài đặt lmdeploy để tối ưu hiệu suất GPU tại: https://github.com/pnnbao97/VieNeu-TTS" - ) from e - - backend_config = TurbomindEngineConfig( - cache_max_entry_count=memory_util, - tp=tp, - enable_prefix_caching=enable_prefix_caching, - dtype='bfloat16', - quant_policy=quant_policy - ) - - self.backbone = pipeline(repo, backend_config=backend_config) - - self.gen_config = GenerationConfig( - top_p=0.95, - top_k=50, - temperature=0.7, - max_new_tokens=2048, - do_sample=True, - min_new_tokens=40, - ) - - print(f" LMDeploy TurbomindEngine initialized") - print(f" - Memory util: {memory_util}") - print(f" - Tensor Parallel: {tp}") - print(f" - Prefix caching: {enable_prefix_caching}") - print(f" - KV quant: {quant_policy} ({'Enabled' if quant_policy > 0 else 'Disabled'})") - - def _load_codec(self, codec_repo, codec_device, enable_triton): - """Load codec with optional Triton compilation""" - print(f"Loading codec from: {codec_repo} on {codec_device}") - - match codec_repo: - case "neuphonic/neucodec": - self.codec = NeuCodec.from_pretrained(codec_repo) - self.codec.eval().to(codec_device) - case "neuphonic/distill-neucodec": - self.codec = DistillNeuCodec.from_pretrained(codec_repo) - self.codec.eval().to(codec_device) - case "neuphonic/neucodec-onnx-decoder-int8": - if codec_device != "cpu": - raise ValueError("ONNX decoder only runs on CPU") - try: - from neucodec import NeuCodecOnnxDecoder - except ImportError as e: - raise ImportError( - "Failed to import ONNX decoder. " - "Ensure onnxruntime and neucodec >= 0.0.4 are installed." - ) from e - self.codec = NeuCodecOnnxDecoder.from_pretrained(codec_repo) - self._is_onnx_codec = True - case _: - raise ValueError(f"Unsupported codec repository: {codec_repo}") - - if enable_triton and not self._is_onnx_codec and codec_device != "cpu": - self._triton_enabled = _compile_codec_with_triton(self.codec) - - def _warmup_model(self): - """Warmup inference pipeline to reduce first-token latency""" - print("🔥 Warming up model...") - try: - dummy_codes = list(range(10)) - dummy_prompt = self._format_prompt(dummy_codes, "warmup", "test") - _ = self.backbone([dummy_prompt], gen_config=self.gen_config, do_preprocess=False) - print(" ✅ Warmup complete") - except Exception as e: - print(f" ⚠️ Warmup failed (non-critical): {e}") - - def encode_reference(self, ref_audio_path: str | Path): - """Encode reference audio to codes""" - wav, _ = librosa.load(ref_audio_path, sr=16000, mono=True) - wav_tensor = torch.from_numpy(wav).float().unsqueeze(0).unsqueeze(0) - with torch.no_grad(): - ref_codes = self.codec.encode_code(audio_or_path=wav_tensor).squeeze(0).squeeze(0) - return ref_codes - - def get_cached_reference(self, voice_name: str, audio_path: str, ref_text: str = None): - """ - Get or create cached reference codes. - - Args: - voice_name: Unique identifier for this voice - audio_path: Path to reference audio - ref_text: Optional reference text (stored with codes) - - Returns: - ref_codes: Encoded reference codes - """ - cache_key = f"{voice_name}_{audio_path}" - - if cache_key not in self._ref_cache: - ref_codes = self.encode_reference(audio_path) - self._ref_cache[cache_key] = { - 'codes': ref_codes, - 'ref_text': ref_text - } - - return self._ref_cache[cache_key]['codes'] - - def add_speaker(self, user_id: int, audio_file: str, ref_text: str): - """ - Add a speaker to the stored dictionary for easy access. - - Args: - user_id: Unique user ID - audio_file: Reference audio file path - ref_text: Reference text - - Returns: - user_id: The user ID for use in streaming - """ - codes = self.encode_reference(audio_file) - - if isinstance(codes, torch.Tensor): - codes = codes.cpu().numpy() - if isinstance(codes, np.ndarray): - codes = codes.flatten().tolist() - - self.stored_dict[f"{user_id}"]['codes'] = codes - self.stored_dict[f"{user_id}"]['ref_text'] = ref_text - - return user_id - - def _decode(self, codes: str): - """Decode speech tokens to audio waveform""" - speech_ids = [int(num) for num in re.findall(r"<\|speech_(\d+)\|>", codes)] - - if len(speech_ids) == 0: - raise ValueError("No valid speech tokens found in output") - - if self._is_onnx_codec: - codes = np.array(speech_ids, dtype=np.int32)[np.newaxis, np.newaxis, :] - recon = self.codec.decode_code(codes) - else: - with torch.no_grad(): - codes = torch.tensor(speech_ids, dtype=torch.long)[None, None, :].to( - self.codec.device - ) - recon = self.codec.decode_code(codes).cpu().numpy() - - return recon[0, 0, :] - - def _decode_batch(self, codes_list: list[str], max_workers: int = None): - """ - Decode multiple code strings in parallel. - - Args: - codes_list: List of code strings to decode - max_workers: Number of parallel workers (auto-tuned if None) - - Returns: - List of decoded audio arrays - """ - # Auto-tune workers based on GPU memory and batch size - if max_workers is None: - if torch.cuda.is_available(): - gpu_mem_gb = torch.cuda.get_device_properties(0).total_memory / 1e9 - # 1 worker per 4GB VRAM, max 4 workers - max_workers = min(max(1, int(gpu_mem_gb / 4)), 4) - else: - max_workers = 2 - - # For small batches, use sequential to avoid overhead - if len(codes_list) <= 2: - return [self._decode(codes) for codes in codes_list] - - # Parallel decoding with controlled workers - with ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = [executor.submit(self._decode, codes) for codes in codes_list] - results = [f.result() for f in futures] - return results - - def _format_prompt(self, ref_codes: list[int], ref_text: str, input_text: str) -> str: - """Format prompt for LMDeploy""" - ref_text_phones = phonemize_with_dict(ref_text) - input_text_phones = phonemize_with_dict(input_text) - - codes_str = "".join([f"<|speech_{idx}|>" for idx in ref_codes]) - - prompt = ( - f"user: Convert the text to speech:<|TEXT_PROMPT_START|>{ref_text_phones} {input_text_phones}" - f"<|TEXT_PROMPT_END|>\nassistant:<|SPEECH_GENERATION_START|>{codes_str}" - ) - - return prompt - - def infer(self, text: str, ref_codes: np.ndarray | torch.Tensor, ref_text: str) -> np.ndarray: - """ - Single inference. - - Args: - text: Input text to synthesize - ref_codes: Encoded reference audio codes - ref_text: Reference text for reference audio - - Returns: - Generated speech waveform as numpy array - """ - if isinstance(ref_codes, torch.Tensor): - ref_codes = ref_codes.cpu().numpy() - if isinstance(ref_codes, np.ndarray): - ref_codes = ref_codes.flatten().tolist() - - prompt = self._format_prompt(ref_codes, ref_text, text) - - # Use LMDeploy pipeline for generation - responses = self.backbone([prompt], gen_config=self.gen_config, do_preprocess=False) - output_str = responses[0].text - - # Decode to audio - wav = self._decode(output_str) - - return wav - - def infer_batch(self, texts: list[str], ref_codes: np.ndarray | torch.Tensor, ref_text: str, max_batch_size: int = None) -> list[np.ndarray]: - """ - Batch inference for multiple texts. - """ - if max_batch_size is None: - max_batch_size = self.max_batch_size - - if not isinstance(texts, list): - texts = [texts] - - if isinstance(ref_codes, torch.Tensor): - ref_codes = ref_codes.cpu().numpy() - if isinstance(ref_codes, np.ndarray): - ref_codes = ref_codes.flatten().tolist() - - all_wavs = [] - - for i in range(0, len(texts), max_batch_size): - batch_texts = texts[i:i+max_batch_size] - prompts = [self._format_prompt(ref_codes, ref_text, text) for text in batch_texts] - responses = self.backbone(prompts, gen_config=self.gen_config, do_preprocess=False) - batch_codes = [response.text for response in responses] - - if len(batch_codes) > 3: - batch_wavs = self._decode_batch(batch_codes) - else: - batch_wavs = [self._decode(codes) for codes in batch_codes] - all_wavs.extend(batch_wavs) - - if i + max_batch_size < len(texts): - if torch.cuda.is_available(): - torch.cuda.empty_cache() - - return all_wavs - - def infer_stream(self, text: str, ref_codes: np.ndarray | torch.Tensor, ref_text: str) -> Generator[np.ndarray, None, None]: - """ - Streaming inference with low latency. - - Args: - text: Input text to synthesize - ref_codes: Encoded reference audio codes - ref_text: Reference text for reference audio - - Yields: - Audio chunks as numpy arrays - """ - if isinstance(ref_codes, torch.Tensor): - ref_codes = ref_codes.cpu().numpy() - if isinstance(ref_codes, np.ndarray): - ref_codes = ref_codes.flatten().tolist() - - prompt = self._format_prompt(ref_codes, ref_text, text) - - audio_cache = [] - token_cache = [f"<|speech_{idx}|>" for idx in ref_codes] - n_decoded_samples = 0 - n_decoded_tokens = len(ref_codes) - - for response in self.backbone.stream_infer([prompt], gen_config=self.gen_config, do_preprocess=False): - output_str = response.text - - # Extract new tokens - new_tokens = output_str[len("".join(token_cache[len(ref_codes):])):] if len(token_cache) > len(ref_codes) else output_str - - if new_tokens: - token_cache.append(new_tokens) - - # Check if we have enough tokens to decode a chunk - if len(token_cache[n_decoded_tokens:]) >= self.streaming_frames_per_chunk + self.streaming_lookforward: - - # Decode chunk with context - tokens_start = max( - n_decoded_tokens - self.streaming_lookback - self.streaming_overlap_frames, - 0 - ) - tokens_end = ( - n_decoded_tokens - + self.streaming_frames_per_chunk - + self.streaming_lookforward - + self.streaming_overlap_frames - ) - sample_start = (n_decoded_tokens - tokens_start) * self.hop_length - sample_end = ( - sample_start - + (self.streaming_frames_per_chunk + 2 * self.streaming_overlap_frames) * self.hop_length - ) - - curr_codes = token_cache[tokens_start:tokens_end] - recon = self._decode("".join(curr_codes)) - recon = recon[sample_start:sample_end] - audio_cache.append(recon) - - # Overlap-add processing - processed_recon = _linear_overlap_add( - audio_cache, stride=self.streaming_stride_samples - ) - new_samples_end = len(audio_cache) * self.streaming_stride_samples - processed_recon = processed_recon[n_decoded_samples:new_samples_end] - n_decoded_samples = new_samples_end - n_decoded_tokens += self.streaming_frames_per_chunk - - yield processed_recon - - # Final chunk - remaining_tokens = len(token_cache) - n_decoded_tokens - if remaining_tokens > 0: - tokens_start = max( - len(token_cache) - (self.streaming_lookback + self.streaming_overlap_frames + remaining_tokens), - 0 - ) - sample_start = ( - len(token_cache) - tokens_start - remaining_tokens - self.streaming_overlap_frames - ) * self.hop_length - - curr_codes = token_cache[tokens_start:] - recon = self._decode("".join(curr_codes)) - recon = recon[sample_start:] - audio_cache.append(recon) - - processed_recon = _linear_overlap_add(audio_cache, stride=self.streaming_stride_samples) - processed_recon = processed_recon[n_decoded_samples:] - yield processed_recon - - def cleanup_memory(self): - """Clean up GPU memory""" - if torch.cuda.is_available(): - torch.cuda.empty_cache() - gc.collect() - print("🧹 Memory cleaned up") - - def get_optimization_stats(self) -> dict: - """ - Get current optimization statistics. - - Returns: - Dictionary with optimization info - """ - return { - 'triton_enabled': self._triton_enabled, - 'max_batch_size': self.max_batch_size, - 'cached_references': len(self._ref_cache), - 'active_sessions': len(self.stored_dict), - 'kv_quant': self.gen_config.__dict__.get('quant_policy', 0), - 'prefix_caching': True, # Always enabled in our config - }