import gradio as gr from gradio import wasm_utils from fastrtc import ReplyOnPause, AlgoOptions, SileroVadOptions, AdditionalOutputs, WebRTC, get_cloudflare_turn_credentials_async, get_cloudflare_turn_credentials #get_hf_turn_credentials, import os from dotenv import load_dotenv import time import numpy as np import sys import uuid sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) from backend.tts import synthesize_text from backend.asr import transcribe_audio, transcribe_typhoon from backend.utils import preprocess_audio, is_valid_turn, preprocess_audio_simplified from backend.main import stream_chat_response from pydub import AudioSegment from backend.utils import get_device if get_device() == "cpu": load_dotenv(override=True) if get_device() == "mps": load_dotenv(override=True) phone_waiting_sound = AudioSegment.from_mp3("frontend/phone-ringing-382734.mp3") sound_samples = np.array(phone_waiting_sound.get_array_of_samples(), dtype=np.int16) if phone_waiting_sound.channels > 1: sound_samples = sound_samples.reshape((-1, phone_waiting_sound.channels)).mean(axis=1) sound_samples = sound_samples.astype(np.float32) / 32768.0 # Normalize to [-1, def startup(*arg): session_identifier = arg[1] if len(arg) > 1 else None yield (phone_waiting_sound.frame_rate, sound_samples) STARTUP_MESSAGE = "สวัสดีค่ะ พลอย 1577Homeshopping ยินดีให้บริการค่ะ" yield from synthesize_text(STARTUP_MESSAGE) time.sleep(1) yield AdditionalOutputs( [{"role": "assistant", "content": STARTUP_MESSAGE}], session_identifier, ) custom_css = """ /* Overall Gradio page styling: hot pink background */ body { /* background-color: #ff69b4; /* Hot pink */ margin: 0; padding: 0; font-family: sans-serif;} /* Title styling */ h1 { color: #fff; text-shadow: 1px 1px 2px #ff85a2; font-size: 2.5em; margin-bottom: 20px; text-align: center; } /* Style the column holding the telephone interface */ .phone-column { max-width: 350px !important; /* Limit the width of the phone column */ margin: 0 auto; /* Center the column */ border-radius: 20px; background-color: #f9cb9c; /* Lighter pink for telephone interface */ box-shadow: 0 0 15px rgba(0, 0, 0, 0.2); padding: 20px; } /* Conversation history box styling */ #conversation-history-chatbot { background-color: #f9cb9c; /* Lighter pink for conversation history */ border: 1px solid #ccc; border-radius: 10px; padding: 10px; box-shadow: 0 0 15px rgba(0, 0, 0, 0.2); } """ def snapshot_history(history): """Return a shallow copy of the current chatbot history.""" return [dict(turn) for turn in history] if history else [] def response( audio: tuple[int, np.ndarray] | None, conversation_history, session_id: str | None, ): """ Handles user audio input, transcribes it, streams LLM text via backend.main, and synthesizes chunks to audio while updating the conversation history. """ print(f"WebRTC input SR: {audio[0]}") print(f"--- Latency Breakdown ---") # # stage_asr = "normal" #["normal,"gemini"] # print('-----------------------------') # print(f"Initial conver:{conversation_history}") # print('-----------------------------') conversation_history = conversation_history or [] start_time = time.time() session_identifier = session_id or "" generated_session_id = False if not session_identifier: session_identifier = str(uuid.uuid4()) generated_session_id = True print(f"[WARN] Missing session_id; generated temporary session {session_identifier}") if generated_session_id: yield AdditionalOutputs(snapshot_history(conversation_history), session_identifier) if not audio or audio[1] is None or not np.any(audio[1]): print("No audio input detected; skipping response generation.") print(f"------------------------") return import soundfile as sf sample_rate, audio_array = audio try: sr , processed_audio = preprocess_audio_simplified((sample_rate, audio_array), target_sr=16000) print(sr, processed_audio.dtype, processed_audio.min(), processed_audio.max(), processed_audio.shape) except Exception as audio_err: print(f"Audio preprocessing failed: {audio_err}") print(f"------------------------") return silence_duration_s = 0.2 # Calculate the number of samples corresponding to the silence duration silence_samples = int(16000 * silence_duration_s) # Create a silent audio segment (an array of zeros) # Ensure the dtype matches your processed audio for compatibility leading_silence = np.zeros(silence_samples, dtype=np.float32) # Prepend the silence to the beginning of your processed audio audio_with_padding = np.concatenate([leading_silence, processed_audio]) print(f"Added {silence_duration_s}s of silence. New shape: {audio_with_padding.shape}") file_name = "temp.wav" sf.write(file_name, audio_with_padding, sr) t0 = time.time() transcription = transcribe_typhoon(file_name) # transcription = transcribe_audio( "debug_processed.wav") t_asr = time.time() - t0 print(f"ASR: {t_asr:.4f}s") if not transcription.strip(): print("No valid transcription; skipping response generation.") print(f"------------------------") return user_turn = {"role": "user", "content": transcription} print(f"User: {transcription}") if is_valid_turn(user_turn): conversation_history.append(user_turn) yield AdditionalOutputs(snapshot_history(conversation_history), session_identifier) # print("Conversation history:", conversation_history) assistant_turn = {"role": "assistant", "content": ""} conversation_history.append(assistant_turn) text_buffer = "" full_response = "" delimiter_count = 0 n_threshold = 2 max_n_threshold = 2 lang = "th" chunk_count = 0 first_chunk_sent = False start_llm_stream = time.time() try: for chunk in stream_chat_response(session_identifier, transcription): # print(f"LLM chunk: {text_chunk}") if isinstance(chunk, str): text_chunk = chunk i = 0 while i < len(text_chunk): char = text_chunk[i] text_buffer += char full_response += char assistant_turn["content"] = full_response.strip() is_delimiter = False # if char in {' ', '\n'}: if char == "|": #check the next character is number, not count as delimiter # if i + 1 < len(text_chunk) and (text_chunk[i + 1].isdigit() or text_chunk[i - 1].isdigit()): # is_delimiter = False # else: # print(f"text_buffer before removing delimiter: '{text_buffer}'") # text_buffer = text_buffer[:-1] # Remove the delimiter from the buffer # print(f"text_buffer after removing delimiter: '{text_buffer}'") is_delimiter = True delimiter_count += 1 if i + 1 < len(text_chunk) and text_chunk[i + 1] == 'ๆ': text_buffer += text_chunk[i + 1] full_response += text_chunk[i + 1] i += 1 send_now = False if not first_chunk_sent: if is_delimiter and text_buffer.strip(): send_now = True else: if delimiter_count >= n_threshold and text_buffer.strip(): send_now = True if n_threshold < max_n_threshold: n_threshold += 1 if send_now: buffer_to_send = text_buffer.strip() try: if buffer_to_send and buffer_to_send.endswith('วันที่'): buffer_to_send = buffer_to_send[:-len('วันที่')] if buffer_to_send and first_chunk_sent and buffer_to_send.endswith('ค่ะ'): buffer_to_send = buffer_to_send[:-len('ค่ะ')] except Exception: buffer_to_send = buffer_to_send.replace('ค่ะ', '') if buffer_to_send: chunk_count += 1 if chunk_count == 1: first_llm_chunk_time = time.time() t_llm_first_token = first_llm_chunk_time - start_llm_stream print(f"LLM TTFC: {t_llm_first_token:.4f}s (Time To First Chunk)") yield from synthesize_text(buffer_to_send, lang=lang) first_chunk_sent = True text_buffer = "" delimiter_count = 0 yield AdditionalOutputs( snapshot_history(conversation_history), session_identifier, ) i += 1 if text_buffer.strip(): buffer_to_send = text_buffer.strip() try: if buffer_to_send and buffer_to_send.endswith('วันที่'): buffer_to_send = buffer_to_send[:-len('วันที่')] if buffer_to_send and first_chunk_sent and buffer_to_send.endswith('ค่ะ'): buffer_to_send = buffer_to_send[:-len('ค่ะ')] except Exception: buffer_to_send = buffer_to_send.replace('ค่ะ', '') if buffer_to_send: chunk_count += 1 if chunk_count == 1: first_llm_chunk_time = time.time() t_llm_first_token = first_llm_chunk_time - start_llm_stream print(f"LLM TTFC: {t_llm_first_token:.4f}s (Time To First Chunk)") yield from synthesize_text(buffer_to_send, lang=lang) first_chunk_sent = True text_buffer = "" delimiter_count = 0 yield AdditionalOutputs( snapshot_history(conversation_history), session_identifier, ) except Exception as e: print(f"An error occurred during response generation or synthesis: {e}") error_message = "ขออภัยค่ะ เกิดข้อผิดพลาดบางอย่าง" try: yield from synthesize_text(error_message, lang=lang) except Exception as synth_error: print(f"Could not synthesize error message: {synth_error}") assistant_turn["content"] = (assistant_turn.get("content", "") + f" [Error: {e}]").strip() yield AdditionalOutputs( snapshot_history(conversation_history), session_identifier, ) total_latency = time.time() - start_time print(f"Total: {total_latency:.4f}s") print(f"------------------------") async def get_credentials(): return await get_cloudflare_turn_credentials_async(hf_token=os.getenv('HF_TOKEN')) def initialize_session_id(): """Create a new session identifier for syncing backend history.""" return str(uuid.uuid4()) with gr.Blocks(css=custom_css, theme=gr.themes.Soft(primary_hue="orange", secondary_hue="orange")) as demo: gr.HTML("""

1577 Voicebot Demo

""") session_state = gr.State(value=None) session_display = gr.Textbox( label="Session ID", value="", interactive=False, ) with gr.Row(): with gr.Column(scale=1, elem_classes=["phone-column"]): audio = WebRTC( mode="send-receive", modality="audio", track_constraints={ "echoCancellation": True, "noiseSuppression": {"exact": True}, "autoGainControl": {"exact": True} }, rtc_configuration=get_credentials, server_rtc_configuration=get_cloudflare_turn_credentials(ttl=360_000), icon="https://i.pinimg.com/originals/0c/67/5a/0c675a8e1061478d2b7b21b330093444.gif", icon_button_color="#17dbaa", pulse_color="#b0f83b", button_labels={"start": "Call", "stop": "Hang up", "waiting": "Connecting…"}, icon_radius=45, height="650px", width="100%", container=False, elem_id="phone-call-webrtc" ) with gr.Column(): conversation_history = gr.Chatbot( label="Conversation History", type="messages", value=[], height="675px", resizable=True, avatar_images=(None, "https://i.pinimg.com/originals/0c/67/5a/0c675a8e1061478d2b7b21b330093444.gif"), ) gr.DeepLinkButton() demo.load( fn=initialize_session_id, inputs=None, outputs=[session_state], queue=False, ) audio.stream( fn=ReplyOnPause( response, algo_options=AlgoOptions( audio_chunk_duration=1.35, started_talking_threshold=0.35, speech_threshold=0.2 ), model_options=SileroVadOptions( threshold=0.65, min_speech_duration_ms=200, max_speech_duration_s=float("inf"), min_silence_duration_ms=1200, speech_pad_ms=300 ), can_interrupt=False, startup_fn=startup, ), inputs=[audio, conversation_history, session_state], outputs=[audio], concurrency_limit=1000, time_limit=8192 ) def _sync_history_and_session(history, session_identifier): return history, session_identifier, session_identifier audio.on_additional_outputs( _sync_history_and_session, outputs=[conversation_history, session_state, session_display], queue=True, show_progress="hidden" ) demo.queue(default_concurrency_limit=100) demo.launch( debug=True, show_error=True, share=False, server_name="0.0.0.0", server_port=int(os.getenv("PORT", 7860)), )