import gradio as gr from gradio import wasm_utils from fastrtc import ReplyOnPause, AlgoOptions, SileroVadOptions, AdditionalOutputs, WebRTC, get_cloudflare_turn_credentials_async, get_cloudflare_turn_credentials #get_hf_turn_credentials, import os from dotenv import load_dotenv import time import numpy as np import sys sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) from backend.tts import synthesize_text from backend.asr import transcribe_audio, transcribe_typhoon from backend.utils import preprocess_audio, is_valid_turn, preprocess_audio_simplified from backend.main import stream_chat_response import json from pydub import AudioSegment import ast from backend.utils import get_device if get_device() == "mps": load_dotenv(override=True) phone_waiting_sound = AudioSegment.from_mp3("frontend/phone-ringing-24000.wav") sound_samples = np.array(phone_waiting_sound.get_array_of_samples(), dtype=np.int16) if phone_waiting_sound.channels > 1: sound_samples = sound_samples.reshape((-1, phone_waiting_sound.channels)).mean(axis=1) sound_samples = sound_samples.astype(np.float32) / 32768.0 # Normalize to [-1, def startup(_): yield (phone_waiting_sound.frame_rate, sound_samples) STARTUP_MESSAGE = "สวัสดีค่ะ พลอย 1577Homeshopping ยินดีให้บริการค่ะ" yield from synthesize_text(STARTUP_MESSAGE) time.sleep(1) yield AdditionalOutputs([{"role": "assistant", "content": STARTUP_MESSAGE}]) custom_css = """ /* Overall Gradio page styling: hot pink background */ body { /* background-color: #ff69b4; /* Hot pink */ margin: 0; padding: 0; font-family: sans-serif;} /* Title styling */ h1 { color: #fff; text-shadow: 1px 1px 2px #ff85a2; font-size: 2.5em; margin-bottom: 20px; text-align: center; } /* Style the column holding the telephone interface */ .phone-column { max-width: 350px !important; /* Limit the width of the phone column */ margin: 0 auto; /* Center the column */ border-radius: 20px; background-color: #f9cb9c; /* Lighter pink for telephone interface */ box-shadow: 0 0 15px rgba(0, 0, 0, 0.2); padding: 20px; } /* Conversation history box styling */ #conversation-history-chatbot { background-color: #f9cb9c; /* Lighter pink for conversation history */ border: 1px solid #ccc; border-radius: 10px; padding: 10px; box-shadow: 0 0 15px rgba(0, 0, 0, 0.2); } """ def format_history_for_chatbot(history): """ Formats the conversation history for display in the Gradio Chatbot. It creates readable strings for tool calls and tool results. """ formatted_history = [] if not history: return [] for turn in history: role = turn.get("role") content = turn.get("content") tool_calls = turn.get("tool_calls") if role == "user": formatted_history.append({"role": "user", "content": content}) elif role == "assistant": if tool_calls: # Display a user-friendly message for the tool call id = tool_calls[0]['id'] func_name = tool_calls[0]['function']['name'] func_args = tool_calls[0]['function']['arguments'] display_content = f"{id}{func_name}{func_args}" # display_content = ( # f"**Calling Tool:**\n" # f"```json\n" # f"{{\n" # f' "name": "{func_name}",\n' # f' "arguments": {func_args}\n' # f"}}\n" # f"```" # ) formatted_history.append({"role": "assistant", "content": display_content}) else: # Regular assistant message formatted_history.append({"role": "assistant", "content": content}) elif role == "tool": # Display a user-friendly message for the tool result id = turn.get("tool_call_id") result_content = json.dumps(json.loads(content), indent=2, ensure_ascii=False) display_content = f"{id}{content}" # display_content = ( # f"**Tool Result:**\n" # f"```json\n" # f"{result_content}\n" # f"```" # ) # Represent tool results as if the "assistant" is providing them formatted_history.append({"role": "assistant", "content": display_content}) return formatted_history import re def revert_to_openai_format(formatted_history): """ Converts a history list formatted for the Gradio Chatbot UI back into the standard OpenAI API format. It parses custom string formats for tool calls and tool results. Args: formatted_history (list): A list of message dictionaries as they appear in the Gradio Chatbot component. Returns: list: A list of message dictionaries compliant with the OpenAI API format. """ openai_history = [] # Pre-compile regex patterns for efficiency # Pattern to find a tool call message tool_call_pattern = re.compile( r"(.*?)(.*?)(.*?)", re.DOTALL # Use DOTALL in case arguments contain newlines ) # Pattern to find a tool result message tool_result_pattern = re.compile( r"(.*?)(.*?)", re.DOTALL ) if not formatted_history: return [] for turn in formatted_history: role = turn.get("role") content = turn.get("content") # If content is None, treat it as an empty string for the regex search. if content is None: content = "" if role == "user": openai_history.append(turn) continue if role == "assistant": # Check if this is a formatted tool call tool_call_match = tool_call_pattern.search(content) if tool_call_match: call_id, func_name, func_args_str = tool_call_match.groups() # Reconstruct the original tool_calls structure reverted_turn = { "role": "assistant", "content": None, "tool_calls": [ { "id": call_id, "type": "function", "function": { "name": func_name, "arguments": func_args_str }, } ], } openai_history.append(reverted_turn) continue # Check if this is a formatted tool result (as per your formatter's logic) tool_result_match = tool_result_pattern.search(content) if tool_result_match: tool_call_id, tool_content = tool_result_match.groups() # Reconstruct the original tool message # NOTE: The role must be 'tool' for the API reverted_turn = { "role": "tool", "tool_call_id": tool_call_id, "content": tool_content.strip() # Remove trailing space } openai_history.append(reverted_turn) continue # If no patterns match, it's a regular assistant message if turn.get("content") is not None: openai_history.append(turn) return openai_history def response(audio: tuple[int, np.ndarray] | None, conversation_history): """ Handles user audio input, transcribes it, streams LLM text via backend.main, and synthesizes chunks to audio while updating the conversation history. """ print(f"WebRTC input SR: {audio[0]}") print(f"--- Latency Breakdown ---") # # stage_asr = "normal" #["normal,"gemini"] # print('-----------------------------') # print(f"Initial conver:{conversation_history}") # print('-----------------------------') conversation_history = revert_to_openai_format(conversation_history) # print(f"After convert:{conversation_history}") start_time = time.time() if conversation_history is None: conversation_history = [] previous_history = list(conversation_history) if not audio or audio[1] is None or not np.any(audio[1]): print("No audio input detected; skipping response generation.") print(f"------------------------") return import soundfile as sf sample_rate, audio_array = audio try: sr , processed_audio = preprocess_audio_simplified((sample_rate, audio_array), target_sr=16000) print(sr, processed_audio.dtype, processed_audio.min(), processed_audio.max(), processed_audio.shape) except Exception as audio_err: print(f"Audio preprocessing failed: {audio_err}") print(f"------------------------") return silence_duration_s = 0.2 # Calculate the number of samples corresponding to the silence duration silence_samples = int(16000 * silence_duration_s) # Create a silent audio segment (an array of zeros) # Ensure the dtype matches your processed audio for compatibility leading_silence = np.zeros(silence_samples, dtype=np.float32) # Prepend the silence to the beginning of your processed audio audio_with_padding = np.concatenate([leading_silence, processed_audio]) print(f"Added {silence_duration_s}s of silence. New shape: {audio_with_padding.shape}") file_name = "temp.wav" sf.write(file_name, audio_with_padding, sr) t0 = time.time() transcription = transcribe_typhoon(file_name) # transcription = transcribe_audio( "debug_processed.wav") t_asr = time.time() - t0 print(f"ASR: {t_asr:.4f}s") if not transcription.strip(): print("No valid transcription; skipping response generation.") print(f"------------------------") return user_turn = {"role": "user", "content": transcription} print(f"User: {transcription}") if is_valid_turn(user_turn): conversation_history.append(user_turn) yield AdditionalOutputs(format_history_for_chatbot(conversation_history)) # print("Conversation history:", conversation_history) assistant_turn = {"role": "assistant", "content": ""} conversation_history.append(assistant_turn) # print(previous_history) history_for_stream = [dict(turn) for turn in previous_history if is_valid_turn(turn)] # print(f"history_for_stream{history_for_stream}") text_buffer = "" full_response = "" delimiter_count = 0 n_threshold = 2 max_n_threshold = 2 lang = "th" chunk_count = 0 first_chunk_sent = False start_llm_stream = time.time() try: for chunk in stream_chat_response(history_for_stream, transcription): # print(f"LLM chunk: {text_chunk}") if isinstance(chunk, str): text_chunk = chunk i = 0 while i < len(text_chunk): char = text_chunk[i] text_buffer += char full_response += char assistant_turn["content"] = full_response.strip() is_delimiter = False # if char in {' ', '\n'}: if char == "|": #check the next character is number, not count as delimiter # if i + 1 < len(text_chunk) and (text_chunk[i + 1].isdigit() or text_chunk[i - 1].isdigit()): # is_delimiter = False # else: # print(f"text_buffer before removing delimiter: '{text_buffer}'") # text_buffer = text_buffer[:-1] # Remove the delimiter from the buffer # print(f"text_buffer after removing delimiter: '{text_buffer}'") is_delimiter = True delimiter_count += 1 if i + 1 < len(text_chunk) and text_chunk[i + 1] == 'ๆ': text_buffer += text_chunk[i + 1] full_response += text_chunk[i + 1] i += 1 send_now = False if not first_chunk_sent: if is_delimiter and text_buffer.strip(): send_now = True else: if delimiter_count >= n_threshold and text_buffer.strip(): send_now = True if n_threshold < max_n_threshold: n_threshold += 1 if send_now: buffer_to_send = text_buffer.strip() try: if buffer_to_send and buffer_to_send.endswith('วันที่'): buffer_to_send = buffer_to_send[:-len('วันที่')] if buffer_to_send and first_chunk_sent and buffer_to_send.endswith('ค่ะ'): buffer_to_send = buffer_to_send[:-len('ค่ะ')] except Exception: buffer_to_send = buffer_to_send.replace('ค่ะ', '') if buffer_to_send: chunk_count += 1 if chunk_count == 1: first_llm_chunk_time = time.time() t_llm_first_token = first_llm_chunk_time - start_llm_stream print(f"LLM TTFC: {t_llm_first_token:.4f}s (Time To First Chunk)") yield from synthesize_text(buffer_to_send, lang=lang) first_chunk_sent = True text_buffer = "" delimiter_count = 0 yield AdditionalOutputs(format_history_for_chatbot(conversation_history)) i += 1 elif isinstance(chunk, dict) and "role" in chunk: # print(f"Received tool message for history: {chunk}") if chunk.get("content") is None: chunk["content"] = "" conversation_history.insert(-1, chunk) # Update the chatbot UI to reflect the new history structure yield AdditionalOutputs(format_history_for_chatbot(conversation_history)) if text_buffer.strip(): buffer_to_send = text_buffer.strip() try: if buffer_to_send and buffer_to_send.endswith('วันที่'): buffer_to_send = buffer_to_send[:-len('วันที่')] if buffer_to_send and first_chunk_sent and buffer_to_send.endswith('ค่ะ'): buffer_to_send = buffer_to_send[:-len('ค่ะ')] except Exception: buffer_to_send = buffer_to_send.replace('ค่ะ', '') if buffer_to_send: chunk_count += 1 if chunk_count == 1: first_llm_chunk_time = time.time() t_llm_first_token = first_llm_chunk_time - start_llm_stream print(f"LLM TTFC: {t_llm_first_token:.4f}s (Time To First Chunk)") yield from synthesize_text(buffer_to_send, lang=lang) first_chunk_sent = True text_buffer = "" delimiter_count = 0 yield AdditionalOutputs(format_history_for_chatbot(conversation_history)) except Exception as e: print(f"An error occurred during response generation or synthesis: {e}") error_message = "ขออภัยค่ะ เกิดข้อผิดพลาดบางอย่าง" try: yield from synthesize_text(error_message, lang=lang) except Exception as synth_error: print(f"Could not synthesize error message: {synth_error}") assistant_turn["content"] = (assistant_turn.get("content", "") + f" [Error: {e}]").strip() yield AdditionalOutputs(format_history_for_chatbot(conversation_history)) total_latency = time.time() - start_time print(f"Total: {total_latency:.4f}s") print(f"------------------------") async def get_credentials(): return await get_cloudflare_turn_credentials_async(hf_token=os.getenv('HF_TOKEN')) with gr.Blocks(css=custom_css, theme=gr.themes.Soft(primary_hue="orange", secondary_hue="orange")) as demo: gr.HTML("""

1577 Voicebot Demo

""") with gr.Row(): with gr.Column(scale=1, elem_classes=["phone-column"]): audio = WebRTC( mode="send-receive", modality="audio", track_constraints={ "echoCancellation": True, "noiseSuppression": {"exact": True}, "autoGainControl": {"exact": True} }, rtc_configuration=get_credentials, server_rtc_configuration=get_cloudflare_turn_credentials(ttl=360_000), icon="https://i.pinimg.com/originals/0c/67/5a/0c675a8e1061478d2b7b21b330093444.gif", icon_button_color="#17dbaa", pulse_color="#b0f83b", button_labels={"start": "Call", "stop": "Hang up", "waiting": "Connecting…"}, icon_radius=45, height="650px", width="100%", container=False, elem_id="phone-call-webrtc" ) with gr.Column(): conversation_history = gr.Chatbot( label="Conversation History", type="messages", value=[], height="675px", resizable=True, avatar_images=(None, "https://i.pinimg.com/originals/0c/67/5a/0c675a8e1061478d2b7b21b330093444.gif"), ) gr.DeepLinkButton() audio.stream( fn=ReplyOnPause( response, algo_options=AlgoOptions( audio_chunk_duration=1.35, started_talking_threshold=0.35, speech_threshold=0.2 ), model_options=SileroVadOptions( threshold=0.65, min_speech_duration_ms=200, max_speech_duration_s=float("inf"), min_silence_duration_ms=1200, ), can_interrupt=False, startup_fn=startup, ), inputs=[audio, conversation_history], outputs=[audio], concurrency_limit=1000, time_limit=8192 ) audio.on_additional_outputs( lambda history: history, outputs=[conversation_history], queue=True, show_progress="hidden" ) demo.queue(default_concurrency_limit=10) demo.launch(debug=True, show_error=True, share=True)