feat: Enhance audio processing and transcription features

app.py

@@ -8,7 +8,7 @@ import numpy as np
 import sys
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
 from backend.tts import synthesize_text
-from backend.asr import transcribe_audio
+from backend.asr import transcribe_audio, transcribe_typhoon
 from backend.utils import preprocess_audio, is_valid_turn, preprocess_audio_simplified
 from backend.main import stream_chat_response
 import json
@@ -240,9 +240,26 @@ def response(audio: tuple[int, np.ndarray] | None, conversation_history):
         print(f"Audio preprocessing failed: {audio_err}")
         print(f"------------------------")
         return
+    
+    silence_duration_s = 0.2
+
+    # Calculate the number of samples corresponding to the silence duration
+    silence_samples = int(16000 * silence_duration_s)
+    
+    # Create a silent audio segment (an array of zeros)
+    # Ensure the dtype matches your processed audio for compatibility
+    leading_silence = np.zeros(silence_samples, dtype=np.float32)
 
+    # Prepend the silence to the beginning of your processed audio
+    audio_with_padding = np.concatenate([leading_silence, processed_audio])
+    
+    print(f"Added {silence_duration_s}s of silence. New shape: {audio_with_padding.shape}")
+    
+    file_name = "temp.wav"
+    sf.write(file_name, audio_with_padding, sr)
     t0 = time.time()
-    transcription = transcribe_audio( processed_audio)
+    transcription = transcribe_typhoon(file_name)
+    # transcription = transcribe_audio( "debug_processed.wav")
     t_asr = time.time() - t0
     print(f"ASR:        {t_asr:.4f}s")
 
@@ -250,7 +267,6 @@ def response(audio: tuple[int, np.ndarray] | None, conversation_history):
         print("No valid transcription; skipping response generation.")
         print(f"------------------------")
         return
-
     user_turn = {"role": "user", "content": transcription}
     print(f"User: {transcription}")
     if is_valid_turn(user_turn):
@@ -434,13 +450,13 @@ with gr.Blocks(css=custom_css, theme=gr.themes.Soft(primary_hue="orange", second
         fn=ReplyOnPause(
             response,
             algo_options=AlgoOptions(
-                audio_chunk_duration=1.0,
-                started_talking_threshold=0.85,
-                speech_threshold=0.6
+                audio_chunk_duration=1.5,
+                started_talking_threshold=0.35,
+                speech_threshold=0.2
             ),
             model_options=SileroVadOptions(
-                threshold=0.6,
-                min_speech_duration_ms=300,
+                threshold=0.65,
+                min_speech_duration_ms=200,
                 max_speech_duration_s=float("inf"),
                 min_silence_duration_ms=1200,
             ),

backend/asr.py

@@ -170,6 +170,15 @@ def _transcribe_with_gpt(audio_array: np.ndarray) -> str:
 
 _ASR_TYPHOON = _initialize_typhoon_pipeline()
 
+def transcribe_typhoon(path: str) -> str:
+    text = _ASR_TYPHOON.transcribe(path)
+    if text[0].text:
+        return text[0].text
+    else :
+        print(text)
+        return ""
+    
+
 
 def transcribe_audio(audio_array: np.ndarray) -> str:
     """Transcribe user audio with the best available backend based on the current stage."""

backend/main.py

@@ -114,7 +114,7 @@ async def _stream_chat_async(history: List[Dict[str, str]], message: str) -> Asy
         #         limited_conversation = full_conversation[-7:]
         # else:
         #     limited_conversation = full_conversation
-        limited_comversation = full_conversation
+        limited_conversation = full_conversation
         response_generator = llm_analyzer.generate_normal_response(limited_conversation)
 
         async for chunk in response_generator:

backend/systemprompt.py

@@ -90,12 +90,16 @@ Stage 1: Consult
 Stage 2: Buying and get info:
 - This phrase when user comfirm that they want to buy the promotion or product.
 - Ask how many product do they want.
-- After get quantity, call the "summary_order" will return the promotion_id,quantity and total_price. Confirm user again that the info is correct. **Do not tell promotion_id to the user, map to the name of the promotion in the promotion description.
-- After user confirm the order, now ask about personal data for the shipping [full name in thai, shipping address, Customer telephone number]. ask until get all the data. Note that the input is from the asr system so the address can be inaccurate, so u can make assumption(the address must be in Thailand), if not sure, you can ask the user back for the clarification.
+- After get quantity, call the "summary_order" will return the promotion_id,quantity and total_price. Confirm user again that the info is correct. **Do not tell promotion_id to the user, map to the name of the promotion in the promotion description. state the user that "เก็ยเงินปลายทาง" is the only payment method available.
+- After user confirm the order, now ask about personal data for the shipping [name, surname, shipping address, postal code and customer telephone number]. 
+    - Ask step by step slowly start from name then surname then address then postal code then telephone number, Ask until get all the data. Note that the input is from the asr system so the address can be inaccurate, so u can make assumption(the address must be in Thailand), if not sure, you can ask the user back for the clarification.
 - After get all the personal data, confirm user 1 last time that the info is correct. 
 Stage 3: Send order to the backend system
 - After all the info correct, call "purchase_product" to send order to the system
 - if successful, tell user that the order is successful, if not call "call_admin"
+Stage 4: After sales service
+- Thank user for buying the product, and ask them to wait for the product to arrive in 5-7 days.
+- Then ask if they have any question about the product or the order.
 
 ## Notes:
 - Do not give a image or any link to the user.

backend/tools.py

@@ -65,9 +65,13 @@ TOOL_DEFINITIONS =[
                         "type": "integer",
                         "description": "Quantity the user wants to buy.",
                     },
-                    "user_name": {
+                    "name": {
                         "type": "string",
-                        "description": "Customer full name in Thai.",
+                        "description": "Customer name in Thai.",
+                    },
+                    "surname": {
+                        "type": "string",
+                        "description": "Customer surname in Thai.",
                     },
                     "address": {
                         "type": "string",
@@ -78,7 +82,7 @@ TOOL_DEFINITIONS =[
                         "description": "Customer telephone number.",
                     },
                 },
-                "required": ["promotion_id", "quantity", "user_name", "address", "tel"],
+                "required": ["promotion_id", "quantity", "name","surname", "address", "tel"],
             },
         },
     },
@@ -186,7 +190,8 @@ def call_admin(cause: str, chat_history : Dict[str, str]):
 def purchase_product(
     promotion_id: str,
     quantity: int,
-    user_name: str,
+    name: str,
+    surname: str,
     address: str,
     tel: str,
 ) -> Dict[str, Any]:
@@ -197,7 +202,7 @@ def purchase_product(
     payload = {
         "promotion_id": promotion_id,
         "quantity": quantity,
-        "user_name": user_name,
+        "user_name": name + " " + surname,
         "address": address,
         "tel": tel
     }

backend/utils.py

@@ -98,7 +98,7 @@ def audiosegment_to_numpy(audio, target_sample_rate=16000):
     return samples.astype(np.float32)
 
 
-def preprocess_audio(audio, target_channels=1, target_frame_rate=16000):
+def preprocess_audio(audio, target_channels=1, target_sr=16000):
     """
     Ensures the audio is mono, target sample rate, and normalized to [-1, 1].
     
@@ -107,7 +107,12 @@ def preprocess_audio(audio, target_channels=1, target_frame_rate=16000):
     Returns:
         tuple: (target_frame_rate, normalized_audio)
     """
+    target_frame_rate = target_sr
+    
     sample_rate, audio_array = audio
+    #save audio array for debug
+    with open("debug_audio_array.npy", "wb") as f:
+        np.save(f, audio_array)
     print(audio_array)
     print(audio_array[0])
     print(len(audio_array[0]))