feat: move question generation to background thread and increase audio chunk to 20s

app.py

@@ -7,6 +7,7 @@ generate as the talk progresses.
 
 import os
 import tempfile
+import threading
 import numpy as np
 import gradio as gr
 from dotenv import load_dotenv
@@ -20,7 +21,8 @@ load_dotenv()
 
 # Constants
 SAMPLE_RATE = 16000
-CHUNK_DURATION = 5  # Process audio every 5 seconds
+CHUNK_DURATION = 20  # Process audio every 20 seconds (longer = better context for Whisper)
+OVERLAP_DURATION = 2  # Keep 2 seconds overlap between chunks to prevent word cutoffs
 MIN_WORDS_FOR_QUESTIONS = 50  # Start generating questions after this many words
 WORDS_BETWEEN_GENERATIONS = 50  # Generate new questions every N new words
 MAX_WORDS_CAP = 500  # Stop processing new content after this many words (rate limiting)
@@ -37,10 +39,41 @@ def get_initial_state():
         "transcript": "",
         "questions_html": format_questions_html([]),
         "log_html": format_activity_log([]),
-        "speaker_name": ""
+        "speaker_name": "",
+        "generation_in_progress": False  # Track if background generation is running
     }
 
 
+def generate_questions_background(state: dict):
+    """
+    Generate questions in background thread.
+    Updates state dict directly (shared reference).
+    """
+    try:
+        ctx = state["context"]
+        speaker = state.get("speaker_name", "")
+        activity_log = state.get("activity_log", [])
+
+        questions, activity_log, questions_html, log_html = generate_questions_sync(
+            context=ctx,
+            existing_questions=state.get("questions", []),
+            activity_log=activity_log,
+            speaker_name=speaker
+        )
+
+        # Update state with results
+        state["questions"] = questions
+        state["questions_html"] = questions_html
+        state["activity_log"] = activity_log
+        state["log_html"] = log_html
+
+    except Exception as e:
+        state["activity_log"].append({"type": "error", "msg": str(e)})
+        state["log_html"] = format_activity_log(state["activity_log"])
+    finally:
+        state["generation_in_progress"] = False
+
+
 def process_streaming_audio(
     audio_chunk,
     state: dict
@@ -102,9 +135,11 @@ def process_streaming_audio(
             state
         )
 
-    # Process the buffered audio
+    # Process the buffered audio with overlap
     audio_to_process = state["audio_buffer"][:samples_needed]
-    state["audio_buffer"] = state["audio_buffer"][samples_needed:]
+    # Keep last OVERLAP_DURATION seconds for next chunk to prevent word cutoffs
+    overlap_samples = sample_rate * OVERLAP_DURATION
+    state["audio_buffer"] = state["audio_buffer"][samples_needed - overlap_samples:]
 
     try:
         # Save to temp file for Whisper
@@ -149,33 +184,31 @@ def process_streaming_audio(
             # Check if we should generate questions
             should_generate = (
                 current_word_count >= MIN_WORDS_FOR_QUESTIONS and
-                words_since_last >= WORDS_BETWEEN_GENERATIONS
+                words_since_last >= WORDS_BETWEEN_GENERATIONS and
+                not state.get("generation_in_progress", False)  # Don't start if already generating
             )
 
             if should_generate:
                 state["last_word_count"] = current_word_count
+                state["generation_in_progress"] = True
 
-                # Generate questions
-                speaker = state.get("speaker_name", "")
-                questions, activity_log, questions_html, log_html = generate_questions_sync(
-                    context=ctx,
-                    existing_questions=state.get("questions", []),
-                    activity_log=activity_log,
-                    speaker_name=speaker
+                # Start background thread for question generation (non-blocking)
+                thread = threading.Thread(
+                    target=generate_questions_background,
+                    args=(state,),
+                    daemon=True
                 )
+                thread.start()
 
-                state["questions"] = questions
-                state["questions_html"] = questions_html
-                state["activity_log"] = activity_log
-                state["log_html"] = log_html
-
-                status = f'<div class="status-bar">✨ Generated {len(questions)} questions | {current_word_count} words</div>'
+                status = f'<div class="status-bar">🎤 Listening... {current_word_count} words | ⚙️ Generating questions...</div>'
             else:
                 # Update log HTML
                 state["log_html"] = format_activity_log(activity_log)
 
-                # Show waiting status if not enough words yet
-                if current_word_count < MIN_WORDS_FOR_QUESTIONS:
+                # Show appropriate status
+                if state.get("generation_in_progress", False):
+                    status = f'<div class="status-bar">🎤 Listening... {current_word_count} words | ⚙️ Generating questions...</div>'
+                elif current_word_count < MIN_WORDS_FOR_QUESTIONS:
                     need_more = MIN_WORDS_FOR_QUESTIONS - current_word_count
                     status = f'<div class="status-bar">🎤 Listening... {current_word_count} words (need {need_more} more to start generating)</div>'
 
@@ -256,23 +289,23 @@ def add_manual_context(text: str, state: dict) -> tuple[str, str, str, str, dict
 
     status = f'<div class="status-bar">Added text. Total: {current_word_count} words</div>'
 
-    # Generate questions if enough content
-    if current_word_count >= MIN_WORDS_FOR_QUESTIONS:
-        speaker = state.get("speaker_name", "")
-        questions, activity_log, questions_html, log_html = generate_questions_sync(
-            context=ctx,
-            existing_questions=state.get("questions", []),
-            activity_log=activity_log,
-            speaker_name=speaker
-        )
-
-        state["questions"] = questions
-        state["questions_html"] = questions_html
-        state["activity_log"] = activity_log
-        state["log_html"] = log_html
+    # Generate questions if enough content and not already generating
+    if current_word_count >= MIN_WORDS_FOR_QUESTIONS and not state.get("generation_in_progress", False):
         state["last_word_count"] = current_word_count
+        state["generation_in_progress"] = True
+
+        # Start background thread for question generation (non-blocking)
+        thread = threading.Thread(
+            target=generate_questions_background,
+            args=(state,),
+            daemon=True
+        )
+        thread.start()
 
-        status = f'<div class="status-bar">✨ Generated {len(questions)} questions | {current_word_count} words</div>'
+        status = f'<div class="status-bar">Added text. Total: {current_word_count} words | ⚙️ Generating questions...</div>'
+    elif state.get("generation_in_progress", False):
+        state["log_html"] = format_activity_log(activity_log)
+        status = f'<div class="status-bar">Added text. Total: {current_word_count} words | ⚙️ Already generating...</div>'
     else:
         state["log_html"] = format_activity_log(activity_log)
         need_more = MIN_WORDS_FOR_QUESTIONS - current_word_count
@@ -449,7 +482,7 @@ with gr.Blocks(css=custom_css, title="Ask The Right Question") as demo:
                     streaming=True,
                     label="Click to start listening"
                 )
-                gr.Markdown("*Audio is processed every 5 seconds*")
+                gr.Markdown("*Audio is processed every 20 seconds*")
 
             with gr.Tab("Paste Text"):
                 manual_text = gr.Textbox(