""" Ask The Right Question - AI-powered Q&A Assistant Continuous listening mode: Start listening, and questions automatically generate as the talk progresses. """ import os import tempfile import threading import numpy as np import gradio as gr from dotenv import load_dotenv from src.transcription import transcribe_audio from src.context import ConversationContext from src.streaming_agent import generate_questions_sync, format_questions_html, format_activity_log # Load environment variables load_dotenv() # Constants SAMPLE_RATE = 16000 CHUNK_DURATION = 20 # Process audio every 20 seconds (longer = better context for Whisper) OVERLAP_DURATION = 2 # Keep 2 seconds overlap between chunks to prevent word cutoffs MIN_WORDS_FOR_QUESTIONS = 50 # Start generating questions after this many words WORDS_BETWEEN_GENERATIONS = 50 # Generate new questions every N new words MAX_WORDS_CAP = 500 # Stop processing new content after this many words (rate limiting) def get_initial_state(): """Create fresh initial state.""" return { "context": ConversationContext(), "questions": [], "last_word_count": 0, "audio_buffer": np.array([], dtype=np.float32), "activity_log": [], "transcript": "", "questions_html": format_questions_html([]), "log_html": format_activity_log([]), "speaker_name": "", "generation_in_progress": False, # Track if background generation is running "transcription_in_progress": False # Track if background transcription is running } def transcribe_audio_background(audio_data: np.ndarray, sample_rate: int, state: dict): """ Transcribe audio in background thread. Updates state dict directly (shared reference). After transcription, may trigger question generation. """ try: import wave # Save to temp file for Whisper with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f: with wave.open(f.name, 'wb') as wav: wav.setnchannels(1) wav.setsampwidth(2) wav.setframerate(sample_rate) wav.writeframes((audio_data * 32767).astype(np.int16).tobytes()) temp_path = f.name # Transcribe new_text = transcribe_audio(temp_path) os.unlink(temp_path) if new_text.strip(): ctx = state["context"] ctx.add_transcript(new_text) state["transcript"] = ctx.get_full_transcript() current_word_count = len(state["transcript"].split()) words_since_last = current_word_count - state.get("last_word_count", 0) # Add transcription to activity log state["activity_log"].append({"type": "transcribe", "words": current_word_count}) state["log_html"] = format_activity_log(state["activity_log"]) # Check if we should generate questions (and not already generating) should_generate = ( current_word_count >= MIN_WORDS_FOR_QUESTIONS and current_word_count < MAX_WORDS_CAP and words_since_last >= WORDS_BETWEEN_GENERATIONS and not state.get("generation_in_progress", False) ) if should_generate: state["last_word_count"] = current_word_count state["generation_in_progress"] = True # Start question generation in another thread thread = threading.Thread( target=generate_questions_background, args=(state,), daemon=True ) thread.start() except Exception as e: state["activity_log"].append({"type": "error", "msg": f"Transcription error: {str(e)}"}) state["log_html"] = format_activity_log(state["activity_log"]) finally: state["transcription_in_progress"] = False def generate_questions_background(state: dict): """ Generate questions in background thread. Updates state dict directly (shared reference). """ try: ctx = state["context"] speaker = state.get("speaker_name", "") activity_log = state.get("activity_log", []) questions, activity_log, questions_html, log_html = generate_questions_sync( context=ctx, existing_questions=state.get("questions", []), activity_log=activity_log, speaker_name=speaker ) # Update state with results state["questions"] = questions state["questions_html"] = questions_html state["activity_log"] = activity_log state["log_html"] = log_html except Exception as e: state["activity_log"].append({"type": "error", "msg": str(e)}) state["log_html"] = format_activity_log(state["activity_log"]) finally: state["generation_in_progress"] = False def process_streaming_audio( audio_chunk, state: dict ) -> tuple[str, str, str, str, dict]: """ Process streaming audio chunk. Args: audio_chunk: Tuple of (sample_rate, audio_data) from gr.Audio streaming state: Session state Returns: Tuple of (transcript, agent_log, questions, status, state) """ if audio_chunk is None: return ( state.get("transcript", ""), state.get("log_html", format_activity_log([])), state.get("questions_html", format_questions_html([])), '
', state ) sample_rate, audio_data = audio_chunk # Initialize state if needed if "context" not in state: state = get_initial_state() activity_log = state.get("activity_log", []) # Convert to float32 if needed if audio_data.dtype != np.float32: if audio_data.dtype == np.int16: audio_data = audio_data.astype(np.float32) / 32768.0 elif audio_data.dtype == np.int32: audio_data = audio_data.astype(np.float32) / 2147483648.0 else: audio_data = audio_data.astype(np.float32) # Handle stereo if len(audio_data.shape) > 1: audio_data = audio_data.mean(axis=1) # Accumulate audio state["audio_buffer"] = np.concatenate([state.get("audio_buffer", np.array([], dtype=np.float32)), audio_data]) # Calculate status info samples_needed = sample_rate * CHUNK_DURATION buffer_samples = len(state["audio_buffer"]) buffer_pct = int(buffer_samples / samples_needed * 100) word_count = len(state.get("transcript", "").split()) is_transcribing = state.get("transcription_in_progress", False) is_generating = state.get("generation_in_progress", False) # Build status parts status_parts = [f"🎤 Buffer: {buffer_pct}%", f"📝 {word_count} words"] if is_transcribing: status_parts.append("🔄 Transcribing...") if is_generating: status_parts.append("⚙️ Generating...") status = f'' # Check if we have enough audio to process (CHUNK_DURATION seconds) if buffer_samples < samples_needed: return ( state.get("transcript", ""), state.get("log_html", format_activity_log(activity_log)), state.get("questions_html", format_questions_html(state.get("questions", []))), status, state ) # Check if transcription is already running (sequential queue) if is_transcribing: # Keep buffering, don't start new transcription yet return ( state.get("transcript", ""), state.get("log_html", format_activity_log(activity_log)), state.get("questions_html", format_questions_html(state.get("questions", []))), status, state ) # Process the buffered audio with overlap audio_to_process = state["audio_buffer"][:samples_needed].copy() # Copy for thread safety # Keep last OVERLAP_DURATION seconds for next chunk to prevent word cutoffs overlap_samples = sample_rate * OVERLAP_DURATION state["audio_buffer"] = state["audio_buffer"][samples_needed - overlap_samples:] # Start background transcription (non-blocking) state["transcription_in_progress"] = True thread = threading.Thread( target=transcribe_audio_background, args=(audio_to_process, sample_rate, state), daemon=True ) thread.start() # Update status to show transcribing started status_parts = [f"🎤 Buffer: {int(overlap_samples / samples_needed * 100)}%", f"📝 {word_count} words", "🔄 Transcribing..."] if is_generating: status_parts.append("⚙️ Generating...") status = f'' return ( state.get("transcript", ""), state.get("log_html", format_activity_log(activity_log)), state.get("questions_html", format_questions_html(state.get("questions", []))), status, state ) def set_speaker_name(name: str, state: dict) -> dict: """Store speaker name in state.""" if "context" not in state: state = get_initial_state() state["speaker_name"] = name return state def add_manual_context(text: str, state: dict) -> tuple[str, str, str, str, dict]: """Add manually typed context and potentially trigger question generation.""" if not text.strip(): return ( state.get("transcript", ""), state.get("log_html", format_activity_log([])), state.get("questions_html", format_questions_html([])), '', state ) # Initialize state if needed if "context" not in state: state = get_initial_state() ctx = state["context"] activity_log = state.get("activity_log", []) ctx.add_transcript(text) state["transcript"] = ctx.get_full_transcript() current_word_count = len(state["transcript"].split()) # Add to activity log activity_log.append({"type": "transcribe", "words": current_word_count}) state["activity_log"] = activity_log # Check if word cap reached if current_word_count >= MAX_WORDS_CAP: state["log_html"] = format_activity_log(activity_log) return ( state["transcript"], state["log_html"], state.get("questions_html", format_questions_html([])), f'', state ) status = f'' # Generate questions if enough content and not already generating if current_word_count >= MIN_WORDS_FOR_QUESTIONS and not state.get("generation_in_progress", False): state["last_word_count"] = current_word_count state["generation_in_progress"] = True # Start background thread for question generation (non-blocking) thread = threading.Thread( target=generate_questions_background, args=(state,), daemon=True ) thread.start() status = f'' elif state.get("generation_in_progress", False): state["log_html"] = format_activity_log(activity_log) status = f'' else: state["log_html"] = format_activity_log(activity_log) need_more = MIN_WORDS_FOR_QUESTIONS - current_word_count status = f'' return ( state["transcript"], state["log_html"], state.get("questions_html", format_questions_html([])), status, state ) def clear_session(state: dict) -> tuple[str, str, str, str, dict]: """Clear all session data.""" new_state = get_initial_state() return ( "", format_activity_log([]), format_questions_html([]), '', new_state ) # Custom CSS custom_css = """ .header { text-align: center; padding: 15px; background: linear-gradient(135deg, #2d4a6f 0%, #1e3a5f 100%); color: white; border-radius: 10px; margin-bottom: 15px; } .header { position: relative; } .header h1 { margin: 0; font-size: 24px; color: white !important; } .header p { margin: 5px 0 0 0; opacity: 0.9; font-size: 14px; color: white !important; } .header .info-btn { display: inline-block; margin-top: 12px; background: rgba(255,255,255,0.2); border: 1px solid rgba(255,255,255,0.4); color: white; padding: 6px 16px; border-radius: 20px; cursor: pointer; font-size: 13px; font-weight: 500; animation: pulse 2s ease-in-out infinite; box-shadow: 0 0 0 0 rgba(255,255,255,0.4); transition: background 0.2s; } .header .info-btn:hover { background: rgba(255,255,255,0.3); animation: none; } @keyframes pulse { 0% { box-shadow: 0 0 0 0 rgba(255,255,255,0.5); } 50% { box-shadow: 0 0 0 8px rgba(255,255,255,0); } 100% { box-shadow: 0 0 0 0 rgba(255,255,255,0); } } .info-modal { display: none; position: fixed; top: 0; left: 0; right: 0; bottom: 0; background: rgba(0,0,0,0.5); z-index: 1000; justify-content: center; align-items: center; } .info-modal.show { display: flex; } .info-modal-content { background: white; padding: 25px; border-radius: 12px; max-width: 600px; max-height: 80vh; overflow-y: auto; position: relative; margin: 20px; line-height: 1.6; } .info-modal-content h2 { margin-top: 0; color: #667eea; } .info-modal-content h3 { color: #764ba2; margin-top: 20px; } .info-modal-content table { width: 100%; border-collapse: collapse; margin: 15px 0; font-size: 14px; } .info-modal-content th, .info-modal-content td { padding: 8px 10px; text-align: left; border-bottom: 1px solid #eee; } .info-modal-content th { background: #f8f9fa; font-weight: 600; } .info-modal .close-btn { position: absolute; top: 10px; right: 15px; background: none; border: none; font-size: 24px; cursor: pointer; color: #666; } .info-modal .close-btn:hover { color: #333; } .status-bar { background: #f0f9ff; padding: 8px 12px; border-radius: 6px; font-size: 13px; border-left: 3px solid #3b82f6; } """ # Build the Gradio interface with gr.Blocks(css=custom_css, title="Ask The Right Question") as demo: # Session state state = gr.State(get_initial_state()) # Header with info button gr.HTML("""Start listening and questions will generate automatically as the talk progresses