Push latest updates

.gitignore

@@ -1,13 +1,27 @@
+# Local / editor
+.vscode/
+.idea/
+*.swp
+*.swo
+.DS_Store
 
-pyproject.toml
+# Python
 __pycache__/
-.gradio/
-app.py.lprof
-.DS_Store
 .venv/
+.python-version
 .ruff_cache/
 pyrightconfig.json
+pyproject.toml
+uv.lock
+
+# App / env
 .env
-.python-version
+.env.*
+!.env.example
+
+# Gradio / profiling
+.gradio/
+app.py.lprof
+
+# Models / assets (large or local-only)
 *.aicmodel
-uv.lock

README.md

@@ -4,7 +4,7 @@ emoji: 🌍
 colorFrom: red
 colorTo: pink
 sdk: gradio
-sdk_version: 5.47.0
+sdk_version: "6.0"
 app_file: app.py
 pinned: false
 short_description: Try out ai-coustics speech enhancement models.

app.py

@@ -1,9 +1,16 @@
+import os
+import threading
+import time
 import gradio as gr
-from constants import STREAM_EVERY
-from hf_dataset_utils import ALL_FILES
+from pathlib import Path
+
+from constants import STREAM_EVERY, WARMUP_SECONDS, APP_TMP_DIR
+from hf_dataset_utils import ALL_FILES, get_transcript
 
 from stream_pipeline import (
     clear_ui,
+    get_live_transcripts,
+    on_stop_recording,
     set_stt_streamer,
     stop_online_backend,
     transcribe_stream,
@@ -12,17 +19,140 @@ from stream_pipeline import (
 from offline_pipeline import (
     load_file_from_dataset,
     load_local_file,
-    denoise_audio,
-    retrieve_audio_information,
+    run_offline_pipeline_streaming,
 )
+from utils import spec_image
 from clean_up import purge_tmp_directory, cleanup_previous_run
 
+# Active light HTML: whole container is the light (gray = warming up, red = ready)
+ACTIVE_LIGHT_GRAY = (
+    '<div class="active-light active-light--off" title="Warming up" style="display:flex;justify-content:center;align-items:center;width:100%;height:100%;min-height:3rem;">'
+    '<span class="active-light__label">Warming up...</span></div>'
+)
+ACTIVE_LIGHT_RED = (
+    '<div class="active-light active-light--on" title="Ready" style="display:flex;justify-content:center;align-items:center;width:100%;height:100%;min-height:3rem;">'
+    '<span class="active-light__label">Ready!</span></div>'
+)
+WARMUP_TICKS = max(1, int(WARMUP_SECONDS * 2))  # timer ticks every 0.5s
+
+
+def warmup_tick(on_stream_tab, warmup_elapsed, _current_html):
+    if not on_stream_tab:
+        return 0, ACTIVE_LIGHT_GRAY, gr.update(active=True)
+    if warmup_elapsed >= WARMUP_TICKS:
+        return warmup_elapsed, ACTIVE_LIGHT_RED, gr.update(active=False)
+    return warmup_elapsed + 1, ACTIVE_LIGHT_GRAY, gr.update(active=True)
+
+
+def process_with_live_transcript(
+    input_array,
+    enhancement_level,
+    sample_stem,
+    stt_model,
+    last_sample_stem,
+):
+    """Generator that runs the offline pipeline in real time (chunked): enhanced audio and
+    both transcripts stream from the first chunk so playback and transcription start immediately."""
+    progress_state = {}
+    result_holder = {}
+
+    def worker():
+        try:
+            result_holder["result"] = run_offline_pipeline_streaming(
+                input_array,
+                enhancement_level,
+                sample_stem,
+                stt_model,
+                progress_state,
+            )
+        except Exception as e:
+            result_holder["error"] = e
+
+    # 1) First yield: ground truth + input spectrogram only (no audio, no enhanced spec, no transcripts yet)
+    cleanup_out = cleanup_previous_run(last_sample_stem)
+    noisy_spec_path = f"{APP_TMP_DIR}/{sample_stem}_noisy_spectrogram.png"
+    if input_array is not None:
+        try:
+            spec_image(input_array).save(noisy_spec_path)
+        except Exception:
+            noisy_spec_path = None
+    else:
+        noisy_spec_path = None
+    try:
+        original_transcript = get_transcript(sample_stem)
+    except Exception:
+        original_transcript = "Unavailable"
+
+    yield (
+        gr.update(visible=True),
+        None,                # enhanced_audio: set only in final yield (smooth playback)
+        gr.update(value=None),  # enhanced_image: clear until step 3 (last)
+        gr.update(value=noisy_spec_path),  # noisy_image: input spectrogram (step 1)
+        original_transcript,
+        "",
+        "",
+        sample_stem,
+        None,
+        "",
+    )
+    # Let the UI render step 1 before we flood with polling updates
+    time.sleep(0.2)
+
+    thread = threading.Thread(target=worker, daemon=True)
+    thread.start()
+
+    poll_interval = 0.05
+    while "result" not in result_holder and "error" not in result_holder:
+        time.sleep(poll_interval)
+        # 2) Realtime: stream transcripts only; audio set in final yield for smooth playback
+        yield (
+            gr.update(visible=True),
+            gr.update(),  # enhanced_audio: set only in final yield, then autoplay
+            gr.update(),  # enhanced_image: reveal only in step 3 (final yield)
+            gr.update(),  # noisy_image already set in step 1
+            gr.update(),  # original_transcript unchanged
+            gr.update(value=progress_state.get("noisy", "")),
+            gr.update(value=progress_state.get("enhanced", "")),
+            gr.update(),
+            gr.update(),
+            gr.update(),
+        )
+
+    if "error" in result_holder:
+        raise result_holder["error"]
+
+    (
+        enhanced_spec_path,
+        enhanced_transcript,
+        noisy_transcript_with_wer,
+        enhanced_audio,
+        last_stem,
+        enhanced_array,
+        precomputed_noisy,
+    ) = result_holder["result"]
 
+    # 3) Last: reveal enhanced spectrogram (and final audio/transcripts)
+    yield (
+        gr.update(visible=True),
+        enhanced_audio,
+        enhanced_spec_path,   # enhanced_image: show only now
+        noisy_spec_path,
+        original_transcript,
+        noisy_transcript_with_wer,
+        enhanced_transcript,
+        last_stem,
+        enhanced_array,
+        precomputed_noisy,
+    )
+
+
+_CSS_DIR = Path(__file__).resolve().parent / "assets"
 with gr.Blocks() as demo:
     sample_stem = gr.State("")
     last_sample_stem = gr.State("")
     input_array = gr.State()
     enhanced_array = gr.State()
+    precomputed_noisy_transcript = gr.State("")
 
     gr.HTML(
         '<a href="https://ai-coustics.com/" target="_blank">'
@@ -39,12 +169,12 @@ with gr.Blocks() as demo:
         value="Soniox STT-RT v3",
         interactive=True,
     )
-    set_stt_streamer("Soniox STT-RT v3")
+    # Streamers are created on first start_recording to avoid Soniox "No audio received" timeout
     enhancement_level = gr.Slider(
         minimum=0,
         maximum=100,
         step=1,
-        value=100,
+        value=80,
         label="Enhancement level (%)",
         scale=2,
     )
@@ -54,38 +184,73 @@ with gr.Blocks() as demo:
             gr.Markdown(open("docs/online.md", "r", encoding="utf-8").read())
             with gr.Group(elem_classes="panel"):
                 stream_state = gr.State(None)
-                audio_stream = gr.Audio(
-                    sources=["microphone"], streaming=True, elem_id="audio_stream"
+                on_stream_tab = gr.State(True)  # True on load: stream tab is the default first tab
+                warmup_elapsed = gr.State(0)
+                input_gain_db = gr.Slider(
+                    minimum=0,
+                    maximum=20,
+                    step=0.5,
+                    value=0,
+                    label="Input gain (dB)",
                 )
+                with gr.Row(elem_classes="stream-row"):
+                    with gr.Column(scale=4, min_width=200):
+                        audio_stream = gr.Audio(
+                            sources=["microphone"], streaming=True, elem_id="audio_stream"
+                        )
+                    with gr.Column(scale=1, min_width=120, elem_classes="active-light-column"):
+                        active_light = gr.HTML(value=ACTIVE_LIGHT_GRAY)
                 with gr.Group(elem_classes="panel"):
                     with gr.Column(scale=5, min_width=320):
                         enhanced_text = gr.Textbox(
-                            label="Enhanced Transcribed Text", lines=6
+                            label="Enhanced Transcribed Text", lines=6, autoscroll=False
                         )
                     with gr.Column(scale=5, min_width=320):
-                        raw_text = gr.Textbox(label="Raw Transcribed Text", lines=6)
+                        raw_text = gr.Textbox(
+                            label="Raw Transcribed Text", lines=6, autoscroll=False
+                        )
+                # Poll transcript globals so interim results show live (streamers update in background)
+                transcript_timer = gr.Timer(0.1, active=True)
+                transcript_timer.tick(
+                    get_live_transcripts,
+                    inputs=None,
+                    outputs=[enhanced_text, raw_text],
+                    show_progress="hidden",
+                )
+                warmup_timer = gr.Timer(0.5, active=True)
+                warmup_timer.tick(
+                    warmup_tick,
+                    inputs=[on_stream_tab, warmup_elapsed, active_light],
+                    outputs=[warmup_elapsed, active_light, warmup_timer],
+                    show_progress="hidden",
+                )
 
         with gr.Tab("Dataset: Dawn Chorus") as dataset_tab:
             with gr.Row():
                 gr.Markdown(open("docs/dawn_chorus.md", "r", encoding="utf-8").read())
-            dataset_dropdown = gr.Dropdown(choices=ALL_FILES, value=None)
+            dataset_dropdown = gr.Dropdown(
+                choices=ALL_FILES, value="en_00412_i_h_36", label="Sample"
+            )
             audio_file_from_dataset = gr.Audio(
-                type="filepath", interactive=False, show_download_button=True
+                type="filepath", interactive=False, buttons=["download"], autoplay=True
             )
 
         with gr.Tab("Upload local file") as upload_tab:
             with gr.Row():
                 gr.Markdown(open("docs/local_file.md", "r", encoding="utf-8").read())
             audio_file_upload = gr.Audio(
-                type="filepath", sources=["upload"], show_download_button=True
+                type="filepath", sources=["upload"], buttons=["download"], autoplay=True
             )
 
-        enhance_btn = gr.Button("Enhance", scale=2, visible=False)
+        enhance_btn = gr.Button("Enhance with Quail Voice Focus 2.0", scale=2, visible=False)
 
     with gr.Group(elem_classes="panel results-card", visible=False) as results_card:
         result_title = gr.Markdown("")
         enhanced_audio = gr.Audio(
-            type="numpy", interactive=False, show_download_button=True
+            type="numpy",
+            interactive=False,
+            buttons=["download"],
+            autoplay=True,
         )
 
         with gr.Row(equal_height=True, elem_classes="results-row"):
@@ -111,32 +276,56 @@ with gr.Blocks() as demo:
     # ------------------------------------------------------
     # GLOBAL EVENTS (TAB CHANGES, MODEL CHANGES)
     # ------------------------------------------------------
+    DEFAULT_DATASET_SAMPLE = "en_00412_i_h_36"
+
+    def load_dataset_sample_on_tab_visit(dropdown_value):
+        """Load the selected sample when visiting the Dataset tab; use default if dropdown is empty."""
+        sample_id = dropdown_value or DEFAULT_DATASET_SAMPLE
+        audio_path, arr, stem = load_file_from_dataset(sample_id)
+        return sample_id, audio_path, arr, stem
+
     stream_tab.select(
         lambda: (
             gr.update(visible=False),
             gr.update(visible=False),
-            gr.update(visible=True, interactive=True, streaming=True),
+            gr.update(visible=True, interactive=True, streaming=True, sources=["microphone"]),
+            True,
+            0,
+            ACTIVE_LIGHT_GRAY,
+            gr.update(active=True),
         ),
         inputs=None,
-        outputs=[results_card, enhance_btn, audio_stream],
+        outputs=[results_card, enhance_btn, audio_stream, on_stream_tab, warmup_elapsed, active_light, warmup_timer],
     )
 
     upload_tab.select(
-        stop_online_backend,
+        lambda: (
+            *stop_online_backend(),
+            False,
+            ACTIVE_LIGHT_GRAY,
+        ),
         inputs=None,
-        outputs=[stream_state, enhanced_text, raw_text, audio_stream],
+        outputs=[stream_state, enhanced_text, raw_text, audio_stream, on_stream_tab, active_light],
     ).then(
         lambda: gr.update(visible=True),
         outputs=enhance_btn,
     )
 
     dataset_tab.select(
-        stop_online_backend,
+        lambda: (
+            *stop_online_backend(),
+            False,
+            ACTIVE_LIGHT_GRAY,
+        ),
         inputs=None,
-        outputs=[stream_state, enhanced_text, raw_text, audio_stream],
+        outputs=[stream_state, enhanced_text, raw_text, audio_stream, on_stream_tab, active_light],
     ).then(
         lambda: gr.update(visible=True),
         outputs=enhance_btn,
+    ).then(
+        load_dataset_sample_on_tab_visit,
+        inputs=[dataset_dropdown],
+        outputs=[dataset_dropdown, audio_file_from_dataset, input_array, sample_stem],
     )
 
     stt_model.change(
@@ -157,7 +346,7 @@ with gr.Blocks() as demo:
 
     audio_stream.stream(
         fn=transcribe_stream,
-        inputs=[stream_state, audio_stream, enhancement_level],
+        inputs=[stream_state, audio_stream, enhancement_level, input_gain_db],
         outputs=[stream_state, enhanced_text, raw_text],
         stream_every=STREAM_EVERY,
         time_limit=60 * 2,
@@ -165,7 +354,7 @@ with gr.Blocks() as demo:
     )
 
     audio_stream.stop_recording(
-        shutdown_streamers,
+        on_stop_recording,
     )
 
     audio_stream.start_recording(
@@ -199,40 +388,33 @@ with gr.Blocks() as demo:
         inputs=None,
         outputs=results_card,
     ).then(
-        load_local_file, inputs=[audio_file_upload], outputs=[input_array, sample_stem]
+        load_local_file,
+        inputs=[audio_file_upload],
+        outputs=[input_array, sample_stem]
     )
 
-    # Enhancement button triggers the audio processing and and shows results when done.
+    # Enhancement button: run pipeline with live transcript progress (dataset + local file modes).
     enhance_btn.click(
-        cleanup_previous_run,
-        inputs=[last_sample_stem],
+        process_with_live_transcript,
+        inputs=[input_array, enhancement_level, sample_stem, stt_model, last_sample_stem],
         outputs=[
+            results_card,
+            enhanced_audio,
             enhanced_image,
             noisy_image,
-            enhanced_transcript,
             original_transcript,
             noisy_transcript,
-        ],
-    ).success(
-        denoise_audio,
-        inputs=[input_array, enhancement_level],
-        outputs=[enhanced_array, enhanced_audio],
-    ).success(
-        lambda: gr.update(visible=True),
-        inputs=None,
-        outputs=results_card,
-    ).success(
-        retrieve_audio_information,
-        inputs=[input_array, enhanced_array, sample_stem, stt_model],
-        outputs=[
-            enhanced_image,
-            noisy_image,
             enhanced_transcript,
-            original_transcript,
-            noisy_transcript,
             last_sample_stem,
+            enhanced_array,
+            precomputed_noisy_transcript,
         ],
     )
 
-purge_tmp_directory(max_age_minutes=0, skip_substrings=[])
-demo.launch(allowed_paths=["/tmp", "/"])
+os.makedirs(APP_TMP_DIR, exist_ok=True)
+purge_tmp_directory(max_age_minutes=0, tmp_dir=APP_TMP_DIR)
+demo.queue()
+demo.launch(
+    css=(_CSS_DIR / "active_light.css").read_text(encoding="utf-8"),
+    allowed_paths=[APP_TMP_DIR, "/tmp", "/"],
+)

assets/active_light.css

@@ -0,0 +1,68 @@
+/* Stream row: stretch columns to match audio height */
+.stream-row {
+  align-items: stretch !important;
+}
+
+/* Active light column: flex container, fill height */
+.stream-row > div:last-child {
+  display: flex !important;
+  align-items: stretch !important;
+  min-height: 100%;
+}
+
+/* Gradio block wrapper: fill and flex so child can stretch */
+.stream-row > div:last-child > div {
+  display: flex !important;
+  width: 100% !important;
+  min-height: 100% !important;
+  height: 100% !important;
+  box-sizing: border-box;
+}
+
+/* All divs in light column: fill so panel stretches */
+.stream-row > div:last-child div {
+  width: 100% !important;
+  min-height: 100% !important;
+  height: 100% !important;
+  box-sizing: border-box;
+}
+
+/* Active light panel: fill container, base styles */
+.active-light {
+  width: 100% !important;
+  min-height: 100% !important;
+  height: 100% !important;
+  box-sizing: border-box;
+  border-radius: 8px;
+  border: 1px solid var(--border-color-primary);
+  transition: background 0.2s, box-shadow 0.2s;
+  padding: 0.5rem;
+}
+
+/* Warming up state */
+.active-light--off {
+  background: #555;
+  box-shadow: inset 0 2px 8px rgba(0, 0, 0, 0.4);
+}
+
+/* Ready state */
+.active-light--on {
+  background: #c00;
+  box-shadow:
+    0 0 16px rgba(220, 0, 0, 0.5),
+    inset 0 0 12px rgba(255, 80, 80, 0.3);
+}
+
+/* Label text */
+.active-light__label {
+  font-size: 12px;
+  font-weight: 500;
+}
+
+.active-light--off .active-light__label {
+  color: rgba(255, 255, 255, 0.85);
+}
+
+.active-light--on .active-light__label {
+  color: #fff;
+}

clean_up.py

@@ -1,24 +1,25 @@
 import os
 import time
 from loguru import logger
-from constants import MINUTES_KEEP
+from constants import MINUTES_KEEP, APP_TMP_DIR
 import gradio as gr
 
 
 def purge_tmp_directory(
     max_age_minutes: int = MINUTES_KEEP,
     skip_substrings: list[str] | None = None,
-    tmp_dir: str = "/tmp",
+    tmp_dir: str = APP_TMP_DIR,
 ) -> tuple[int, int]:
     """
     Delete files in `tmp_dir` older than `max_age_minutes`, optionally skipping paths
-    that contain any substring in `skip_substrings`.
+    that contain any substring in `skip_substrings`. Use APP_TMP_DIR so we only
+    touch our own temp files (spectrograms, audio, etc.), not other apps' files in /tmp.
 
     Args:
         max_age_minutes: Files older than this many minutes will be deleted.
         skip_substrings: If provided, any file path containing one of these substrings
             will NOT be deleted.
-        tmp_dir: Directory to scan (default: /tmp).
+        tmp_dir: Directory to scan (default: APP_TMP_DIR).
 
     Returns:
         (removed, skipped): Number of files removed and skipped.
@@ -28,7 +29,8 @@ def purge_tmp_directory(
     removed = 0
 
     if not os.path.exists(tmp_dir):
-        logger.info(f"Temp directory does not exist: {tmp_dir}")
+        os.makedirs(tmp_dir, exist_ok=True)
+        logger.info(f"Temp directory created: {tmp_dir}")
         return removed, skipped
 
     now = time.time()
@@ -64,7 +66,7 @@ def purge_tmp_directory(
     return removed, skipped
 
 
-def remove_files_related_to(stem: str, tmp_dir: str = "/tmp") -> int:
+def remove_files_related_to(stem: str, tmp_dir: str = APP_TMP_DIR) -> int:
     deleted = 0
 
     if not stem:
@@ -89,7 +91,7 @@ def remove_files_related_to(stem: str, tmp_dir: str = "/tmp") -> int:
 
 def cleanup_previous_run(
     sample_stem: str,
-    tmp_dir: str = "/tmp",
+    tmp_dir: str = APP_TMP_DIR,
     max_age_minutes: int = MINUTES_KEEP,
 ) -> tuple[None, None, str, str, str]:
     gr.Info("Processing started. This may take a moment. Please do not refresh or close the window.")

constants.py

@@ -10,6 +10,9 @@ MODEL_ID: Final = "quail-vf-2.0-l-16khz"
 
 MINUTES_KEEP: Final = 60
 
+# All app temp files (spectrograms, audio, etc.) go here so we only purge our own files.
+APP_TMP_DIR: Final = "/tmp/voicefocus"
+
 DATASET_NAME: Final = "ai-coustics/dawn_chorus_en"
 DEFAULT_SPLIT: Final = "eval"
 MIX_DIR: Final = "mix"
@@ -18,6 +21,7 @@ TRANS_DIR: Final = "transcripts"
 
 DEFAULT_SR: Final = 16000
 STREAM_EVERY: Final = 0.2
+WARMUP_SECONDS: Final = 2  # seconds before "recording ready" light turns on
 
 STREAMER_CLASSES: Final = {
     "Deepgram Nova-3 RT": DeepgramStreamer,

docs/dawn_chorus.md

@@ -1 +1 @@
-Choose a sample from our open-source [Dawn Chorus evaluation dataset](https://huggingface.co/datasets/ai-coustics/dawn_chorus_en). This dataset includes audio with varying levels of background voice activity, ideal for testing the model performance. To learn more about the dataset, visit *placeholder to upcoming website/blog about dataset*.
+Choose a sample from our open-source [Dawn Chorus evaluation dataset](https://huggingface.co/datasets/ai-coustics/dawn_chorus_en). This dataset includes audio with varying levels of background voice activity, ideal for testing the model performance.

offline_pipeline.py

@@ -1,28 +1,42 @@
 import os
+from concurrent.futures import ThreadPoolExecutor
+from typing import Optional
 
 import gradio as gr
 import librosa
 from sdk import SDKWrapper
 from utils import spec_image, compute_wer, transcribe_audio, to_gradio_audio
 from hf_dataset_utils import get_audio, get_transcript
-from constants import DEFAULT_SR
-import numpy as np  
+from constants import DEFAULT_SR, APP_TMP_DIR, STREAMER_CLASSES
+import numpy as np
+
 
 def retrieve_audio_information(
     original_array: np.ndarray,
     enhanced_array: np.ndarray,
     sample_id: str,
     stt_model: str,
+    noisy_transcript: str,
+    progress_state: Optional[dict] = None,
 ) -> tuple[str, str, str, str, str, str]:
+    """Build spectrograms, transcribe enhanced audio, and compute WER. Caller must supply
+    noisy_transcript (transcription of original_array) so STT on the original is never run here.
+    If progress_state is provided, progress_state['enhanced'] is updated with partial transcript as enhanced STT streams."""
     if original_array is None or enhanced_array is None:
         raise ValueError("Audio arrays are not available.")
-    noisy_spec_path = f"/tmp/{sample_id}_noisy_spectrogram.png"
-    enhanced_spec_path = f"/tmp/{sample_id}_enhanced_spectrogram.png"
+    noisy_spec_path = f"{APP_TMP_DIR}/{sample_id}_noisy_spectrogram.png"
+    enhanced_spec_path = f"{APP_TMP_DIR}/{sample_id}_enhanced_spectrogram.png"
     spec_image(original_array).save(noisy_spec_path)
     spec_image(enhanced_array).save(enhanced_spec_path)
-    enhanced_transcript = transcribe_audio(enhanced_array, DEFAULT_SR, stt_model, stream_name=f"{sample_id}_enhanced")
-    noisy_transcript = transcribe_audio(original_array, DEFAULT_SR, stt_model, stream_name=f"{sample_id}_noisy")
-    try: 
+    on_enhanced = (lambda t: progress_state.__setitem__("enhanced", t)) if progress_state is not None else None
+    enhanced_transcript = transcribe_audio(
+        enhanced_array,
+        DEFAULT_SR,
+        stt_model,
+        stream_name=f"{sample_id}_enhanced",
+        on_update=on_enhanced,
+    )
+    try:
         original_transcript = get_transcript(sample_id)
         wer_enhanced = compute_wer(original_transcript, enhanced_transcript)
         wer_noisy = compute_wer(original_transcript, noisy_transcript)
@@ -33,10 +47,64 @@ def retrieve_audio_information(
     return enhanced_spec_path, noisy_spec_path, enhanced_transcript, original_transcript, noisy_transcript, sample_id
 
 
+def denoise_and_transcribe_noisy_parallel(
+    sample_16k: np.ndarray,
+    enhancement_level: float,
+    sample_id: str,
+    stt_model: str,
+    progress_state: Optional[dict] = None,
+) -> tuple[np.ndarray | None, tuple[int, np.ndarray] | None, str]:
+    """Run denoising and noisy transcription in parallel. Returns (enhanced_array, gradio_audio, noisy_transcript).
+    If progress_state is provided, progress_state['noisy'] is updated with partial transcript as noisy STT streams."""
+    if sample_16k is None:
+        raise ValueError("No audio to enhance. Please upload a file first.")
+    on_noisy = (lambda t: progress_state.__setitem__("noisy", t)) if progress_state is not None else None
+    with ThreadPoolExecutor(max_workers=2) as executor:
+        future_denoise = executor.submit(
+            _denoise_audio_impl, sample_16k, float(enhancement_level) / 100.0
+        )
+        future_noisy = executor.submit(
+            transcribe_audio,
+            sample_16k,
+            DEFAULT_SR,
+            stt_model,
+            stream_name=f"{sample_id}_noisy",
+            on_update=on_noisy,
+        )
+        enhanced_array, gradio_enhanced_audio = future_denoise.result()
+        noisy_transcript = future_noisy.result()
+    return enhanced_array, gradio_enhanced_audio, noisy_transcript
+
+
+def _denoise_audio_impl(
+    sample_16k: np.ndarray, enhancement_level_frac: float
+) -> tuple[np.ndarray, tuple[int, np.ndarray]]:
+    """Run the enhancer on a single array and return both the array and Gradio-ready audio.
+
+    Used by denoise_and_transcribe_noisy_parallel so the denoise step can run in a
+    thread without Gradio UI calls. Expects enhancement_level_frac in [0, 1] (not percent).
+    Returns (enhanced_array, (sample_rate, numpy_array)) for Gradio Audio.
+    """
+    sdk = SDKWrapper()
+    sdk.init_processor(
+        sample_rate=DEFAULT_SR,
+        enhancement_level=enhancement_level_frac,
+    )
+    enhanced_array = sdk.process_sync(sample_16k)
+    gradio_enhanced_audio = to_gradio_audio(enhanced_array, DEFAULT_SR)
+    return enhanced_array, gradio_enhanced_audio
+
+
 def denoise_audio(
     sample_16k: np.ndarray,
     enhancement_level: float = 50.0,
 ) -> tuple[np.ndarray | None , tuple[int, np.ndarray]| None]:
+    """Enhance-only entry point: run the SDK on the given audio and return enhanced array + Gradio audio.
+
+    The main app uses denoise_and_transcribe_noisy_parallel (denoise + noisy STT in parallel)
+    instead. This function remains for backward compatibility, scripts, or any caller that
+    needs only enhancement without transcription (e.g. backup flows, tests).
+    """
     if sample_16k is None:
         raise ValueError("No audio to enhance. Please upload a file first.")
     try:
@@ -50,6 +118,207 @@ def denoise_audio(
     return enhanced_array, gradio_enhanced_audio
 
 
+def run_offline_pipeline_ordered(
+    sample_16k: np.ndarray,
+    enhancement_level: float,
+    sample_id: str,
+    stt_model: str,
+    progress_state: dict,
+) -> tuple[str, str, str, tuple[int, np.ndarray], str, np.ndarray, str]:
+    """Run pipeline in UI order: 1) Denoise only. 2) When done, set progress_state['enhanced_spec_path']
+    and progress_state['enhanced_audio']. 3) Run noisy STT and enhanced STT in parallel (both stream via
+    progress_state['noisy'] and progress_state['enhanced']). 4) Return final transcripts with WER.
+
+    Returns: (enhanced_spec_path, enhanced_transcript, noisy_transcript_with_wer, enhanced_audio,
+              last_stem, enhanced_array, precomputed_noisy).
+    """
+    if sample_16k is None:
+        raise ValueError("No audio to enhance. Please upload a file first.")
+    # 1) Denoise only
+    enhanced_array, gradio_enhanced_audio = _denoise_audio_impl(
+        sample_16k, float(enhancement_level) / 100.0
+    )
+    # 2) As soon as enhanced audio is ready: build enhanced spectrogram and expose to UI
+    enhanced_spec_path = f"{APP_TMP_DIR}/{sample_id}_enhanced_spectrogram.png"
+    spec_image(enhanced_array).save(enhanced_spec_path)
+    progress_state["enhanced_spec_path"] = enhanced_spec_path
+    progress_state["enhanced_audio"] = gradio_enhanced_audio
+    # 3) Noisy and enhanced transcription starting both at the same time (parallel, both with on_update).
+    # Sync so the UI shows both boxes updating together: only write to noisy/enhanced once both have sent at least one update.
+    progress_state["noisy_pending"] = ""
+    progress_state["enhanced_pending"] = ""
+    progress_state["noisy_has_sent"] = False
+    progress_state["enhanced_has_sent"] = False
+
+    def _flush_both():
+        if progress_state.get("noisy_has_sent") and progress_state.get("enhanced_has_sent"):
+            progress_state["noisy"] = progress_state.get("noisy_pending", "")
+            progress_state["enhanced"] = progress_state.get("enhanced_pending", "")
+
+    def on_noisy(t: str):
+        progress_state["noisy_pending"] = t
+        progress_state["noisy_has_sent"] = True
+        _flush_both()
+
+    def on_enhanced(t: str):
+        progress_state["enhanced_pending"] = t
+        progress_state["enhanced_has_sent"] = True
+        _flush_both()
+
+    with ThreadPoolExecutor(max_workers=2) as executor:
+        future_noisy = executor.submit(
+            transcribe_audio,
+            sample_16k,
+            DEFAULT_SR,
+            stt_model,
+            stream_name=f"{sample_id}_noisy",
+            on_update=on_noisy,
+        )
+        future_enhanced = executor.submit(
+            transcribe_audio,
+            enhanced_array,
+            DEFAULT_SR,
+            stt_model,
+            stream_name=f"{sample_id}_enhanced",
+            on_update=on_enhanced,
+        )
+        noisy_transcript = future_noisy.result()
+        enhanced_transcript = future_enhanced.result()
+    # 4) WER and final strings
+    precomputed_noisy = noisy_transcript
+    try:
+        original_transcript = get_transcript(sample_id)
+        wer_enhanced = compute_wer(original_transcript, enhanced_transcript)
+        wer_noisy = compute_wer(original_transcript, noisy_transcript)
+        enhanced_transcript += f" (WER: {wer_enhanced:.2f})"
+        noisy_transcript += f" (WER: {wer_noisy:.2f})"
+    except Exception:
+        original_transcript = "Unavailable"
+    return (
+        enhanced_spec_path,
+        enhanced_transcript,
+        noisy_transcript,
+        gradio_enhanced_audio,
+        sample_id,
+        enhanced_array,
+        precomputed_noisy,
+    )
+
+
+def _close_stt_stream(streamer) -> None:
+    """Signal end-of-stream; streamer type may be Soniox (close_stream) or Deepgram (close)."""
+    if hasattr(streamer, "close_stream"):
+        streamer.close_stream()
+    else:
+        streamer.close()
+
+
+def run_offline_pipeline_streaming(
+    sample_16k: np.ndarray,
+    enhancement_level: float,
+    sample_id: str,
+    stt_model: str,
+    progress_state: dict,
+) -> tuple[str, str, str, tuple[int, np.ndarray], str, np.ndarray, str]:
+    """Run enhancement and both STTs in real time by processing in chunks. Transcripts stream
+    via progress_state['noisy'] and progress_state['enhanced']. Enhanced audio is returned
+    only at the end; the app plays it automatically when processing is complete.
+    Returns same tuple as run_offline_pipeline_ordered."""
+    if sample_16k is None:
+        raise ValueError("No audio to enhance. Please upload a file first.")
+    sample_16k = np.asarray(sample_16k, dtype=np.float32).flatten()
+
+    sdk = SDKWrapper()
+    sdk.init_processor(
+        sample_rate=DEFAULT_SR,
+        enhancement_level=float(enhancement_level) / 100.0,
+    )
+    chunk_size = sdk.num_frames
+
+    # Sync transcript callbacks so both boxes update together
+    progress_state["noisy_pending"] = ""
+    progress_state["enhanced_pending"] = ""
+    progress_state["noisy_has_sent"] = False
+    progress_state["enhanced_has_sent"] = False
+
+    def _flush_both():
+        if progress_state.get("noisy_has_sent") and progress_state.get("enhanced_has_sent"):
+            progress_state["noisy"] = progress_state.get("noisy_pending", "")
+            progress_state["enhanced"] = progress_state.get("enhanced_pending", "")
+
+    def on_noisy(t: str):
+        progress_state["noisy_pending"] = t
+        progress_state["noisy_has_sent"] = True
+        _flush_both()
+
+    def on_enhanced(t: str):
+        progress_state["enhanced_pending"] = t
+        progress_state["enhanced_has_sent"] = True
+        _flush_both()
+
+    if stt_model not in STREAMER_CLASSES:
+        raise ValueError(f"Unknown STT model: {stt_model}")
+    StreamerClass = STREAMER_CLASSES[stt_model]
+    streamer_noisy = StreamerClass(DEFAULT_SR, f"{sample_id}_noisy", on_update=on_noisy)
+    streamer_enhanced = StreamerClass(DEFAULT_SR, f"{sample_id}_enhanced", on_update=on_enhanced)
+
+    accumulated_enhanced: list[np.ndarray] = []
+    n = len(sample_16k)
+
+    for i in range(0, n, chunk_size):
+        raw_chunk = sample_16k[i : i + chunk_size]
+        if raw_chunk.size < chunk_size:
+            raw_chunk = np.pad(
+                raw_chunk,
+                (0, chunk_size - raw_chunk.size),
+                mode="constant",
+                constant_values=0.0,
+            )
+        raw_2d = raw_chunk.reshape(1, -1)
+        enhanced_chunk = sdk.process_chunk(raw_2d)
+        enhanced_1d = np.asarray(enhanced_chunk).flatten()
+        streamer_noisy.process_chunk(raw_chunk)
+        streamer_enhanced.process_chunk(enhanced_1d)
+        accumulated_enhanced.append(enhanced_1d)
+
+    _close_stt_stream(streamer_noisy)
+    _close_stt_stream(streamer_enhanced)
+    streamer_noisy.finished_event.wait()
+    streamer_enhanced.finished_event.wait()
+
+    with streamer_noisy.lock:
+        noisy_transcript = streamer_noisy.render_tokens(streamer_noisy.final_tokens, [])
+    with streamer_enhanced.lock:
+        enhanced_transcript = streamer_enhanced.render_tokens(streamer_enhanced.final_tokens, [])
+
+    enhanced_array = np.concatenate(accumulated_enhanced).astype(np.float32)
+    gradio_enhanced_audio = to_gradio_audio(enhanced_array, DEFAULT_SR)
+
+    enhanced_spec_path = f"{APP_TMP_DIR}/{sample_id}_enhanced_spectrogram.png"
+    spec_image(enhanced_array).save(enhanced_spec_path)
+    progress_state["enhanced_spec_path"] = enhanced_spec_path
+
+    precomputed_noisy = noisy_transcript
+    try:
+        original_transcript = get_transcript(sample_id)
+        wer_enhanced = compute_wer(original_transcript, enhanced_transcript)
+        wer_noisy = compute_wer(original_transcript, noisy_transcript)
+        enhanced_transcript += f" (WER: {wer_enhanced:.2f})"
+        noisy_transcript += f" (WER: {wer_noisy:.2f})"
+    except Exception:
+        pass
+
+    return (
+        enhanced_spec_path,
+        enhanced_transcript,
+        noisy_transcript,
+        gradio_enhanced_audio,
+        sample_id,
+        enhanced_array,
+        precomputed_noisy,
+    )
+
+
 def load_local_file(
     sample_path: str
     ) -> tuple[np.ndarray, str]:
@@ -63,10 +332,10 @@ def load_local_file(
     y_16k, _ = librosa.load(sample_path, sr=DEFAULT_SR, dtype="float32", mono=True)
     return y_16k, new_sample_stem
 
-def load_file_from_dataset(sample_id: str) -> tuple[tuple, np.ndarray, str]:
+def load_file_from_dataset(sample_id: str) -> tuple[tuple | None, np.ndarray | None, str]:
     if not sample_id:
         gr.Warning("Please select a sample from the dropdown.")
-        raise ValueError("No sample selected. Please select a sample from the dropdown.")
+        return None, None, ""
     new_sample_stem = sample_id
     try:
         y_16k, sr = get_audio(sample_id, prefix="mix")

requirements.txt

@@ -1,4 +1,5 @@
 # Requires Python 3.11
+gradio>=6.0
 matplotlib>=3.8,<3.10
 soundfile>=0.12.1
 aiohttp>=3.9,<4

stream_pipeline.py

@@ -12,7 +12,6 @@ from dataclasses import dataclass
 _ENHANCED_TRANSCRIPT: str = ""
 _RAW_TRANSCRIPT: str = ""
 
-
 def _set_transcript_enhanced(text: str) -> None:
     """Deepgram callback: update latest transcript text (no printing)."""
     global _ENHANCED_TRANSCRIPT
@@ -24,14 +23,24 @@ def _set_transcript_raw(text: str) -> None:
     _RAW_TRANSCRIPT = text
 
 
+def get_live_transcripts() -> tuple[str, str]:
+    """Return current enhanced and raw transcript for live UI updates."""
+    return _ENHANCED_TRANSCRIPT, _RAW_TRANSCRIPT
+
+
 SDK = SDKWrapper()
 SDK.init_processor(
-    sample_rate=DEFAULT_SR, 
-    enhancement_level=1.0, 
-    allow_variable_frames=False, 
+    sample_rate=DEFAULT_SR,
+    enhancement_level=1.0,
+    allow_variable_frames=True,  # streaming chunks are variable-sized
     num_channels=1,
-) 
+)
 
+# Created on first start_recording (lazy) to avoid Soniox "No audio received" timeout at app load
+Streamer_enhanced = None
+Streamer_raw = None
+_streamer_generation = 0
+_last_stop_generation = 1  # so first stop doesn't skip (1 > 1 is False)
 
 
 @dataclass
@@ -66,15 +75,25 @@ def _to_float32_mono(y: np.ndarray) -> np.ndarray:
     return y
 
 
-def transcribe_stream(session: StreamSession | None, new_chunk, enhancement_level):
-    if Streamer_enhanced.ws is None or Streamer_raw.ws is None:
-        print("Streamers not connected. Ignoring audio chunk.")
+def transcribe_stream(session: StreamSession | None, new_chunk, enhancement_level, input_gain_db: float = 0.0):
+    if (
+        Streamer_enhanced is None
+        or Streamer_raw is None
+        or Streamer_enhanced.ws is None
+        or Streamer_raw.ws is None
+    ):
         return session, _ENHANCED_TRANSCRIPT, _RAW_TRANSCRIPT
     if new_chunk is None or new_chunk[1] is None:
         return session, _ENHANCED_TRANSCRIPT, _RAW_TRANSCRIPT
 
     sr, y = new_chunk
     y = _to_float32_mono(y)
+    # Apply input gain: linear = 10^(dB/20), clip to avoid overflow
+    if input_gain_db is not None and input_gain_db > 0:
+        gain_linear = np.float32(10.0 ** (float(input_gain_db) / 20.0))
+        y = (y * gain_linear).astype(np.float32)
+        y = np.clip(y, -1.0, 1.0)
+
     session = _get_or_init_session(session, sr)
     SDK.change_enhancement_level(float(enhancement_level) / 100.0)
     if session.resampler is not None:
@@ -82,28 +101,58 @@ def transcribe_stream(session: StreamSession | None, new_chunk, enhancement_leve
     else:
         y_16k = y
 
+    # Ensure 1D float32 for SDK and streamers (resample_chunk can return 0 samples or 2D)
+    y_16k = np.asarray(y_16k, dtype=np.float32).flatten()
+
     # Ringbuffer (nicht unendlich konkatenieren)
     if y_16k.size > 0:
         tail = np.concatenate([session.tail_16k, y_16k])
         if tail.size > session.tail_max:
             tail = tail[-session.tail_max:]
         session.tail_16k = tail
+
+    # Only send when we have samples (resample_chunk can return empty; SDK needs valid input)
+    if y_16k.size == 0:
+        return session, _ENHANCED_TRANSCRIPT, _RAW_TRANSCRIPT
+
+    # Parallel path: send raw to STT immediately, then enhance and send enhanced.
+    # SDK requires fixed num_frames (AudioConfigMismatchError if we use process_chunk with variable size).
+    Streamer_raw.process_chunk(y_16k)
     enhanced_chunk_16k = SDK.process_sync(y_16k)
-    Streamer_enhanced.process_chunk(enhanced_chunk_16k.flatten())
-    Streamer_raw.process_chunk(y_16k.flatten())
+    out_1d = np.asarray(enhanced_chunk_16k, dtype=np.float32).flatten()
+    # Always send something to enhanced so Soniox doesn't close with "No audio received"
+    if out_1d.size > 0:
+        Streamer_enhanced.process_chunk(out_1d)
+    else:
+        Streamer_enhanced.process_chunk(np.zeros(160, dtype=np.float32))
     return session, _ENHANCED_TRANSCRIPT, _RAW_TRANSCRIPT
 
 
-def shutdown_streamers():
+def shutdown_streamers(from_stop_recording: bool = False):
+    """Shut down STT streamers. If from_stop_recording, skip when streamers were
+    created after the last stop (avoids delayed stop killing new streamers)."""
+    global Streamer_enhanced, Streamer_raw, _streamer_generation, _last_stop_generation
+    if from_stop_recording and _streamer_generation > _last_stop_generation:
+        return
+    gen = _streamer_generation
     try:
-        if Streamer_enhanced.ws is not None:
+        if Streamer_enhanced is not None and Streamer_enhanced.ws is not None:
             Streamer_enhanced.shutdown()
-        if Streamer_raw.ws is not None:
+        if Streamer_raw is not None and Streamer_raw.ws is not None:
             Streamer_raw.shutdown()
     except Exception:
-        print("Failed to shutdown raw streamer.")
-        pass
+        print("Failed to shutdown streamers.")
+    finally:
+        Streamer_enhanced = None
+        Streamer_raw = None
+        if from_stop_recording:
+            _last_stop_generation = gen
     
+def on_stop_recording():
+    """Call from Gradio stop_recording so streamers shut down when user clicks Stop."""
+    shutdown_streamers(from_stop_recording=True)
+
+
 def clear_ui():
     global _ENHANCED_TRANSCRIPT, _RAW_TRANSCRIPT
     _ENHANCED_TRANSCRIPT = ""
@@ -111,22 +160,32 @@ def clear_ui():
     return None, _ENHANCED_TRANSCRIPT, _RAW_TRANSCRIPT
 
 def stop_online_backend():
+    """Stop streamers and clear transcripts. Do not update the Audio component:
+    toggling streaming=False then back to True can make the frontend lose the
+    microphone (getUserMedia not re-called), so we leave it unchanged."""
     shutdown_streamers()
     session, enhanced_transcript, raw_transcript = clear_ui()
-    return  session, enhanced_transcript, raw_transcript, gr.update(streaming=False, interactive=False)
+    return session, enhanced_transcript, raw_transcript, gr.update()
 
 
 def set_stt_streamer(model_name):
     StreamerCls = STREAMER_CLASSES.get(model_name, DeepgramStreamer)
-    global Streamer_enhanced, Streamer_raw
-    Streamer_enhanced = StreamerCls(
+    global Streamer_enhanced, Streamer_raw, _streamer_generation
+    # Shut down current streamers first so we don't leak
+    if Streamer_enhanced is not None or Streamer_raw is not None:
+        shutdown_streamers()
+    # Create both before assigning so transcribe_stream never sees one new and one old
+    new_enhanced = StreamerCls(
         fs_hz=DEFAULT_SR,
         stream_name="enhanced",
         on_update=_set_transcript_enhanced,
     )
-    Streamer_raw = StreamerCls(
+    new_raw = StreamerCls(
         fs_hz=DEFAULT_SR,
         stream_name="raw",
         on_update=_set_transcript_raw,
     )
+    _streamer_generation += 1
+    Streamer_enhanced = new_enhanced
+    Streamer_raw = new_raw
     

stt_streamers/soniox_streamer.py

@@ -172,7 +172,9 @@ class SonioxStreamer:
                         self.ws.send(json.dumps({"type": "keepalive"}))  # Soniox-format citeturn2view0
                     self._last_send_ts = time.monotonic()
                 except Exception as e:
-                    print(f"[Soniox:{self.stream_name}] keepalive failed: {e}")
+                    # Normal closure (1000 OK) when stream ends; don't log as failure
+                    if "1000" not in str(e):
+                        print(f"[Soniox:{self.stream_name}] keepalive failed: {e}")
                     self.ws = None
                     return
 

utils.py

@@ -1,4 +1,4 @@
-from typing import Optional
+from typing import Callable, Optional
 
 import numpy as np
 import librosa
@@ -9,6 +9,8 @@ import resampy
 from constants import DEFAULT_SR, STREAMER_CLASSES
 
 def to_gradio_audio(x: np.ndarray, sr: int) -> tuple[int, np.ndarray]:
+    """Return (sample_rate, int16 mono array) for Gradio Audio. Gradio expects int16;
+    passing float32 triggers an internal conversion and a warning."""
     x = np.asarray(x)
 
     # Remove extra dims like (1, n, 1) etc.
@@ -22,9 +24,10 @@ def to_gradio_audio(x: np.ndarray, sr: int) -> tuple[int, np.ndarray]:
     if x.ndim == 2 and x.shape[1] == 1:
         x = x[:, 0]
 
-    # Return float32 in [-1, 1] (simplest for Gradio)
     x = x.astype(np.float32)
     x = np.clip(x, -1.0, 1.0)
+    # Gradio Audio expects int16; convert here so Gradio doesn't convert and warn
+    x = (x * 32767).astype(np.int16)
 
     return (sr, x)
 
@@ -94,7 +97,13 @@ def compute_wer(reference: str, hypothesis: str) -> float:
 
 
 
-def transcribe_audio(audio_array: np.ndarray, sr: int, streamer_type: str = "Deepgram", stream_name: str = "RAW"):
+def transcribe_audio(
+    audio_array: np.ndarray,
+    sr: int,
+    streamer_type: str = "Deepgram",
+    stream_name: str = "RAW",
+    on_update: Optional[Callable[[str], None]] = None,
+):
     """
     Transcribe an audio array using the specified STT streamer.
 
@@ -103,15 +112,14 @@ def transcribe_audio(audio_array: np.ndarray, sr: int, streamer_type: str = "Dee
         sr (int): Sample rate of the audio array
         streamer_type (str): "Soniox" or "Deepgram"
         stream_name (str): Optional label for streamer instance ("RAW", "ENHANCED", etc.)
+        on_update: Optional callback(text: str) called with partial transcript as results stream in.
 
     Returns:
-        str: Transcript text
+        str: Final transcript text
     """
-
     if sr != DEFAULT_SR:
         audio_array = resampy.resample(audio_array, sr, DEFAULT_SR)
         sr = DEFAULT_SR
-        
 
     if streamer_type not in STREAMER_CLASSES:
         raise ValueError(
@@ -120,7 +128,7 @@ def transcribe_audio(audio_array: np.ndarray, sr: int, streamer_type: str = "Dee
         )
 
     StreamerClass = STREAMER_CLASSES[streamer_type]
-    streamer = StreamerClass(sr, stream_name)
+    streamer = StreamerClass(sr, stream_name, on_update=on_update)
 
     transcript = streamer.stream_array(audio_array)