from typing import Optional
import numpy as np
import librosa
from PIL import Image
import io
import matplotlib.pyplot as plt
from constants import TARGET_LOUDNESS, TARGET_TP
import pyloudnorm as pyln
VAD_ON_HTML = """
"""
VAD_OFF_HTML = """
"""
SUB_ON = "🟢"
SUB_OFF = "âš«"
def get_vad_labels(vad_timestamps: list[list[float]], length: float) -> list[dict]:
subtitles = []
cur = 0.0
for start, end in vad_timestamps:
if start > cur:
subtitles.append({
"text": f"Voice Detection: {SUB_OFF}",
"timestamp": [cur, start]
})
subtitles.append({
"text": f"Voice Detection: {SUB_ON}",
"timestamp": [start, end]
})
cur = end
if cur < length:
subtitles.append({
"text": f"Voice Detection: {SUB_OFF}",
"timestamp": [cur, length]
})
return subtitles
def render_vad_led(is_speech: bool) -> str:
return VAD_ON_HTML if is_speech else VAD_OFF_HTML
def to_gradio_audio(x: np.ndarray, sr: int) -> tuple[int, np.ndarray]:
"""Return (sample_rate, int16 mono array) for Gradio Audio. Gradio expects int16;
passing float32 triggers an internal conversion and a warning."""
x = np.asarray(x)
# Remove extra dims like (1, n, 1) etc.
x = np.squeeze(x)
# If it's (channels, samples), transpose to (samples, channels)
if x.ndim == 2 and x.shape[0] in (1, 2) and x.shape[1] > x.shape[0]:
x = x.T
# Ensure mono is (n_samples,)
if x.ndim == 2 and x.shape[1] == 1:
x = x[:, 0]
x = x.astype(np.float32)
x = np.clip(x, -1.0, 1.0)
# Gradio Audio expects int16; convert here so Gradio doesn't convert and warn
x = (x * 32767).astype(np.int16)
return (sr, x)
def spec_image(
audio_array: np.ndarray,
sr: int,
n_fft: int = 2048,
hop_length: int = 512,
n_mels: int = 128,
fmax: Optional[float] = None,
) -> Image.Image:
"""
Generate a mel-spectrogram image from an audio array.
"""
y = audio_array.flatten() # Ensure it's 1D
S = librosa.feature.melspectrogram(
y=y,
sr=sr,
n_fft=n_fft,
hop_length=hop_length,
n_mels=n_mels,
fmax=fmax or sr // 2,
)
S_db = librosa.power_to_db(S, ref=np.max(S))
fig, ax = plt.subplots(figsize=(8, 3), dpi=150)
img = librosa.display.specshow(
S_db, sr=sr, hop_length=hop_length, x_axis="time", y_axis="mel", ax=ax
)
cbar = fig.colorbar(img, ax=ax, format="%+2.0f dB")
cbar.set_label("dB")
ax.set_title("Mel-spectrogram")
ax.set_xlabel("Time in s")
ax.set_ylabel("Frequency in Hz")
fig.tight_layout(pad=0.2)
buf = io.BytesIO()
fig.savefig(buf, format="png", bbox_inches="tight", pad_inches=0)
plt.close(fig)
buf.seek(0)
return Image.open(buf).convert("RGB")
def compute_wer(reference: str, hypothesis: str) -> float:
"""
Compute Word Error Rate (WER) between reference and hypothesis transcripts.
"""
ref_words = reference.split()
hyp_words = hypothesis.split()
d = np.zeros((len(ref_words) + 1, len(hyp_words) + 1), dtype=np.uint8)
for i in range(len(ref_words) + 1):
d[i][0] = i
for j in range(len(hyp_words) + 1):
d[0][j] = j
for i in range(1, len(ref_words) + 1):
for j in range(1, len(hyp_words) + 1):
if ref_words[i - 1] == hyp_words[j - 1]:
cost = 0
else:
cost = 1
d[i][j] = min(
d[i - 1][j] + 1, # Deletion
d[i][j - 1] + 1, # Insertion
d[i - 1][j - 1] + cost, # Substitution
)
wer = d[len(ref_words)][len(hyp_words)] / max(len(ref_words), 1)
return wer
def measure_loudness(x: np.ndarray, sr: int) -> float:
meter = pyln.Meter(sr)
return float(meter.integrated_loudness(x))
def true_peak_limiter(x: np.ndarray, sr: int, max_true_peak: float = TARGET_TP) -> np.ndarray:
upsampled_sr = 192000
x_upsampled = librosa.resample(x, orig_sr=sr, target_sr=upsampled_sr)
true_peak = np.max(np.abs(x_upsampled))
if true_peak > 0:
true_peak_db = 20 * np.log10(true_peak)
if true_peak_db > max_true_peak:
gain_db = max_true_peak - true_peak_db
gain = 10 ** (gain_db / 20)
x_upsampled = x_upsampled * gain
x_limited = librosa.resample(x_upsampled, orig_sr=upsampled_sr, target_sr=sr)
x_limited = librosa.util.fix_length(x_limited, size=x.shape[-1])
return x_limited.astype("float32")
def normalize_lufs(x: np.ndarray, sr: int) -> np.ndarray:
"""
Normalize audio to a fixed integrated loudness target and limit true peak.
"""
try:
current_lufs = measure_loudness(x, sr)
if not np.isfinite(current_lufs):
return x.astype("float32")
gain_db = TARGET_LOUDNESS - current_lufs
gain = 10 ** (gain_db / 20)
y = x * gain
y = true_peak_limiter(y, sr, max_true_peak=TARGET_TP)
return y.astype("float32")
except Exception as e:
warnings.warn(f"LUFS normalization failed, returning input unchanged: {e}")
return x.astype("float32")