#!/usr/bin/env python3 """ FireRedVAD ONNX inference — no PyTorch / kaldi_native_fbank required. Dependencies: numpy, scipy, soundfile, onnxruntime Usage: python infer_onnx.py assets/hello_zh.wav --model_dir /path/to/FireRedVAD_onnx python infer_onnx.py assets/hello_zh.wav python infer_onnx.py assets/hello_zh.wav --model_dir /path/to/FireRedVAD_onnx python infer_onnx.py assets/hello_en.wav --speech_threshold 0.4 --min_speech_frame 20 """ import argparse import json import math import os from collections import deque import numpy as np import soundfile as sf import onnxruntime as ort # --------------------------------------------------------------------------- # Constants (matches fireredvad/core/constants.py) # --------------------------------------------------------------------------- SAMPLE_RATE = 16000 FRAME_LENGTH_MS = 25 FRAME_SHIFT_MS = 10 FRAME_LENGTH_S = 0.025 FRAME_SHIFT_S = 0.010 # --------------------------------------------------------------------------- # Kaldi-compatible Fbank (replaces kaldi_native_fbank dependency) # --------------------------------------------------------------------------- def _mel_to_hz(mel): return 700.0 * (np.exp(mel / 1127.0) - 1.0) def _hz_to_mel(hz): return 1127.0 * np.log(1.0 + hz / 700.0) def _build_mel_filterbank(n_fft: int, n_mels: int = 80, f_min: float = 0.0, f_max: float = 8000.0, sample_rate: int = 16000) -> np.ndarray: """ Build Kaldi-style mel filterbank matrix. Returns shape (n_mels, n_fft // 2 + 1) — real spectrum bins. Kaldi uses triangular filters defined on the mel scale, NOT on the FFT bin scale. """ n_freqs = n_fft // 2 + 1 freq_bins = np.linspace(0, sample_rate / 2, n_freqs) # Hz for each FFT bin mel_min = _hz_to_mel(f_min) mel_max = _hz_to_mel(f_max) # n_mels + 2 points: left edge, n_mels centers, right edge mel_points = np.linspace(mel_min, mel_max, n_mels + 2) hz_points = _mel_to_hz(mel_points) # For each mel band, compute the triangular weight for each FFT bin filters = np.zeros((n_mels, n_freqs), dtype=np.float32) for m in range(1, n_mels + 1): left = hz_points[m - 1] center = hz_points[m] right = hz_points[m + 1] for k, f in enumerate(freq_bins): if left <= f <= center: filters[m - 1, k] = (f - left) / (center - left) elif center < f <= right: filters[m - 1, k] = (right - f) / (right - center) return filters def _extract_kaldi_fbank(wav_int16: np.ndarray, sample_rate: int = 16000, num_mel_bins: int = 80, frame_length_ms: float = 25.0, frame_shift_ms: float = 10.0) -> np.ndarray: """ Compute log Mel filterbank features identical to kaldi_native_fbank with: samp_freq=16000, frame_length_ms=25, frame_shift_ms=10, dither=0, snip_edges=True, num_mel_bins=80 Input : int16 PCM array (1-D) Output: (T, 80) float32 array — log fbank energies """ assert wav_int16.dtype == np.int16 and wav_int16.ndim == 1 frame_len = int(round(sample_rate * frame_length_ms / 1000)) # 400 frame_shift = int(round(sample_rate * frame_shift_ms / 1000)) # 160 n_fft = 1 << (frame_len - 1).bit_length() # next power of 2 >= 400 → 512 # Hann window (Kaldi uses the same) window = np.hanning(frame_len).astype(np.float32) # Convert to float, Kaldi: raw int16 → float32 (NOT divided by 32768 at fbank stage) # Actually Kaldi processes the waveform as-is (int16 values as float). wav_f = wav_int16.astype(np.float32) # Pre-emphasis (Kaldi default: 0.97) wav_preemph = np.append(wav_f[0], wav_f[1:] - 0.97 * wav_f[:-1]) # Framing (snip_edges=True: only frames that fit completely) n_frames = 1 + (len(wav_preemph) - frame_len) // frame_shift if n_frames <= 0: return np.zeros((0, num_mel_bins), dtype=np.float32) frames = np.stack([ wav_preemph[i * frame_shift: i * frame_shift + frame_len] * window for i in range(n_frames) ]) # (T, frame_len) # FFT → power spectrum spec = np.fft.rfft(frames, n=n_fft) # (T, n_fft//2+1) complex power = (spec.real ** 2 + spec.imag ** 2).astype(np.float32) # (T, n_fft//2+1) # Mel filterbank mel_fb = _build_mel_filterbank(n_fft, num_mel_bins, f_min=0.0, f_max=sample_rate / 2, sample_rate=sample_rate) # (80, n_fft//2+1) mel_energy = power @ mel_fb.T # (T, 80) # Log compression — floor at 1.0 (Kaldi: log_energy_floor = FLT_MIN, effectively 0) mel_energy = np.maximum(mel_energy, 1.0) log_mel = np.log(mel_energy).astype(np.float32) return log_mel # (T, 80) # --------------------------------------------------------------------------- # CMVN (matches fireredvad/core/audio_feat.py CMVN.__call__) # --------------------------------------------------------------------------- class CMVN: def __init__(self, cmvn_json_path: str): with open(cmvn_json_path) as f: d = json.load(f) self.means = np.array(d["means"], dtype=np.float32) self.inv_std = np.array(d["inverse_std_variances"], dtype=np.float32) def __call__(self, fbank: np.ndarray) -> np.ndarray: return (fbank - self.means) * self.inv_std # --------------------------------------------------------------------------- # VAD postprocessor (pure Python, mirrors vad_postprocessor.py) # --------------------------------------------------------------------------- class VadPostprocessor: def __init__(self, smooth_window_size=5, prob_threshold=0.4, min_speech_frame=20, max_speech_frame=2000, min_silence_frame=20, merge_silence_frame=0, extend_speech_frame=0): self.smooth_window_size = max(1, smooth_window_size) self.prob_threshold = prob_threshold self.min_speech_frame = min_speech_frame self.max_speech_frame = max_speech_frame self.min_silence_frame = min_silence_frame self.merge_silence_frame = merge_silence_frame self.extend_speech_frame = extend_speech_frame def process(self, raw_probs): if not raw_probs: return [] smoothed = self._smooth(raw_probs) binary = (np.asarray(smoothed) >= self.prob_threshold).astype(int).tolist() decisions = self._state_machine(binary) decisions = self._fix_start(decisions) decisions = self._merge_silence(decisions) decisions = self._extend_speech(decisions) decisions = self._split_long(decisions, raw_probs) return decisions def decisions_to_segments(self, decisions, wav_dur=None): segments = [] speech_start = None for t, d in enumerate(decisions): if d == 1 and speech_start is None: speech_start = t elif d == 0 and speech_start is not None: segments.append((speech_start * FRAME_SHIFT_S, t * FRAME_SHIFT_S)) speech_start = None if speech_start is not None: end = len(decisions) * FRAME_SHIFT_S + FRAME_LENGTH_S if wav_dur is not None: end = min(end, wav_dur) segments.append((speech_start * FRAME_SHIFT_S, end)) return [(round(s, 3), round(e, 3)) for s, e in segments] def _smooth(self, probs): if self.smooth_window_size <= 1: return probs probs_np = np.array(probs) kernel = np.ones(self.smooth_window_size) / self.smooth_window_size smoothed = np.convolve(probs_np, kernel, mode='full')[:len(probs)] for i in range(min(self.smooth_window_size - 1, len(probs))): smoothed[i] = np.mean(probs_np[:i + 1]) return smoothed def _state_machine(self, binary): SILENCE, POSSIBLE_SPEECH, SPEECH, POSSIBLE_SILENCE = 0, 1, 2, 3 decisions = [0] * len(binary) state = SILENCE speech_start = silence_start = -1 for t, is_speech in enumerate(binary): if state == SILENCE: if is_speech: state = POSSIBLE_SPEECH; speech_start = t elif state == POSSIBLE_SPEECH: if is_speech: if t - speech_start >= self.min_speech_frame: state = SPEECH decisions[speech_start:t] = [1] * (t - speech_start) else: state = SILENCE; speech_start = -1 elif state == SPEECH: if not is_speech: state = POSSIBLE_SILENCE; silence_start = t elif state == POSSIBLE_SILENCE: if not is_speech: if t - silence_start >= self.min_silence_frame: state = SILENCE; speech_start = -1 else: state = SPEECH; silence_start = -1 decisions[t] = 1 if state in (SPEECH, POSSIBLE_SILENCE) else 0 return decisions def _fix_start(self, decisions): new = decisions.copy() for t, d in enumerate(decisions): if t > 0 and decisions[t - 1] == 0 and d == 1: start = max(0, t - self.smooth_window_size) new[start:t] = [1] * (t - start) return new def _merge_silence(self, decisions): if self.merge_silence_frame <= 0: return decisions new = decisions.copy() silence_start = None for t, d in enumerate(decisions): if t > 0 and decisions[t - 1] == 1 and d == 0 and silence_start is None: silence_start = t elif t > 0 and decisions[t - 1] == 0 and d == 1 and silence_start is not None: if t - silence_start < self.merge_silence_frame: new[silence_start:t] = [1] * (t - silence_start) silence_start = None return new def _extend_speech(self, decisions): if self.extend_speech_frame <= 0: return decisions d = np.array(decisions) k = np.ones(2 * self.extend_speech_frame + 1) return (np.convolve(d, k, mode='same') > 0).astype(int).tolist() def _split_long(self, decisions, probs): new = decisions.copy() segments = self.decisions_to_segments(decisions) for s_s, e_s in segments: sf_ = int(s_s / FRAME_SHIFT_S) ef_ = int(e_s / FRAME_SHIFT_S) if ef_ - sf_ > self.max_speech_frame: seg_probs = probs[sf_:ef_] for split in self._find_splits(seg_probs): new[sf_ + split] = 0 return new def _find_splits(self, probs): splits, L, start = [], len(probs), 0 while start < L: if (L - start) <= self.max_speech_frame: break ws = int(start + self.max_speech_frame / 2) we = int(start + self.max_speech_frame) splits.append(ws + int(np.argmin(probs[ws:we]))) start = splits[-1] + 1 return splits # --------------------------------------------------------------------------- # Main inference class # --------------------------------------------------------------------------- class FireRedVadOnnx: """ End-to-end FireRedVAD inference using ONNX Runtime. No PyTorch or kaldi_native_fbank required. """ def __init__(self, model_dir: str, smooth_window_size: int = 5, speech_threshold: float = 0.4, min_speech_frame: int = 20, max_speech_frame: int = 2000, min_silence_frame: int = 20, merge_silence_frame: int = 0, extend_speech_frame: int = 0, chunk_max_frame: int = 30000, use_coreml: bool = False): # CMVN params # Support both naming conventions: cmvn.json and cmvn_params.json for name in ("cmvn.json", "cmvn_params.json"): p = os.path.join(model_dir, name) if os.path.exists(p): self.cmvn = CMVN(p) break else: raise FileNotFoundError(f"No cmvn JSON found in {model_dir}") # ONNX session model_path = os.path.join(model_dir, "model.onnx") providers = ["CPUExecutionProvider"] if use_coreml: providers = ["CoreMLExecutionProvider"] + providers opts = ort.SessionOptions() opts.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL self.session = ort.InferenceSession(model_path, sess_options=opts, providers=providers) # Fbank config (fixed to match training) self.num_mel_bins = 80 self.chunk_max_frame = chunk_max_frame # Postprocessor self.postprocessor = VadPostprocessor( smooth_window_size=smooth_window_size, prob_threshold=speech_threshold, min_speech_frame=min_speech_frame, max_speech_frame=max_speech_frame, min_silence_frame=min_silence_frame, merge_silence_frame=merge_silence_frame, extend_speech_frame=extend_speech_frame) def _load_audio(self, audio_path: str): wav, sr = sf.read(audio_path, dtype="int16") assert sr == SAMPLE_RATE, f"Expected 16kHz, got {sr}Hz. Convert with ffmpeg first." assert wav.ndim == 1, "Expected mono audio." return wav def _extract_features(self, wav_int16: np.ndarray) -> np.ndarray: """Returns (T, 80) CMVN-normalized log-fbank.""" fbank = _extract_kaldi_fbank(wav_int16, SAMPLE_RATE, self.num_mel_bins) return self.cmvn(fbank) def _run_model(self, feat: np.ndarray) -> np.ndarray: """ feat: (T, 80) Returns probs: (T,) float32 """ T = feat.shape[0] all_probs = [] for chunk_start in range(0, T, self.chunk_max_frame): chunk = feat[chunk_start: chunk_start + self.chunk_max_frame] inp = chunk[np.newaxis, :, :].astype(np.float32) # (1, t, 80) outputs = self.session.run(["output"], {"input": inp}) probs_chunk = outputs[0][0, :, 0] # (t,) all_probs.append(probs_chunk) return np.concatenate(all_probs) # (T,) def detect(self, audio_path: str): """ Run VAD on a 16kHz mono WAV file. Returns: result : dict with keys 'dur', 'timestamps', 'wav_path' probs : (T,) float32 array of raw per-frame speech probabilities """ wav = self._load_audio(audio_path) dur = len(wav) / SAMPLE_RATE feat = self._extract_features(wav) # (T, 80) probs = self._run_model(feat) # (T,) decisions = self.postprocessor.process(probs.tolist()) segments = self.postprocessor.decisions_to_segments(decisions, dur) result = { "dur": round(dur, 3), "timestamps": segments, "wav_path": audio_path, } return result, probs # --------------------------------------------------------------------------- # CLI # --------------------------------------------------------------------------- def main(): parser = argparse.ArgumentParser(description="FireRedVAD ONNX inference") parser.add_argument("wav_path", help="Path to 16kHz mono WAV file") parser.add_argument("--model_dir", default="/path/to/FireRedVAD_onnx", help="Directory containing model.onnx and cmvn.json") parser.add_argument("--smooth_window_size", type=int, default=5) parser.add_argument("--speech_threshold", type=float, default=0.4) parser.add_argument("--min_speech_frame", type=int, default=20) parser.add_argument("--max_speech_frame", type=int, default=2000) parser.add_argument("--min_silence_frame", type=int, default=20) parser.add_argument("--merge_silence_frame", type=int, default=0) parser.add_argument("--extend_speech_frame", type=int, default=0) parser.add_argument("--use_coreml", action="store_true", help="Use CoreML execution provider (Apple Silicon)") args = parser.parse_args() vad = FireRedVadOnnx( model_dir=args.model_dir, smooth_window_size=args.smooth_window_size, speech_threshold=args.speech_threshold, min_speech_frame=args.min_speech_frame, max_speech_frame=args.max_speech_frame, min_silence_frame=args.min_silence_frame, merge_silence_frame=args.merge_silence_frame, extend_speech_frame=args.extend_speech_frame, use_coreml=args.use_coreml, ) import time start = time.time() result, probs = vad.detect(args.wav_path) du = time.time() - start print(f"vad : {du}s") print(f"Duration : {result['dur']:.3f}s") print(f"Segments : {len(result['timestamps'])}") for i, (s, e) in enumerate(result["timestamps"]): print(f" [{i+1:3d}] {s:.3f}s -- {e:.3f}s ({e-s:.3f}s)") if __name__ == "__main__": main() # python infer_onnx.py assets/hello_zh.wav --model_dir /path/to/FireRedVAD_onnx