private-voice-agent / src /services /audio-capture.ts
aut4rk's picture
Deploy static voice agent
009fd18 verified
Raw
History Blame
7.52 kB
import type {
AudioCaptureService,
AudioDurationCaptureProgress,
MicPermissionState,
} from "../app/types";
const DEFAULT_THRESHOLD = 0.024;
const DEFAULT_SILENCE_MS = 900;
const DEFAULT_MAX_WAIT_MS = 12000;
export class BrowserAudioCaptureService implements AudioCaptureService {
#stream: MediaStream | null = null;
#audioContext: AudioContext | null = null;
#analyser: AnalyserNode | null = null;
#source: MediaStreamAudioSourceNode | null = null;
#recorder: MediaRecorder | null = null;
async ensureMicrophonePermission(): Promise<MicPermissionState> {
try {
await this.#ensureStream();
return "granted";
} catch {
return "denied";
}
}
async listenForUtterance(options: { signal?: AbortSignal } = {}): Promise<Blob> {
const stream = await this.#ensureStream();
if (typeof MediaRecorder === "undefined") {
throw new Error("MediaRecorder is unavailable in this browser.");
}
const mimeType = BrowserAudioCaptureService.#pickMimeType();
const recorder = mimeType
? new MediaRecorder(stream, { mimeType })
: new MediaRecorder(stream);
this.#recorder = recorder;
const chunks: BlobPart[] = [];
let detectedSpeech = false;
let lastSpeechAt = 0;
let startedAt = 0;
let loopHandle = 0;
recorder.ondataavailable = (event) => {
if (event.data.size > 0) {
chunks.push(event.data);
}
};
const stop = () => {
cancelAnimationFrame(loopHandle);
if (recorder.state !== "inactive") {
recorder.stop();
}
};
options.signal?.addEventListener(
"abort",
() => {
stop();
},
{ once: true },
);
const data = new Float32Array(this.#analyser?.fftSize ?? 2048);
const monitor = () => {
if (!this.#analyser) {
loopHandle = requestAnimationFrame(monitor);
return;
}
this.#analyser.getFloatTimeDomainData(data);
const rms = BrowserAudioCaptureService.#rms(data);
const now = performance.now();
if (rms > DEFAULT_THRESHOLD) {
detectedSpeech = true;
lastSpeechAt = now;
}
if (
detectedSpeech &&
lastSpeechAt > 0 &&
now - lastSpeechAt >= DEFAULT_SILENCE_MS
) {
stop();
return;
}
if (!detectedSpeech && now - startedAt >= DEFAULT_MAX_WAIT_MS) {
stop();
return;
}
loopHandle = requestAnimationFrame(monitor);
};
const result = await new Promise<Blob>((resolve, reject) => {
recorder.onstop = () => {
cancelAnimationFrame(loopHandle);
if (chunks.length === 0) {
reject(new Error("No audio was captured."));
return;
}
resolve(new Blob(chunks, { type: recorder.mimeType || "audio/webm" }));
};
recorder.onerror = () => {
cancelAnimationFrame(loopHandle);
reject(new Error("Audio capture failed."));
};
startedAt = performance.now();
recorder.start(250);
loopHandle = requestAnimationFrame(monitor);
});
this.#recorder = null;
return result;
}
async recordForDuration(options: {
durationMs: number;
signal?: AbortSignal;
onProgress?: (progress: AudioDurationCaptureProgress) => void;
}): Promise<Blob> {
const stream = await this.#ensureStream();
if (typeof MediaRecorder === "undefined") {
throw new Error("MediaRecorder is unavailable in this browser.");
}
const mimeType = BrowserAudioCaptureService.#pickMimeType();
const recorder = mimeType
? new MediaRecorder(stream, { mimeType })
: new MediaRecorder(stream);
this.#recorder = recorder;
const chunks: BlobPart[] = [];
const durationMs = Math.max(1_000, options.durationMs);
let timerHandle = 0;
let progressHandle = 0;
let startedAt = 0;
let aborted = false;
recorder.ondataavailable = (event) => {
if (event.data.size > 0) {
chunks.push(event.data);
}
};
const emitProgress = (elapsedMs: number, isRecording: boolean) => {
options.onProgress?.({
progress: Math.max(0, Math.min(1, elapsedMs / durationMs)),
elapsedMs,
remainingMs: Math.max(0, durationMs - elapsedMs),
isRecording,
});
};
const stop = () => {
window.clearTimeout(timerHandle);
window.clearInterval(progressHandle);
if (recorder.state !== "inactive") {
recorder.stop();
}
};
options.signal?.addEventListener(
"abort",
() => {
aborted = true;
stop();
},
{ once: true },
);
try {
return await new Promise<Blob>((resolve, reject) => {
recorder.onstop = () => {
window.clearTimeout(timerHandle);
window.clearInterval(progressHandle);
if (aborted) {
reject(new Error("Microphone calibration was cancelled."));
return;
}
if (chunks.length === 0) {
reject(new Error("No microphone calibration audio was captured."));
return;
}
emitProgress(durationMs, false);
resolve(new Blob(chunks, { type: recorder.mimeType || "audio/webm" }));
};
recorder.onerror = () => {
window.clearTimeout(timerHandle);
window.clearInterval(progressHandle);
reject(new Error("Microphone calibration recording failed."));
};
startedAt = performance.now();
emitProgress(0, true);
recorder.start(250);
timerHandle = window.setTimeout(() => {
stop();
}, durationMs);
progressHandle = window.setInterval(() => {
const elapsedMs = Math.min(durationMs, performance.now() - startedAt);
emitProgress(elapsedMs, true);
}, 100);
});
} finally {
this.#recorder = null;
}
}
stop(): void {
if (this.#recorder && this.#recorder.state !== "inactive") {
this.#recorder.stop();
}
}
async dispose(): Promise<void> {
this.stop();
this.#stream?.getTracks().forEach((track) => track.stop());
this.#stream = null;
this.#source?.disconnect();
this.#source = null;
this.#analyser?.disconnect();
this.#analyser = null;
await this.#audioContext?.close();
this.#audioContext = null;
}
async #ensureStream(): Promise<MediaStream> {
if (this.#stream) {
return this.#stream;
}
const stream = await navigator.mediaDevices.getUserMedia({
audio: {
channelCount: 1,
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
},
});
this.#stream = stream;
this.#audioContext = new AudioContext();
this.#source = this.#audioContext.createMediaStreamSource(stream);
this.#analyser = this.#audioContext.createAnalyser();
this.#analyser.fftSize = 2048;
this.#source.connect(this.#analyser);
return stream;
}
static #pickMimeType(): string | undefined {
const candidates = [
"audio/webm;codecs=opus",
"audio/mp4",
"audio/webm",
];
for (const candidate of candidates) {
if (MediaRecorder.isTypeSupported(candidate)) {
return candidate;
}
}
return undefined;
}
static #rms(values: Float32Array): number {
let total = 0;
for (const value of values) {
total += value * value;
}
return Math.sqrt(total / values.length);
}
}