// avatar/audio-queue.js // // WebAudio decode + scheduling. Emits speech-start / speech-end. // // The one thing that matters here: playBuffer hands the SCHEDULED start time back // through onStart(when) before the buffer plays. The player must key its clock off // that value, not off the wall-clock moment the call was made, or every utterance // begins ~50 ms out of sync. const START_LEAD = 0.05; // a small lead so the first viseme is not already late // How long playBuffer waits for a resume() it issued itself before declaring the context // blocked. Chromium keeps the resume() promise PENDING - neither resolved nor rejected - // while its autoplay policy blocks the context, so an unbounded await would hang the turn // forever with the status stuck on "speaking..."; the bound is what makes the refusal in // playBuffer reachable. There is no signal that separates "blocked" from "slow", so the // wait depends on the one fact that predicts it: whether the document has ever received a // user gesture (navigator.userActivation.hasBeenActive). Never activated - nothing can // unblock it, say so quickly. Activated - the resume() is legitimate and only waiting for // the audio sink to start: a few milliseconds on a device, 0.7-5 s measured on the // headless test fleet under load, so the wait is generous rather than a false error. const RESUME_GRACE_MS = 2000; const RESUME_GRACE_ACTIVATED_MS = 8000; function documentHasBeenActivated() { const activation = typeof navigator !== 'undefined' ? navigator.userActivation : null; // Unknown (older browsers): assume it has, and prefer a longer wait to a false error. return activation ? activation.hasBeenActive : true; } // Shown by the host's status line. The remedy is the gesture: every control resumes the // context inside its own tap (stagePort.unlockAudio), so one tap is enough. const AUDIO_BLOCKED_MESSAGE = 'audio is blocked by the browser - tap Say hello or Send once, then try again'; let lastDecoded = null; let lastUrl = null; /** The most recently decoded AudioBuffer, so a replay costs no network. */ export function getLastDecoded() { return lastDecoded; } /** The URL the cached buffer came from, for diagnostics only. */ export function getLastUrl() { return lastUrl; } export function clearCache() { lastDecoded = null; lastUrl = null; } async function toAudioBuffer(audioCtx, input) { if (input && typeof input.getChannelData === 'function') return input; // already decoded let bytes; if (typeof input === 'string') { const res = await fetch(input); if (!res.ok) throw new Error(`audio fetch failed: ${res.status} ${input}`); bytes = await res.arrayBuffer(); lastUrl = input; } else if (input instanceof ArrayBuffer) { bytes = input; lastUrl = null; } else { throw new Error('playBuffer needs a URL string, an ArrayBuffer or an AudioBuffer'); } // decodeAudioData detaches its input, so hand it a copy and keep ours usable. return audioCtx.decodeAudioData(bytes.slice(0)); } /** resume() with a deadline: see RESUME_GRACE_MS for why the deadline exists. */ async function resumeBounded(audioCtx) { if (audioCtx.state === 'running') return; const grace = documentHasBeenActivated() ? RESUME_GRACE_ACTIVATED_MS : RESUME_GRACE_MS; let timer = null; await Promise.race([ audioCtx.resume().catch(() => {}), new Promise((resolve) => { timer = setTimeout(resolve, grace); }), ]); clearTimeout(timer); } /** * Decode (if needed), schedule and play. Resolves at speech-end. * * Rejects - after emitting 'error' with where: 'audio' - when the context is not running, * instead of scheduling against a clock that never advances. * * @param {AudioContext} audioCtx * @param {string|ArrayBuffer|AudioBuffer} input * @param {(name: string, data?: object) => void} emit * @param {(scheduledStart: number) => void} onStart */ export async function playBuffer(audioCtx, input, emit = () => {}, onStart = () => {}) { // This is NOT where the unlock belongs. playBuffer runs after the server round trip, // seconds outside any user gesture, so on a gesture-gated browser (iOS Safari, a // cross-origin embed in Chromium) a resume() issued here is refused. The unlock that // counts is stagePort.unlockAudio(), issued synchronously inside the gesture by // turn-loop.js and the host glue; this bounded attempt only picks up a context that is // already allowed to run (a fresh context under a relaxed policy, or a headless run). await resumeBounded(audioCtx); // Never emit speech-start against a stopped clock. A suspended context never advances // currentTime: the viseme player would never tick, onended would never fire, and the // page would sit on "speaking..." in silence. Say so and let the caller fail loudly. if (audioCtx.state !== 'running') { emit('error', { where: 'audio', message: AUDIO_BLOCKED_MESSAGE, audioState: audioCtx.state }); throw new Error(AUDIO_BLOCKED_MESSAGE); } const buffer = await toAudioBuffer(audioCtx, input); lastDecoded = buffer; const node = audioCtx.createBufferSource(); node.buffer = buffer; node.connect(audioCtx.destination); const when = audioCtx.currentTime + START_LEAD; onStart(when); // the player arms its clock against this exact value node.start(when); // Emitted at scheduling time rather than START_LEAD later: the 50 ms lead is below // the resolution of anything that listens, and a timer here would be a second clock. emit('speech-start', { when, duration: buffer.duration }); return new Promise((resolve) => { node.onended = () => { emit('speech-end', { duration: buffer.duration }); resolve({ duration: buffer.duration }); }; }); }