avatar/audio-queue.js

// avatar/audio-queue.js
//
// WebAudio decode + scheduling. Emits speech-start / speech-end.
//
// The one thing that matters here: playBuffer hands the SCHEDULED start time back
// through onStart(when) before the buffer plays. The player must key its clock off
// that value, not off the wall-clock moment the call was made, or every utterance
// begins ~50 ms out of sync.

const START_LEAD = 0.05; // a small lead so the first viseme is not already late

// How long playBuffer waits for a resume() it issued itself before declaring the context
// blocked. Chromium keeps the resume() promise PENDING - neither resolved nor rejected -
// while its autoplay policy blocks the context, so an unbounded await would hang the turn
// forever with the status stuck on "speaking..."; the bound is what makes the refusal in
// playBuffer reachable. There is no signal that separates "blocked" from "slow", so the
// wait depends on the one fact that predicts it: whether the document has ever received a
// user gesture (navigator.userActivation.hasBeenActive). Never activated - nothing can
// unblock it, say so quickly. Activated - the resume() is legitimate and only waiting for
// the audio sink to start: a few milliseconds on a device, 0.7-5 s measured on the
// headless test fleet under load, so the wait is generous rather than a false error.
const RESUME_GRACE_MS = 2000;
const RESUME_GRACE_ACTIVATED_MS = 8000;

function documentHasBeenActivated() {
  const activation = typeof navigator !== 'undefined' ? navigator.userActivation : null;
  // Unknown (older browsers): assume it has, and prefer a longer wait to a false error.
  return activation ? activation.hasBeenActive : true;
}

// Shown by the host's status line. The remedy is the gesture: every control resumes the
// context inside its own tap (stagePort.unlockAudio), so one tap is enough.
const AUDIO_BLOCKED_MESSAGE =
  'audio is blocked by the browser - tap Say hello or Send once, then try again';

let lastDecoded = null;
let lastUrl = null;

/** The most recently decoded AudioBuffer, so a replay costs no network. */
export function getLastDecoded() {
  return lastDecoded;
}

/** The URL the cached buffer came from, for diagnostics only. */
export function getLastUrl() {
  return lastUrl;
}

export function clearCache() {
  lastDecoded = null;
  lastUrl = null;
}

async function toAudioBuffer(audioCtx, input) {
  if (input && typeof input.getChannelData === 'function') return input; // already decoded
  let bytes;
  if (typeof input === 'string') {
    const res = await fetch(input);
    if (!res.ok) throw new Error(`audio fetch failed: ${res.status} ${input}`);
    bytes = await res.arrayBuffer();
    lastUrl = input;
  } else if (input instanceof ArrayBuffer) {
    bytes = input;
    lastUrl = null;
  } else {
    throw new Error('playBuffer needs a URL string, an ArrayBuffer or an AudioBuffer');
  }
  // decodeAudioData detaches its input, so hand it a copy and keep ours usable.
  return audioCtx.decodeAudioData(bytes.slice(0));
}

/** resume() with a deadline: see RESUME_GRACE_MS for why the deadline exists. */
async function resumeBounded(audioCtx) {
  if (audioCtx.state === 'running') return;
  const grace = documentHasBeenActivated() ? RESUME_GRACE_ACTIVATED_MS : RESUME_GRACE_MS;
  let timer = null;
  await Promise.race([
    audioCtx.resume().catch(() => {}),
    new Promise((resolve) => {
      timer = setTimeout(resolve, grace);
    }),
  ]);
  clearTimeout(timer);
}

/**
 * Decode (if needed), schedule and play. Resolves at speech-end.
 *
 * Rejects - after emitting 'error' with where: 'audio' - when the context is not running,
 * instead of scheduling against a clock that never advances.
 *
 * @param {AudioContext} audioCtx
 * @param {string|ArrayBuffer|AudioBuffer} input
 * @param {(name: string, data?: object) => void} emit
 * @param {(scheduledStart: number) => void} onStart
 */
export async function playBuffer(audioCtx, input, emit = () => {}, onStart = () => {}) {
  // This is NOT where the unlock belongs. playBuffer runs after the server round trip,
  // seconds outside any user gesture, so on a gesture-gated browser (iOS Safari, a
  // cross-origin embed in Chromium) a resume() issued here is refused. The unlock that
  // counts is stagePort.unlockAudio(), issued synchronously inside the gesture by
  // turn-loop.js and the host glue; this bounded attempt only picks up a context that is
  // already allowed to run (a fresh context under a relaxed policy, or a headless run).
  await resumeBounded(audioCtx);

  // Never emit speech-start against a stopped clock. A suspended context never advances
  // currentTime: the viseme player would never tick, onended would never fire, and the
  // page would sit on "speaking..." in silence. Say so and let the caller fail loudly.
  if (audioCtx.state !== 'running') {
    emit('error', { where: 'audio', message: AUDIO_BLOCKED_MESSAGE, audioState: audioCtx.state });
    throw new Error(AUDIO_BLOCKED_MESSAGE);
  }

  const buffer = await toAudioBuffer(audioCtx, input);
  lastDecoded = buffer;

  const node = audioCtx.createBufferSource();
  node.buffer = buffer;
  node.connect(audioCtx.destination);

  const when = audioCtx.currentTime + START_LEAD;
  onStart(when); // the player arms its clock against this exact value
  node.start(when);

  // Emitted at scheduling time rather than START_LEAD later: the 50 ms lead is below
  // the resolution of anything that listens, and a timer here would be a second clock.
  emit('speech-start', { when, duration: buffer.duration });

  return new Promise((resolve) => {
    node.onended = () => {
      emit('speech-end', { duration: buffer.duration });
      resolve({ duration: buffer.duration });
    };
  });
}