feat(01-03): add the transport-agnostic VRM stage, viseme player and audio queue

- vrm-stage.js: single-instance esm.sh module load (?deps= pinned, no import map),
VRM mount with 0.0/1.0 facing branch, additive blink/breathe/sway idle life, and a
numeric __debug surface including threeInstanceCount
- lipsync.js: AudioContext-clocked timeline player with a 50 ms attack cross-fade and
a moving cursor; no timer, no frame counter, no amplitude fallback
- audio-queue.js: decode + schedule, speech-start/speech-end, and a decoded-buffer
cache so replay costs no network
- none of the three reference a host framework

avatar/audio-queue.js

@@ -0,0 +1,86 @@
+// avatar/audio-queue.js
+//
+// WebAudio decode + scheduling. Emits speech-start / speech-end.
+//
+// The one thing that matters here: playBuffer hands the SCHEDULED start time back
+// through onStart(when) before the buffer plays. The player must key its clock off
+// that value, not off the wall-clock moment the call was made, or every utterance
+// begins ~50 ms out of sync.
+
+const START_LEAD = 0.05; // a small lead so the first viseme is not already late
+
+let lastDecoded = null;
+let lastUrl = null;
+
+/** The most recently decoded AudioBuffer, so a replay costs no network. */
+export function getLastDecoded() {
+  return lastDecoded;
+}
+
+/** The URL the cached buffer came from, for diagnostics only. */
+export function getLastUrl() {
+  return lastUrl;
+}
+
+export function clearCache() {
+  lastDecoded = null;
+  lastUrl = null;
+}
+
+async function toAudioBuffer(audioCtx, input) {
+  if (input && typeof input.getChannelData === 'function') return input; // already decoded
+  let bytes;
+  if (typeof input === 'string') {
+    const res = await fetch(input);
+    if (!res.ok) throw new Error(`audio fetch failed: ${res.status} ${input}`);
+    bytes = await res.arrayBuffer();
+    lastUrl = input;
+  } else if (input instanceof ArrayBuffer) {
+    bytes = input;
+    lastUrl = null;
+  } else {
+    throw new Error('playBuffer needs a URL string, an ArrayBuffer or an AudioBuffer');
+  }
+  // decodeAudioData detaches its input, so hand it a copy and keep ours usable.
+  return audioCtx.decodeAudioData(bytes.slice(0));
+}
+
+/**
+ * Decode (if needed), schedule and play. Resolves at speech-end.
+ *
+ * @param {AudioContext} audioCtx
+ * @param {string|ArrayBuffer|AudioBuffer} input
+ * @param {(name: string, data?: object) => void} emit
+ * @param {(scheduledStart: number) => void} onStart
+ */
+export async function playBuffer(audioCtx, input, emit = () => {}, onStart = () => {}) {
+  if (audioCtx.state === 'suspended') {
+    try {
+      await audioCtx.resume();
+    } catch {
+      /* a gesture-gated context stays suspended; the caller still gets its events */
+    }
+  }
+
+  const buffer = await toAudioBuffer(audioCtx, input);
+  lastDecoded = buffer;
+
+  const node = audioCtx.createBufferSource();
+  node.buffer = buffer;
+  node.connect(audioCtx.destination);
+
+  const when = audioCtx.currentTime + START_LEAD;
+  onStart(when); // the player arms its clock against this exact value
+  node.start(when);
+
+  // Emitted at scheduling time rather than START_LEAD later: the 50 ms lead is below
+  // the resolution of anything that listens, and a timer here would be a second clock.
+  emit('speech-start', { when, duration: buffer.duration });
+
+  return new Promise((resolve) => {
+    node.onended = () => {
+      emit('speech-end', { duration: buffer.duration });
+      resolve({ duration: buffer.duration });
+    };
+  });
+}

avatar/lipsync.js

@@ -0,0 +1,100 @@
+// avatar/lipsync.js
+//
+// The dumb player. All of the hard arithmetic (frame quantisation, banker's rounding,
+// pause-mora ordering) happens in Python and arrives here as a finished timeline.
+// This module's only job is to be on time.
+//
+// CLOCK DISCIPLINE: the clock is audioCtx.currentTime minus the SCHEDULED start.
+// Never a timer, never a frame counter. A frame counter drifts against the audio
+// hardware clock and the drift shows up as lip-sync sliding late over the last third
+// of a long sentence - the exact symptom this design exists to prevent.
+
+const VISEMES = ['aa', 'ih', 'ou', 'ee', 'oh'];
+const ATTACK = 0.05; // 50 ms cross-fade, inside the 40-60 ms band that reads as speech
+
+/**
+ * @param {object} stage a handle returned by mountStage()
+ * @returns {{start: Function, stop: Function, tick: Function, isActive: Function}}
+ */
+export function makePlayer(stage) {
+  let timeline = null;
+  let audioCtx = null;
+  let startedAt = 0;
+  let cursor = 0; // moving index; never re-scan the whole timeline per frame
+  let active = false;
+
+  const zeros = () => {
+    const w = {};
+    for (const v of VISEMES) w[v] = 0;
+    return w;
+  };
+
+  function shut() {
+    active = false;
+    timeline = null;
+    cursor = 0;
+    stage.setExpressionWeights(zeros());
+    stage.setClockOffset(0);
+  }
+
+  return {
+    /**
+     * @param {Array<{t:number,dur:number,viseme:string,weight:number}>} tl
+     * @param {AudioContext} ctx
+     * @param {number} scheduledStart the value passed to source.start(), NOT Date.now()
+     */
+    start(tl, ctx, scheduledStart) {
+      // No timeline means no mouth movement at all. Phase 1 must never fall back to
+      // amplitude/RMS flapping - AVTR-02 disqualifies it explicitly.
+      if (!Array.isArray(tl) || tl.length === 0) {
+        shut();
+        return false;
+      }
+      timeline = tl;
+      audioCtx = ctx;
+      startedAt = scheduledStart;
+      cursor = 0;
+      active = true;
+      return true;
+    },
+
+    stop() {
+      shut();
+    },
+
+    isActive() {
+      return active;
+    },
+
+    tick(dt) {
+      if (!active || !audioCtx) return;
+
+      const t = audioCtx.currentTime - startedAt;
+      stage.setClockOffset(t);
+
+      while (cursor < timeline.length && t >= timeline[cursor].t + timeline[cursor].dur) {
+        cursor += 1;
+      }
+      const ev = cursor < timeline.length && t >= timeline[cursor].t ? timeline[cursor] : null;
+
+      // 'closed' (from N, cl and pau) matches none of the five names, so every target
+      // is 0 and the mouth shuts. That fall-through is deliberate, not accidental.
+      const k = Math.min(1, Math.max(0, dt) / ATTACK);
+      const weights = {};
+      let residual = 0;
+      for (const v of VISEMES) {
+        const target = ev && ev.viseme === v ? ev.weight : 0;
+        const cur = stage.vrm?.expressionManager?.getValue(v) ?? 0;
+        let next = cur + (target - cur) * k;
+        if (next < 0.001) next = 0;
+        weights[v] = next;
+        residual += next;
+      }
+      stage.setExpressionWeights(weights);
+
+      // Past the end of the timeline, keep ticking until the mouth has faded shut,
+      // then stand down so idle life owns the face again.
+      if (cursor >= timeline.length && residual === 0) shut();
+    },
+  };
+}

avatar/vrm-stage.js

@@ -0,0 +1,247 @@
+// avatar/vrm-stage.js
+//
+// THE RENDERER. It knows about three.js, VRM and DOM nodes, and about NOTHING else.
+// It receives a plain <canvas>, a URL string and an emit(name, data) callback. It never
+// looks at a host framework, never reads a global, and never touches the network except
+// to load its own modules and the VRM itself.
+//
+// That isolation is the point: swapping the host (inline component vs an embedded frame)
+// must change only the small stagePort object in a transport file, never this file.
+
+const THREE_URL = 'https://esm.sh/three@0.185.1';
+const LOADER_URL = 'https://esm.sh/three@0.185.1/examples/jsm/loaders/GLTFLoader.js';
+const VRM_URL = 'https://esm.sh/@pixiv/three-vrm@3.5.5?deps=three@0.185.1';
+
+// The five VRM 1.0 vowel expression presets. Verified present in avatar/assets/tutor.vrm.
+export const VISEME_NAMES = ['aa', 'ih', 'ou', 'ee', 'oh'];
+
+// Idle-life constants. Blink cadence is a human-plausible 1.8-5.8 s.
+const BLINK_MIN = 1.8;
+const BLINK_SPREAD = 4.0;
+const BLINK_DURATION = 0.12; // 120 ms triangular ramp 0 -> 1 -> 0
+const BREATH_PERIOD = 4.0;
+const BREATH_SPINE = 0.012; // rad
+const BREATH_BOB = 0.004; // metres of camera bob
+const SWAY_PERIOD = 9.0;
+const SWAY_HIPS = 0.02; // rad
+const THINK_TILT = 0.08; // rad
+const LISTEN_LEAN = 0.05; // rad
+
+/**
+ * Copy only structured-cloneable scalars out of a VRM meta block.
+ * `vrm.meta.thumbnailImage` is an HTMLImageElement, which cannot cross a frame
+ * boundary and cannot be serialised by a test harness. Plan 01-09 asserts the
+ * licence fields against LICENSES.md, so those must survive; the image must not.
+ */
+function plainMeta(meta) {
+  if (!meta) return null;
+  const out = {};
+  for (const k of Object.keys(meta)) {
+    const v = meta[k];
+    const kind = typeof v;
+    if (v === null || kind === 'string' || kind === 'number' || kind === 'boolean') {
+      out[k] = v;
+    } else if (Array.isArray(v) && v.every((x) => typeof x === 'string')) {
+      out[k] = v.slice();
+    }
+  }
+  return out;
+}
+
+/**
+ * Mount a VRM onto a canvas and start the render loop.
+ *
+ * @param {HTMLCanvasElement} canvasEl
+ * @param {string} vrmUrl
+ * @param {(name: string, data?: object) => void} emit
+ * @returns {Promise<object>} the stage handle
+ */
+export async function mountStage(canvasEl, vrmUrl, emit = () => {}) {
+  // One module graph. All three URLs resolve to the identical absolute
+  // https://esm.sh/three@0.185.1/es2022/three.mjs, so the browser's module map
+  // guarantees a single instance. Dropping ?deps= gives two instances and the VRM
+  // silently degrades to a T-posed glTF with no humanoid and no expressions.
+  // No import map: Gradio's frontend is an already-booted ES-module app, and a late
+  // import map throws "An import map is added after module script load was triggered."
+  const [THREE, { GLTFLoader }, { VRMLoaderPlugin, VRMUtils }] = await Promise.all([
+    import(THREE_URL),
+    import(LOADER_URL),
+    import(VRM_URL),
+  ]);
+
+  const host = canvasEl.parentElement || canvasEl;
+  const measure = () => ({
+    w: Math.max(1, host.clientWidth || canvasEl.clientWidth || 640),
+    h: Math.max(1, host.clientHeight || canvasEl.clientHeight || 480),
+  });
+
+  const renderer = new THREE.WebGLRenderer({ canvas: canvasEl, alpha: true, antialias: true });
+  renderer.setPixelRatio(Math.min(globalThis.devicePixelRatio || 1, 2));
+  let dims = measure();
+  renderer.setSize(dims.w, dims.h, false);
+  renderer.setClearAlpha(0);
+
+  const scene = new THREE.Scene();
+  const camera = new THREE.PerspectiveCamera(30, dims.w / dims.h, 0.1, 20);
+  camera.position.set(0, 1.35, 1.6);
+  const lookTarget = new THREE.Vector3(0, 1.3, 0);
+  camera.lookAt(lookTarget);
+
+  const keyLight = new THREE.DirectionalLight(0xffffff, 2.0);
+  keyLight.position.set(1, 1, 1);
+  scene.add(keyLight);
+  scene.add(new THREE.AmbientLight(0xffffff, 0.6));
+
+  const loader = new GLTFLoader();
+  loader.register((p) => new VRMLoaderPlugin(p));
+  const gltf = await loader.loadAsync(vrmUrl);
+  const vrm = gltf.userData.vrm;
+  if (!vrm) throw new Error('loaded file exposed no gltf.userData.vrm - is it a VRM?');
+
+  VRMUtils.combineSkeletons?.(gltf.scene);
+  VRMUtils.removeUnnecessaryJoints?.(gltf.scene);
+
+  // VRM 0.0 models face -Z and need a half turn; VRM 1.0 already faces +Z.
+  const metaVersion = String(vrm.meta?.metaVersion ?? '0');
+  if (metaVersion === '0') vrm.scene.rotation.y = Math.PI;
+  scene.add(vrm.scene);
+
+  // Surface the single-instance property as a number instead of only logging it,
+  // so 01-VALIDATION.md's "exactly one three.mjs resource entry" is a numeric assertion.
+  const threeInstanceCount = performance
+    .getEntriesByType('resource')
+    .filter((e) => e.name.includes('three.mjs')).length;
+
+  if (!vrm.expressionManager) {
+    emit('error', {
+      message: 'VRM has no expressionManager - check three instance count',
+      threeInstanceCount,
+    });
+  }
+
+  const debug = {
+    ready: false,
+    vrmMetaTitle: vrm.meta?.name ?? vrm.meta?.title ?? '',
+    vrmMeta: plainMeta(vrm.meta),
+    vrmSpecVersion: metaVersion,
+    threeInstanceCount,
+    currentVisemes: { aa: 0, ih: 0, ou: 0, ee: 0, oh: 0 },
+    blinkValue: 0,
+    breathValue: 0,
+    clockOffset: 0,
+    thinking: false,
+    listening: false,
+  };
+
+  const bone = (name) => vrm.humanoid?.getNormalizedBoneNode?.(name) ?? null;
+  const spine = bone('spine');
+  const hips = bone('hips');
+  const head = bone('head');
+  const rest = {
+    spineX: spine ? spine.rotation.x : 0,
+    hipsY: hips ? hips.rotation.y : 0,
+    headX: head ? head.rotation.x : 0,
+    headY: head ? head.rotation.y : 0,
+    cameraY: camera.position.y,
+  };
+
+  const clock = new THREE.Clock();
+  let elapsed = 0;
+  let nextBlinkAt = BLINK_MIN + Math.random() * BLINK_SPREAD;
+  let blinkPhase = -1; // >= 0 while a blink is in flight
+
+  // Idle life is ADDITIVELY COMPOSITED with speech and never switched off.
+  // There is deliberately no "idle vs talking" state machine.
+  function idle(dt) {
+    elapsed += dt;
+
+    if (blinkPhase < 0 && elapsed >= nextBlinkAt) blinkPhase = 0;
+    let blinkValue = 0;
+    if (blinkPhase >= 0) {
+      blinkPhase += dt;
+      const p = blinkPhase / BLINK_DURATION;
+      if (p >= 1) {
+        blinkPhase = -1;
+        nextBlinkAt = elapsed + BLINK_MIN + Math.random() * BLINK_SPREAD;
+      } else {
+        blinkValue = p < 0.5 ? p * 2 : (1 - p) * 2;
+      }
+    }
+    debug.blinkValue = blinkValue;
+    vrm.expressionManager?.setValue('blink', blinkValue);
+
+    const breath = Math.sin((elapsed * 2 * Math.PI) / BREATH_PERIOD);
+    debug.breathValue = breath;
+    if (spine) spine.rotation.x = rest.spineX + breath * BREATH_SPINE;
+    camera.position.y = rest.cameraY + breath * BREATH_BOB;
+
+    const sway = Math.sin((elapsed * 2 * Math.PI) / SWAY_PERIOD);
+    if (hips) hips.rotation.y = rest.hipsY + sway * SWAY_HIPS;
+
+    // Thinking and listening are postures layered on top of idle, not replacements
+    // for it, and both are visible with no audio playing.
+    if (head) {
+      head.rotation.x = rest.headX + (debug.thinking ? THINK_TILT : 0);
+      head.rotation.y = rest.headY + (debug.listening ? LISTEN_LEAN : 0);
+    }
+    vrm.expressionManager?.setValue('relaxed', debug.thinking ? 0.25 : 0);
+    camera.lookAt(lookTarget);
+  }
+
+  let onTick = null;
+  renderer.setAnimationLoop(() => {
+    const dt = clock.getDelta();
+    idle(dt);
+    if (onTick) onTick(dt);
+    vrm.update(dt); // MUST run after expression values are set, every frame
+    renderer.render(scene, camera);
+  });
+
+  const ro = new ResizeObserver(() => {
+    dims = measure();
+    camera.aspect = dims.w / dims.h;
+    camera.updateProjectionMatrix();
+    renderer.setSize(dims.w, dims.h, false);
+  });
+  ro.observe(host);
+
+  const handle = {
+    vrm,
+    scene,
+    camera,
+    renderer,
+    getDebug: () => ({ ...debug, currentVisemes: { ...debug.currentVisemes } }),
+    setExpressionWeights(weights) {
+      const em = vrm.expressionManager;
+      for (const v of VISEME_NAMES) {
+        const value = Number(weights?.[v] ?? 0);
+        debug.currentVisemes[v] = value;
+        em?.setValue(v, value);
+      }
+    },
+    setClockOffset(t) {
+      debug.clockOffset = t;
+    },
+    setThinking(b) {
+      debug.thinking = !!b;
+    },
+    setListening(b) {
+      debug.listening = !!b;
+    },
+    setOnTick(fn) {
+      onTick = typeof fn === 'function' ? fn : null;
+    },
+    dispose() {
+      ro.disconnect();
+      renderer.setAnimationLoop(null);
+    },
+  };
+
+  debug.ready = true;
+  emit('ready', {
+    vrmMetaTitle: debug.vrmMetaTitle,
+    vrmSpecVersion: metaVersion,
+    threeInstanceCount,
+  });
+  return handle;
+}