// avatar/lipsync.js // // The dumb player. All of the hard arithmetic (frame quantisation, banker's rounding, // pause-mora ordering) happens in Python and arrives here as a finished timeline. // This module's only job is to be on time. // // CLOCK DISCIPLINE: the clock is audioCtx.currentTime minus the SCHEDULED start. // Never a timer, never a frame counter. A frame counter drifts against the audio // hardware clock and the drift shows up as lip-sync sliding late over the last third // of a long sentence - the exact symptom this design exists to prevent. const VISEMES = ['aa', 'ih', 'ou', 'ee', 'oh']; const ATTACK = 0.05; // 50 ms cross-fade, inside the 40-60 ms band that reads as speech /** * @param {object} stage a handle returned by mountStage() * @returns {{start: Function, stop: Function, tick: Function, isActive: Function}} */ export function makePlayer(stage) { let timeline = null; let audioCtx = null; let startedAt = 0; let cursor = 0; // moving index; never re-scan the whole timeline per frame let active = false; const zeros = () => { const w = {}; for (const v of VISEMES) w[v] = 0; return w; }; function shut() { active = false; timeline = null; cursor = 0; stage.setExpressionWeights(zeros()); stage.setClockOffset(0); } return { /** * @param {Array<{t:number,dur:number,viseme:string,weight:number}>} tl * @param {AudioContext} ctx * @param {number} scheduledStart the value passed to source.start(), NOT Date.now() */ start(tl, ctx, scheduledStart) { // No timeline means no mouth movement at all. Phase 1 must never fall back to // amplitude/RMS flapping - AVTR-02 disqualifies it explicitly. if (!Array.isArray(tl) || tl.length === 0) { shut(); return false; } timeline = tl; audioCtx = ctx; startedAt = scheduledStart; cursor = 0; active = true; return true; }, stop() { shut(); }, isActive() { return active; }, tick(dt) { if (!active || !audioCtx) return; const t = audioCtx.currentTime - startedAt; stage.setClockOffset(t); while (cursor < timeline.length && t >= timeline[cursor].t + timeline[cursor].dur) { cursor += 1; } const ev = cursor < timeline.length && t >= timeline[cursor].t ? timeline[cursor] : null; // 'closed' (from N, cl and pau) matches none of the five names, so every target // is 0 and the mouth shuts. That fall-through is deliberate, not accidental. const k = Math.min(1, Math.max(0, dt) / ATTACK); const weights = {}; let residual = 0; for (const v of VISEMES) { const target = ev && ev.viseme === v ? ev.weight : 0; const cur = stage.vrm?.expressionManager?.getValue(v) ?? 0; let next = cur + (target - cur) * k; if (next < 0.001) next = 0; weights[v] = next; residual += next; } stage.setExpressionWeights(weights); // Past the end of the timeline, keep ticking until the mouth has faded shut, // then stand down so idle life owns the face again. if (cursor >= timeline.length && residual === 0) shut(); }, }; }