Reachy: the official head wobbler (speech tapper port) and silent emotion moves

Ports the official Reachy Mini head wobbler into the browser simulator and makes the Reachy emotion moves silent.

- `source/src/game/reachy/speechTapper.js`: a line-for-line port of `reachy_mini/motion/speech_tapper.py` (SwayRollRT: 16 kHz PCM, 20 ms frames, 50 ms hops, the VAD with hysteresis and attack/release, the loudness curve, the six oscillators with the daemon's seeded phases).
- `source/src/game/reachy/headWobbler.js`: runs the tapper offline on a decoded line (for wav lines) and, for the browser's speech synthesis (no samples), on a synthetic voice: voiced bursts at a natural syllable rate (~4 Hz, jittered) while the utterance plays, silence otherwise. Replaces the fixed sine sway.
- `reachyController.js`: the tapper's offsets are composed on the head target in the world frame before the IK exactly as the daemon's backend does (`compose_world_offset`: R = R_off R_abs, t = t_abs + t_off).
- `emotionController.js`: `reachySay` drives the wobbler while speaking; the Reachy emotion moves play SILENT (movement only, no .ogg), as the real robot player does with `play_move(sound=False)`: only Reachy's spoken voice is heard. The .ogg files are still loaded harmlessly by nothing; they can stay.

Tested in RemiFabre/microduck-reachy-simulator (same code at the root `src/` layout there); recorded videos of the conversation script and of a two-robot scene look right: small syllable-synchronous head motion while Reachy speaks, no wobble on the moves.

source/src/game/emotions/emotionController.js

@@ -2,13 +2,16 @@ import { playMicroduckEmotion } from "../microduck/microduckEmotions.js";
 import { EMOTIONS, loadReachyEmotion } from "./emotions.js";
 import { playUrl } from "../audio.js";
 import { signed } from "../signed.js";
+import { createSyntheticSpeech } from "../reachy/headWobbler.js";
 
 let selectedReachyVoice = null;
 
 export function reachySay(text, reachy = null) {
   try { globalThis.dispatchEvent?.(new CustomEvent("reachy-speech-start", { detail: { text: String(text) } })); } catch { /* recording hook is optional */ }
   if (!globalThis.speechSynthesis || !globalThis.SpeechSynthesisUtterance) return Promise.resolve();
-  reachy?.setWobbler?.(true, "speech");
+  // The official head wobbler (the daemon's speech tapper) on a synthetic voice: speech synthesis
+  // gives no samples to tap, so voiced bursts at a syllable rate stand in while the line plays.
+  const stopTaps = reachy?.setSpeechOffsets ? createSyntheticSpeech((o) => reachy.setSpeechOffsets(o)) : () => {};
   return new Promise((resolve) => {
     let spoken = false;
     const speak = () => {
@@ -26,7 +29,7 @@ export function reachySay(text, reachy = null) {
       // Same playful childlike profile for every sentence.
       utterance.pitch = 1.28;
       utterance.rate = 1.02;
-      const done = () => { reachy?.setWobbler?.(false, "speech"); resolve(); };
+      const done = () => { stopTaps(); resolve(); };
       utterance.onend = done;
       utterance.onerror = done;
       speechSynthesis.cancel();
@@ -85,9 +88,8 @@ export function createEmotionController({ runtime, reachy, setState }) {
 
   async function playReachy(emotion, definition) {
     const motion = await loadReachyEmotion(definition.motion);
-    playUrl(signed(`./robot/reachy/emotions/${definition.motion}.ogg`), { gain: 0.9 });
-    reachy.setWobbler(true, "emotion");
-    try { await reachy.playMotion(motion); }
-    finally { reachy.setWobbler(false, "emotion"); }
+    // Movement only: the real player plays the library moves silent (`play_move(sound=False)`),
+    // only Reachy's spoken voice is heard; the wobble follows the voice, not the move.
+    await reachy.playMotion(motion);
   }
 }

source/src/game/reachy/headWobbler.js

@@ -0,0 +1,55 @@
+// The official head wobbler, offline: the daemon feeds the speaker's PCM (resampled to 16 kHz)
+// through the speech tapper and applies each 50 ms hop's offsets at that hop's playback time.
+// Here the whole line is known in advance, so the hops are computed once from the decoded wav
+// and applied against the audio clock while it plays (same timeline, deterministic).
+import { SwayRollRT, HOP_MS } from "./speechTapper.js";
+
+export const ZERO_OFFSETS = Object.freeze({ x: 0, y: 0, z: 0, roll: 0, pitch: 0, yaw: 0 });
+const TAPPER_RATE = 16000;
+
+// AudioBuffer -> [{x,y,z,roll,pitch,yaw}] per 50 ms hop, from t = 0 of the buffer.
+export async function analyseLine(buffer) {
+  let pcm;
+  if (buffer.sampleRate === TAPPER_RATE && buffer.numberOfChannels === 1) pcm = buffer.getChannelData(0);
+  else {
+    const frames = Math.ceil(buffer.duration * TAPPER_RATE);
+    const off = new OfflineAudioContext(1, frames, TAPPER_RATE);
+    const src = off.createBufferSource(); src.buffer = buffer; src.connect(off.destination); src.start();
+    pcm = (await off.startRendering()).getChannelData(0);
+  }
+  return new SwayRollRT(TAPPER_RATE).feed(pcm);
+}
+
+// The hop in force `seconds` into the line (the daemon fires hop i at play_at + i * 50 ms).
+export function offsetsAt(hops, seconds) {
+  if (!hops?.length || seconds < 0) return ZERO_OFFSETS;
+  const i = Math.floor(seconds * 1000 / HOP_MS);
+  return i < hops.length ? hops[i] : ZERO_OFFSETS;
+}
+
+// Speech without samples (the browser's speech synthesis): feed the tapper a synthetic voice
+// instead of a fixed sine, so the wobble keeps the official character (VAD, syllable-rate taps,
+// loudness curve) while an utterance plays. Voiced bursts at a natural syllable rate, ~4 Hz with
+// jitter, 55-65 % duty, noise at a speech-like level; silence otherwise. One hop (50 ms) per tick.
+export function createSyntheticSpeech(apply) {
+  const tapper = new SwayRollRT(TAPPER_RATE);
+  const hop = Math.floor(TAPPER_RATE * HOP_MS / 1000);
+  let t = 0, nextSyllable = 0, voicedUntil = 0, level = 0.09, seed = 12345;
+  const rnd = () => { seed = (seed * 1664525 + 1013904223) >>> 0; return seed / 4294967296; };
+  const timer = setInterval(() => {
+    if (t >= nextSyllable) {
+      const period = 0.25 * (0.8 + 0.4 * rnd());        // ~4 Hz, jittered
+      voicedUntil = t + period * (0.55 + 0.1 * rnd());
+      nextSyllable = t + period;
+      level = 0.06 + 0.06 * rnd();                       // -25 .. -19 dBFS rms
+      if (rnd() < 0.12) nextSyllable += 0.3;             // a breath between phrases now and then
+    }
+    const voiced = t < voicedUntil;
+    const pcm = new Float32Array(hop);
+    const a = voiced ? level : 0.0007;
+    for (let i = 0; i < hop; i++) pcm[i] = a * (rnd() * 2 - 1) * 1.7;
+    for (const o of tapper.feed(pcm)) apply(o);
+    t += HOP_MS / 1000;
+  }, HOP_MS);
+  return () => { clearInterval(timer); apply(ZERO_OFFSETS); };
+}

source/src/game/reachy/reachyController.js

@@ -1,7 +1,7 @@
 import { MOTOR_NAMES, HOME, INITIAL_MOTOR_POSITIONS, poseToTarget, interpolateTarget, createReachyKinematics } from "./reachyKinematics.js";
 import { REACHY_PREFIX } from "./reachyWorld.js";
 import { abortError } from "../simulationTimeline.js";
-import { Matrix4 } from "three";
+import { Matrix4, Euler } from "three";
 
 export function createReachyController({ mujoco, getWorld, calibration, timeline, onState = () => {}, isLocked = () => false }) {
   const ik = createReachyKinematics(calibration);
@@ -9,6 +9,10 @@ export function createReachyController({ mujoco, getWorld, calibration, timeline
   let boundModel, addresses = [];
   let active = null, queue = [], speed = 1;
   const wobblerSources = new Set();
+  // The official head wobbler's per-hop offsets (reachy_mini/motion/speech_tapper.py, ported in
+  // ./speechTapper.js): {x, y, z} metres, {roll, pitch, yaw} radians, composed in the world frame
+  // before the IK exactly as the daemon does (R = R_off R_abs, t = t_abs + t_off).
+  let speechOffsets = { x: 0, y: 0, z: 0, roll: 0, pitch: 0, yaw: 0 };
   const publish = (extra = {}) => onState({ playing: !!active, queue: queue.map((q) => q.label), speed, ...extra });
   function apply(next) {
     const motors = ik(next); // validate the complete target before changing any motor
@@ -26,9 +30,16 @@ export function createReachyController({ mujoco, getWorld, calibration, timeline
       boundModel = model;
     }
     let output = motorTargets;
-    if (wobblerSources.size) {
-      const wobbleTarget = { ...target, head: target.head.clone().multiply(new Matrix4().makeRotationZ(Math.sin(performance.now() * 0.012) * 0.055)) };
-      output = ik(wobbleTarget);
+    const o = speechOffsets;
+    if (o.x || o.y || o.z || o.roll || o.pitch || o.yaw) {
+      const rot = new Matrix4().makeRotationFromEuler(new Euler(o.roll, o.pitch, o.yaw, "ZYX"));
+      const head = target.head.clone();
+      const e = head.elements;
+      const tx = e[12], ty = e[13], tz = e[14];
+      head.setPosition(0, 0, 0);
+      head.premultiply(rot);
+      head.setPosition(tx + o.x, ty + o.y, tz + o.z);
+      try { output = ik({ ...target, head }); } catch { output = motorTargets; }
     }
     addresses.forEach((id, i) => { data.ctrl[id] = output[i]; });
   }
@@ -62,6 +73,8 @@ export function createReachyController({ mujoco, getWorld, calibration, timeline
       if (value) wobblerSources.add(source);
       else wobblerSources.delete(source);
     },
+    // The official wobbler's offsets for this hop (see ./headWobbler.js).
+    setSpeechOffsets(offsets) { speechOffsets = offsets || { x: 0, y: 0, z: 0, roll: 0, pitch: 0, yaw: 0 }; },
     get target() { return target; },
     get motorTargets() { return [...motorTargets]; },
     stop() { active?.abort(); },

source/src/game/reachy/speechTapper.js

@@ -0,0 +1,89 @@
+// Port of the OFFICIAL Reachy Mini speech tapper (reachy_mini/motion/speech_tapper.py,
+// "SwayRollRT"): 16 kHz float PCM in, one sway dict per 50 ms hop out. Same tunables, same
+// VAD (hysteresis + attack / release), same loudness curve, same six oscillators; the seeded
+// oscillator phases are the numpy default_rng(7) values the daemon draws.
+export const FRAME_MS = 20;
+export const HOP_MS = 50;
+const SWAY_MASTER = 1.5;
+const SENS_DB_OFFSET = 4.0;
+const VAD_DB_ON = -35.0, VAD_DB_OFF = -45.0;
+const VAD_ATTACK_MS = 40, VAD_RELEASE_MS = 250;
+const ENV_FOLLOW_GAIN = 0.65;
+const SWAY_F_PITCH = 2.2, SWAY_A_PITCH_DEG = 4.5;
+const SWAY_F_YAW = 0.6, SWAY_A_YAW_DEG = 7.5;
+const SWAY_F_ROLL = 1.3, SWAY_A_ROLL_DEG = 2.25;
+const SWAY_F_X = 0.35, SWAY_A_X_MM = 4.5;
+const SWAY_F_Y = 0.45, SWAY_A_Y_MM = 3.75;
+const SWAY_F_Z = 0.25, SWAY_A_Z_MM = 2.25;
+const SWAY_DB_LOW = -46.0, SWAY_DB_HIGH = -18.0;
+const LOUDNESS_GAMMA = 0.9;
+const SWAY_ATTACK_MS = 50, SWAY_RELEASE_MS = 250;
+const ATTACK_FR = Math.max(1, Math.floor(VAD_ATTACK_MS / HOP_MS));
+const RELEASE_FR = Math.max(1, Math.floor(VAD_RELEASE_MS / HOP_MS));
+const SWAY_ATTACK_FR = Math.max(1, Math.floor(SWAY_ATTACK_MS / HOP_MS));
+const SWAY_RELEASE_FR = Math.max(1, Math.floor(SWAY_RELEASE_MS / HOP_MS));
+// np.random.default_rng(7).random() * 2 pi, six draws, in the daemon's order (pitch, yaw, roll, x, y, z).
+const PHASES = [3.927590651355011, 5.637360571650786, 4.873776931938056, 1.4150185072200883, 1.8860003910648933, 5.488698173149897];
+const rad = Math.PI / 180;
+
+function rmsDbfs(x) {
+  let s = 0;
+  for (let i = 0; i < x.length; i++) s += x[i] * x[i];
+  const rms = Math.sqrt(s / x.length + 1e-12);
+  return 20 * Math.log10(rms + 1e-12);
+}
+function loudnessGain(db) {
+  let t = (db + SENS_DB_OFFSET - SWAY_DB_LOW) / (SWAY_DB_HIGH - SWAY_DB_LOW);
+  t = Math.min(1, Math.max(0, t));
+  return LOUDNESS_GAMMA !== 1 ? t ** LOUDNESS_GAMMA : t;
+}
+
+export class SwayRollRT {
+  constructor(sampleRate = 16000) {
+    this.sampleRate = sampleRate | 0;
+    this.frame = Math.floor(this.sampleRate * FRAME_MS / 1000);
+    this.hop = Math.floor(this.sampleRate * HOP_MS / 1000);
+    this.reset();
+  }
+  reset() {
+    this.samples = new Float32Array(0); this.carry = new Float32Array(0);
+    this.vadOn = false; this.vadAbove = 0; this.vadBelow = 0;
+    this.swayEnv = 0; this.swayUp = 0; this.swayDown = 0; this.t = 0;
+  }
+  // Feed float32 mono PCM at this.sampleRate; returns [{pitch, yaw, roll, x, y, z}] per hop
+  // (radians and metres, the units the daemon composes on the head pose).
+  feed(pcm) {
+    if (!pcm.length) return [];
+    if (this.carry.length) { const c = new Float32Array(this.carry.length + pcm.length); c.set(this.carry); c.set(pcm, this.carry.length); this.carry = c; }
+    else this.carry = pcm;
+    const out = [];
+    while (this.carry.length >= this.hop) {
+      const hop = this.carry.subarray(0, this.hop);
+      this.carry = this.carry.subarray(this.hop);
+      if (this.samples.length) { const c = new Float32Array(this.samples.length + hop.length); c.set(this.samples); c.set(hop, this.samples.length); this.samples = c.subarray(Math.max(0, c.length - this.frame)); }
+      else this.samples = hop.subarray(Math.max(0, hop.length - this.frame)).slice();
+      if (this.samples.length < this.frame) { this.t += HOP_MS / 1000; continue; }
+      const db = rmsDbfs(this.samples);
+      if (db >= VAD_DB_ON) { this.vadAbove += 1; this.vadBelow = 0; if (!this.vadOn && this.vadAbove >= ATTACK_FR) this.vadOn = true; }
+      else if (db <= VAD_DB_OFF) { this.vadBelow += 1; this.vadAbove = 0; if (this.vadOn && this.vadBelow >= RELEASE_FR) this.vadOn = false; }
+      if (this.vadOn) { this.swayUp = Math.min(SWAY_ATTACK_FR, this.swayUp + 1); this.swayDown = 0; }
+      else { this.swayDown = Math.min(SWAY_RELEASE_FR, this.swayDown + 1); this.swayUp = 0; }
+      const up = this.swayUp / SWAY_ATTACK_FR, down = 1 - this.swayDown / SWAY_RELEASE_FR;
+      const target = this.vadOn ? up : down;
+      this.swayEnv += ENV_FOLLOW_GAIN * (target - this.swayEnv);
+      this.swayEnv = Math.min(1, Math.max(0, this.swayEnv));
+      const loud = loudnessGain(db) * SWAY_MASTER, env = this.swayEnv;
+      this.t += HOP_MS / 1000;
+      const w = 2 * Math.PI * this.t, g = loud * env;
+      out.push({
+        pitch: SWAY_A_PITCH_DEG * rad * g * Math.sin(w * SWAY_F_PITCH + PHASES[0]),
+        yaw: SWAY_A_YAW_DEG * rad * g * Math.sin(w * SWAY_F_YAW + PHASES[1]),
+        roll: SWAY_A_ROLL_DEG * rad * g * Math.sin(w * SWAY_F_ROLL + PHASES[2]),
+        x: SWAY_A_X_MM / 1000 * g * Math.sin(w * SWAY_F_X + PHASES[3]),
+        y: SWAY_A_Y_MM / 1000 * g * Math.sin(w * SWAY_F_Y + PHASES[4]),
+        z: SWAY_A_Z_MM / 1000 * g * Math.sin(w * SWAY_F_Z + PHASES[5]),
+      });
+    }
+    return out;
+  }
+}