Synchronize Reachy speech in recordings

app/record-sim.mjs

@@ -23,6 +23,9 @@ await page.evaluate(() => {
 });
 await page.evaluate((s) => {
   window.rl.chaseCam = false;
+  window.__speechMarks = [];
+  window.__filmStart = performance.now();
+  window.addEventListener('reachy-speech-start', (event) => window.__speechMarks.push({ text: event.detail?.text || '', at: (performance.now() - window.__filmStart) / 1000 }));
   window.__gameApi.script.load(s);
   window.__gameApi.script.run();
 }, script);
@@ -47,17 +50,18 @@ await page.evaluate(async () => {
   });
 });
 await page.waitForTimeout(1200);
-const filmB64 = await page.evaluate(() => new Promise((resolve) => {
+const capture = await page.evaluate(() => new Promise((resolve) => {
   const recorder = window.__filmRecorder;
   recorder.onstop = () => {
     const blob = new Blob(window.__filmChunks, { type: 'video/webm' });
     const reader = new FileReader();
-    reader.onload = () => resolve(reader.result.split(',')[1]);
+    reader.onload = () => resolve({ b64: reader.result.split(',')[1], marks: window.__speechMarks });
     reader.readAsDataURL(blob);
   };
   recorder.stop();
 }));
-fs.writeFileSync('reachy-microduck-synced.webm', Buffer.from(filmB64, 'base64'));
+fs.writeFileSync('reachy-microduck-synced.webm', Buffer.from(capture.b64, 'base64'));
+fs.writeFileSync('reachy-microduck-speech-marks.json', JSON.stringify(capture.marks, null, 2));
 await context.close();
 await browser.close();
 console.log('reachy-microduck-synced.webm');

app/src/game/emotions/emotionController.js

@@ -0,0 +1,80 @@
+import { playMicroduckEmotion } from "../microduck/microduckEmotions.js";
+import { EMOTIONS, loadReachyEmotion } from "./emotions.js";
+import { playUrl } from "../audio.js";
+import { signed } from "../signed.js";
+
+export function reachySay(text, reachy = null) {
+  try { globalThis.dispatchEvent?.(new CustomEvent("reachy-speech-start", { detail: { text: String(text) } })); } catch { /* recording hook is optional */ }
+  if (!globalThis.speechSynthesis || !globalThis.SpeechSynthesisUtterance) return Promise.resolve();
+  reachy?.setWobbler?.(true);
+  return new Promise((resolve) => {
+    const utterance = new SpeechSynthesisUtterance(text);
+    utterance.lang = "en-US";
+    const voices = speechSynthesis.getVoices?.() ?? [];
+    // Prefer a lower, traditionally masculine English system voice. Voice
+    // names vary by browser/OS, so keep a small ordered set of known names
+    // and fall back to any English voice when none is installed.
+    const maleHints = /daniel|alex|david|mark|arthur|thomas|george|google uk english male|microsoft (david|mark|ryan)/i;
+    const voice = voices.find((candidate) => maleHints.test(candidate.name) && /^en[-_]gb/i.test(candidate.lang))
+      ?? voices.find((candidate) => maleHints.test(candidate.name) && /^en[-_]/i.test(candidate.lang))
+      ?? voices.find((candidate) => /^en[-_]gb/i.test(candidate.lang))
+      ?? voices.find((candidate) => /^en[-_]/i.test(candidate.lang));
+    if (voice) utterance.voice = voice;
+    utterance.pitch = 0.58;
+    utterance.rate = 0.82;
+    const done = () => { reachy?.setWobbler?.(false); resolve(); };
+    utterance.onend = done;
+    utterance.onerror = done;
+    speechSynthesis.cancel();
+    speechSynthesis.speak(utterance);
+  });
+}
+
+export function createEmotionController({ runtime, reachy, setState }) {
+  const active = new Map();
+  const runTask = (kind, task, ownsRuntime = false) => {
+    if (active.has(kind)) return active.get(kind);
+    const promise = (async () => {
+      setState({ emotionBusy: true, emotionError: null });
+      if (ownsRuntime) runtime.start();
+      try { await task(); }
+      catch (error) { setState({ emotionError: error?.message || String(error) }); throw error; }
+      finally {
+        if (ownsRuntime) runtime.finish();
+        active.delete(kind);
+        setState({ emotionBusy: active.size > 0, emotionStage: active.size ? "active" : null });
+      }
+    })();
+    active.set(kind, promise);
+    return promise;
+  };
+  return {
+    get busy() { return !!active; },
+    play(emotion) {
+      const definition = EMOTIONS[emotion];
+      if (!definition) throw new Error(`Unknown emotion: ${emotion}`);
+      return runTask("sequence", async () => {
+        setState({ emotionStage: "microduck" });
+        await playMicroduckEmotion(emotion, runtime);
+        setState({ emotionStage: "reachy" });
+        await playReachy(emotion, definition);
+      }, true);
+    },
+    playMicroduck(emotion) {
+      if (!EMOTIONS[emotion]) return Promise.reject(new Error(`Unknown emotion: ${emotion}`));
+      return runTask("microduck", async () => { setState({ emotionStage: "microduck" }); await playMicroduckEmotion(emotion, runtime); }, true);
+    },
+    playReachy(emotion) {
+      const definition = EMOTIONS[emotion];
+      if (!definition) return Promise.reject(new Error(`Unknown emotion: ${emotion}`));
+      return runTask("reachy", async () => { setState({ emotionStage: "reachy" }); await playReachy(emotion, definition); });
+    },
+    say: (text) => reachySay(text, reachy),
+  };
+
+  async function playReachy(emotion, definition) {
+    const motion = await loadReachyEmotion(definition.motion);
+    playUrl(signed(`./robot/reachy/emotions/${definition.motion}.ogg`), { gain: 0.9 });
+    await reachy.playMotion(motion);
+  }
+}