Capture simulator video and audio in sync

app/record-sim.mjs

@@ -1,22 +1,65 @@
-import { chromium } from '@playwright/test';
+import { chromium } from '@playwright/test';
 import fs from 'node:fs';
-const out='video-recordings'; fs.mkdirSync(out,{recursive:true});
-const browser=await chromium.launch({channel:'msedge',headless:true,args:['--enable-unsafe-swiftshader']});
-const context=await browser.newContext({viewport:{width:1440,height:900},recordVideo:{dir:out,size:{width:1440,height:900}}});
-const page=await context.newPage();
+
+const browser = await chromium.launch({ channel: 'msedge', headless: true, args: ['--enable-unsafe-swiftshader'] });
+const context = await browser.newContext({ viewport: { width: 1440, height: 900 } });
+const page = await context.newPage();
 await page.goto('http://127.0.0.1:5175/?boot=1');
-await page.waitForFunction(()=>window.__store?.getState().bootDone);
-await page.evaluate(()=>window.__gameApi.startEntrance());
-await page.waitForFunction(()=>window.rl&&!window.rl.inputLocked);
-await page.waitForFunction(()=>window.__audioCaptureStream?.active);
-await page.evaluate(()=>{
-  const r=new MediaRecorder(window.__audioCaptureStream,{mimeType:'audio/webm;codecs=opus'});
-  window.__simAudioChunks=[]; r.ondataavailable=e=>e.data.size&&window.__simAudioChunks.push(e.data); r.start(250); window.__simAudioRecorder=r;
+await page.waitForFunction(() => window.__store?.getState().bootDone);
+await page.evaluate(() => window.__gameApi.startEntrance());
+await page.waitForFunction(() => window.rl && !window.rl.inputLocked);
+await page.waitForFunction(() => window.__audioCaptureStream?.active);
+
+const script = JSON.parse(fs.readFileSync('public/scripts/reachy-microduck-conversation.json', 'utf8'));
+await page.evaluate(() => {
+  const canvas = document.querySelector('canvas');
+  const videoStream = canvas.captureStream(30);
+  const tracks = [...videoStream.getVideoTracks(), ...window.__audioCaptureStream.getAudioTracks()];
+  const recorder = new MediaRecorder(new MediaStream(tracks), { mimeType: 'video/webm;codecs=vp8,opus' });
+  window.__filmChunks = [];
+  recorder.ondataavailable = (event) => { if (event.data.size) window.__filmChunks.push(event.data); };
+  recorder.start(250);
+  window.__filmRecorder = recorder;
 });
-const script=JSON.parse(fs.readFileSync('public/scripts/reachy-microduck-conversation.json','utf8'));
-await page.evaluate(s=>{window.rl.chaseCam=false;window.__gameApi.script.load(s);window.__gameApi.script.run();},script);
-await page.evaluate(async()=>{const start=performance.now();await new Promise(resolve=>{const f=now=>{const t=(now-start)/1000;let x=0,y=0.82,z=2.8,tx=0,ty=0.25,tz=0;if(t<13){const a=t*.42;x=Math.sin(a)*2.35;z=Math.cos(a)*2.35;tx=0;ty=.24;}else if(t<28){const a=(t-13)*.5;x=1.55+Math.sin(a)*.35;z=2.05+Math.cos(a)*.25;y=.68;tx=0;ty=.2;}else{const a=(t-28)*.75;x=Math.sin(a)*2.45;z=Math.cos(a)*2.45;y=.88+Math.sin(t*.5)*.08;tx=0;ty=.23;}window.rl.camera.position.set(x,y,z);window.rl.controls.target.set(tx,ty,tz);window.rl.controls.update();if(t<55)requestAnimationFrame(f);else resolve();};requestAnimationFrame(f);});});
-await page.waitForTimeout(1500);
-const audioB64=await page.evaluate(()=>new Promise(resolve=>{const r=window.__simAudioRecorder;r.onstop=()=>{const b=new Blob(window.__simAudioChunks,{type:'audio/webm'});const fr=new FileReader();fr.onload=()=>resolve(fr.result.split(',')[1]);fr.readAsDataURL(b);};r.stop();}));
-fs.writeFileSync('sim-audio.webm',Buffer.from(audioB64,'base64'));
-const video=page.video(); await context.close(); const path=await video.path(); const final='reachy-microduck-conversation.webm'; fs.copyFileSync(path,final); console.log(final); await browser.close();
+await page.evaluate((s) => {
+  window.rl.chaseCam = false;
+  window.__gameApi.script.load(s);
+  window.__gameApi.script.run();
+}, script);
+
+await page.evaluate(async () => {
+  const start = performance.now();
+  await new Promise((resolve) => {
+    const frame = (now) => {
+      const t = (now - start) / 1000;
+      let x = 0, y = 0.82, z = 2.8;
+      if (t < 13) {
+        const a = t * 0.42; x = Math.sin(a) * 2.35; z = Math.cos(a) * 2.35;
+      } else if (t < 28) {
+        const a = (t - 13) * 0.5; x = 1.55 + Math.sin(a) * 0.35; z = 2.05 + Math.cos(a) * 0.25; y = 0.68;
+      } else {
+        const a = (t - 28) * 0.75; x = Math.sin(a) * 2.45; z = Math.cos(a) * 2.45; y = 0.88 + Math.sin(t * 0.5) * 0.08;
+      }
+      window.rl.camera.position.set(x, y, z);
+      window.rl.controls.target.set(0, 0.23, 0);
+      window.rl.controls.update();
+      if (t < 55) requestAnimationFrame(frame); else resolve();
+    };
+    requestAnimationFrame(frame);
+  });
+});
+await page.waitForTimeout(1200);
+const filmB64 = await page.evaluate(() => new Promise((resolve) => {
+  const recorder = window.__filmRecorder;
+  recorder.onstop = () => {
+    const blob = new Blob(window.__filmChunks, { type: 'video/webm' });
+    const reader = new FileReader();
+    reader.onload = () => resolve(reader.result.split(',')[1]);
+    reader.readAsDataURL(blob);
+  };
+  recorder.stop();
+}));
+fs.writeFileSync('reachy-microduck-synced.webm', Buffer.from(filmB64, 'base64'));
+await context.close();
+await browser.close();
+console.log('reachy-microduck-synced.webm');