WolfDavid commited on
Commit
969caf4
·
1 Parent(s): a60327b

feat(02-06): analyze/translate/languageInfo on the shared turn loop; tokens on the turn event

Browse files

- turn-loop.js: three host round trips implemented once so both transports get
them; requireBridge()/checkResult() share the missing-bridge, undefined and
{error} checks with dispatchTurn (whose setThinking-before-first-await order
is unchanged); the turn event carries tokens and the debug slice gains the
language keys, initialised at construction
- facade.js: AVATAR_SURFACE += analyze, translate, languageInfo
- tests/test_transport_seam.py: language surface declared once, lives only in
the shared module, debug keys present, tokens inside the turn event, every
bridge call packs one payload

avatar/facade.js CHANGED
@@ -33,6 +33,9 @@
33
  * unlockAudio () -> state string, SYNCHRONOUS: resumes the AudioContext inside the
34
  * caller's user gesture. Resolved from the stagePort so a host can
35
  * pre-unlock on any tap; the turn loop also calls it itself - plan 01-11
 
 
 
36
  */
37
  export const AVATAR_SURFACE = [
38
  'mount',
@@ -47,6 +50,9 @@ export const AVATAR_SURFACE = [
47
  'dispatchTurn',
48
  'requestSlower',
49
  'unlockAudio',
 
 
 
50
  ];
51
 
52
  /**
 
33
  * unlockAudio () -> state string, SYNCHRONOUS: resumes the AudioContext inside the
34
  * caller's user gesture. Resolved from the stagePort so a host can
35
  * pre-unlock on any tap; the turn loop also calls it itself - plan 01-11
36
+ * analyze (text) -> {tokens} - learner lines are tokenised through the bridge (plan 02-06)
37
+ * translate (text, lineId) -> {text} - CPU MT on the host; cached per line by the host page (D-17)
38
+ * languageInfo () -> the host's language-asset facts (sizes, warm-up, container env)
39
  */
40
  export const AVATAR_SURFACE = [
41
  'mount',
 
50
  'dispatchTurn',
51
  'requestSlower',
52
  'unlockAudio',
53
+ 'analyze',
54
+ 'translate',
55
+ 'languageInfo',
56
  ];
57
 
58
  /**
avatar/turn-loop.js CHANGED
@@ -29,6 +29,13 @@
29
  // gate audio behind a gesture (iOS Safari; Chromium in a cross-origin embed) refuse a
30
  // resume() issued after the server round trip, which is exactly where the only other
31
  // resume() lives (audio-queue.js), and the owner's phone was silent for that reason.
 
 
 
 
 
 
 
32
 
33
  import { createAsr } from './asr.js';
34
  import { createMic, isHallucination, REJECT } from './mic.js';
@@ -81,6 +88,18 @@ export function createTurnLoop({
81
  lastTranscript: null,
82
  micRejectedCount: 0,
83
  micLastRejectReason: null,
 
 
 
 
 
 
 
 
 
 
 
 
84
  };
85
 
86
  // Set when a turn or a replay has been dispatched and its speech-start has not yet
@@ -176,6 +195,33 @@ export function createTurnLoop({
176
  return err instanceof Error ? err : new Error(message);
177
  }
178
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
179
  /**
180
  * One turn: text in, speech out. Resolves at speech-end with a summary of the turn.
181
  *
@@ -193,12 +239,11 @@ export function createTurnLoop({
193
  const busy = busyReason();
194
  if (busy) throw fail('dispatchTurn', new Error(busy));
195
 
196
- const bridge = getServer();
197
- if (!bridge || typeof bridge.turn !== 'function') {
198
- throw fail(
199
- 'dispatchTurn',
200
- new Error('no host bridge: the standalone stage has nothing to synthesise with')
201
- );
202
  }
203
 
204
  performance.mark('turn:dispatch');
@@ -208,18 +253,13 @@ export function createTurnLoop({
208
  emit('turn-start', { text: greeting ? null : text, speed, greeting });
209
 
210
  try {
211
- const directive = greeting
212
- ? await bridge.greeting()
213
- : await bridge.turn({ text: String(text ?? ''), speed });
 
214
  performance.mark('turn:response');
215
  const responseMs = Math.round(now() - dispatchedAt);
216
 
217
- // The host's client swallows an HTTP error into `undefined`, and the far side
218
- // answers a bad request with {error} rather than raising, so both are checked.
219
- if (directive === undefined || directive === null) {
220
- throw new Error('the host returned nothing for this turn - see its log');
221
- }
222
- if (directive.error) throw new Error(directive.error);
223
  if (!directive.audio_url || !Array.isArray(directive.timeline)) {
224
  throw new Error('the host returned a directive with no audio or no timeline');
225
  }
@@ -229,12 +269,17 @@ export function createTurnLoop({
229
  state.lastSubtitle = directive.subtitle ?? null;
230
  state.lastSpeed = directive.speed ?? speed;
231
  state.lastStageTimings = directive.timings ?? null;
 
 
 
 
232
  state.lastError = null;
233
  emit('turn', {
234
  turnId: state.lastTurnId,
235
  subtitle: state.lastSubtitle,
236
  speed: state.lastSpeed,
237
  timings: state.lastStageTimings,
 
238
  responseMs,
239
  greeting,
240
  });
@@ -311,6 +356,83 @@ export function createTurnLoop({
311
  return dispatchTurn(state.lastSubtitle, { speed: SLOWER_SPEED });
312
  },
313
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
314
  /**
315
  * pointerdown on the push-to-talk control. The host binds the control; the behaviour
316
  * is here so both transports get it from one implementation.
 
29
  // gate audio behind a gesture (iOS Safari; Chromium in a cross-origin embed) refuse a
30
  // resume() issued after the server round trip, which is exactly where the only other
31
  // resume() lives (audio-queue.js), and the owner's phone was silent for that reason.
32
+ //
33
+ // Plan 02-06: analyze / translate / languageInfo are host round trips implemented HERE so
34
+ // both transports get them; the host page renders tokens and caches translations (D-17);
35
+ // this module holds no per-line cache. Every bridge call packs ONE payload object (the
36
+ // host's bridge would turn two arguments into a list), and every result goes through the
37
+ // same two checks dispatchTurn applies: `undefined` (the host's client swallowed an HTTP
38
+ // error) and `{error}` (the far side refused the request without raising).
39
 
40
  import { createAsr } from './asr.js';
41
  import { createMic, isHallucination, REJECT } from './mic.js';
 
88
  lastTranscript: null,
89
  micRejectedCount: 0,
90
  micLastRejectReason: null,
91
+ // Plan 02-06: the language bridge. Tokens ride on every directive; the learner's own
92
+ // lines are tokenised through analyze(); translations are counted and timed here and
93
+ // cached by the host page, never by this module (D-17).
94
+ analyzeCount: 0,
95
+ lastAnalyzeMs: null,
96
+ lastTokenCount: null,
97
+ lastTokens: null,
98
+ translateCount: 0,
99
+ lastTranslateMs: null,
100
+ lastTranslation: null,
101
+ lastTranslateError: null,
102
+ languageInfo: null,
103
  };
104
 
105
  // Set when a turn or a replay has been dispatched and its speech-start has not yet
 
195
  return err instanceof Error ? err : new Error(message);
196
  }
197
 
198
+ /**
199
+ * The host bridge, or a throw that names the missing function. The standalone stage
200
+ * (asr-harness.html, stage.html) has no bridge at all; a host that forgot to register
201
+ * a server function has a bridge without the method. Both are the same failure to a
202
+ * caller.
203
+ */
204
+ function requireBridge(name, without) {
205
+ const bridge = getServer();
206
+ if (!bridge || typeof bridge[name] !== 'function') {
207
+ throw new Error(`no host bridge: the standalone stage has ${without}`);
208
+ }
209
+ return bridge;
210
+ }
211
+
212
+ /**
213
+ * The two checks every bridge result needs. The host's client swallows an HTTP error
214
+ * into `undefined`, and the far side answers a bad request with {error} rather than
215
+ * raising, so both are checked before a result is trusted.
216
+ */
217
+ function checkResult(result, what) {
218
+ if (result === undefined || result === null) {
219
+ throw new Error(`the host returned nothing for ${what} - see its log`);
220
+ }
221
+ if (result.error) throw new Error(result.error);
222
+ return result;
223
+ }
224
+
225
  /**
226
  * One turn: text in, speech out. Resolves at speech-end with a summary of the turn.
227
  *
 
239
  const busy = busyReason();
240
  if (busy) throw fail('dispatchTurn', new Error(busy));
241
 
242
+ let bridge;
243
+ try {
244
+ bridge = requireBridge(greeting ? 'greeting' : 'turn', 'nothing to synthesise with');
245
+ } catch (err) {
246
+ throw fail('dispatchTurn', err);
 
247
  }
248
 
249
  performance.mark('turn:dispatch');
 
253
  emit('turn-start', { text: greeting ? null : text, speed, greeting });
254
 
255
  try {
256
+ const directive = checkResult(
257
+ greeting ? await bridge.greeting() : await bridge.turn({ text: String(text ?? ''), speed }),
258
+ 'this turn'
259
+ );
260
  performance.mark('turn:response');
261
  const responseMs = Math.round(now() - dispatchedAt);
262
 
 
 
 
 
 
 
263
  if (!directive.audio_url || !Array.isArray(directive.timeline)) {
264
  throw new Error('the host returned a directive with no audio or no timeline');
265
  }
 
269
  state.lastSubtitle = directive.subtitle ?? null;
270
  state.lastSpeed = directive.speed ?? speed;
271
  state.lastStageTimings = directive.timings ?? null;
272
+ // Tokens are optional on the wire (an older host, or an analysis that failed and
273
+ // yielded []): the turn still speaks, the host page just has nothing to make tappable.
274
+ state.lastTokens = Array.isArray(directive.tokens) ? directive.tokens : [];
275
+ state.lastTokenCount = state.lastTokens.length;
276
  state.lastError = null;
277
  emit('turn', {
278
  turnId: state.lastTurnId,
279
  subtitle: state.lastSubtitle,
280
  speed: state.lastSpeed,
281
  timings: state.lastStageTimings,
282
+ tokens: state.lastTokens,
283
  responseMs,
284
  greeting,
285
  });
 
356
  return dispatchTurn(state.lastSubtitle, { speed: SLOWER_SPEED });
357
  },
358
 
359
+ /**
360
+ * Tokenise a learner line through the host's language core (plan 02-06). No audio is
361
+ * involved, so no unlockAudio() and no busy check: the host page calls this for the
362
+ * text the learner typed or the transcript ASR produced, then renders the tokens.
363
+ *
364
+ * @param {string} text
365
+ * @returns {Promise<{tokens: object[], timings: object|null}>}
366
+ */
367
+ async analyze(text) {
368
+ if (typeof text !== 'string' || !text.trim()) {
369
+ throw fail('analyze', new Error('analyze: text is required'));
370
+ }
371
+ const t0 = now();
372
+ try {
373
+ const bridge = requireBridge('analyze', 'no language core to analyze with');
374
+ const result = checkResult(await bridge.analyze({ text }), 'analyze');
375
+ state.analyzeCount += 1;
376
+ state.lastAnalyzeMs = Math.round(now() - t0);
377
+ return {
378
+ tokens: Array.isArray(result.tokens) ? result.tokens : [],
379
+ timings: result.timings ?? null,
380
+ };
381
+ } catch (err) {
382
+ throw fail('analyze', err);
383
+ }
384
+ },
385
+
386
+ /**
387
+ * Translate one line to English on the host's CPU (D-15). The host page caches the
388
+ * answer per line for the session (D-17) - this module deliberately does not, so the
389
+ * cache has exactly one owner. `lineId` is echoed back so the page can file it.
390
+ *
391
+ * @param {string} text
392
+ * @param {string|null} [lineId]
393
+ * @returns {Promise<{text: string, lineId: string|null, timings: object|null, ms: number}>}
394
+ */
395
+ async translate(text, lineId = null) {
396
+ if (typeof text !== 'string' || !text.trim()) {
397
+ throw fail('translate', new Error('translate: text is required'));
398
+ }
399
+ const t0 = now();
400
+ try {
401
+ const bridge = requireBridge('translate', 'no translator to translate with');
402
+ const result = checkResult(await bridge.translate({ text, line_id: lineId }), 'translate');
403
+ const ms = Math.round(now() - t0);
404
+ state.translateCount += 1;
405
+ state.lastTranslateMs = ms;
406
+ state.lastTranslation = result.text ?? null;
407
+ state.lastTranslateError = null;
408
+ return {
409
+ text: result.text,
410
+ lineId: result.line_id ?? lineId,
411
+ timings: result.timings ?? null,
412
+ ms,
413
+ };
414
+ } catch (err) {
415
+ state.lastTranslateError = String(err?.message ?? err);
416
+ throw fail('translate', err);
417
+ }
418
+ },
419
+
420
+ /**
421
+ * The host's language-asset facts (sizes, warm-up measurement, container env), read
422
+ * once through the bridge and kept on the debug slice so a deployed probe can assert
423
+ * on them after the fact.
424
+ */
425
+ async languageInfo() {
426
+ try {
427
+ const bridge = requireBridge('language_info', 'no language core to describe');
428
+ const info = checkResult(await bridge.language_info(), 'languageInfo');
429
+ state.languageInfo = info;
430
+ return info;
431
+ } catch (err) {
432
+ throw fail('languageInfo', err);
433
+ }
434
+ },
435
+
436
  /**
437
  * pointerdown on the push-to-talk control. The host binds the control; the behaviour
438
  * is here so both transports get it from one implementation.
tests/test_transport_seam.py CHANGED
@@ -20,6 +20,16 @@ TRANSPORTS = ["avatar.js", "avatar-iframe.js"]
20
  GRADIO_TOKENS = ["gradio", "gradio_api", "server.", "trigger("]
21
  AMPLITUDE_TOKENS = ["AnalyserNode", "getByteFrequencyData", "getFloatTimeDomainData"]
22
  TURN_SURFACE = ["startListening", "stopListening", "dispatchTurn", "requestSlower"]
 
 
 
 
 
 
 
 
 
 
23
  # Plan 01-07. Mic capture and ASR run in the PARENT document under both transports -
24
  # only rendering and audio playback live inside the iframe - so these modules must hang
25
  # off the shared turn loop, never off a transport.
@@ -126,6 +136,9 @@ def test_transports_only_do_plumbing(name):
126
  "dispatchTurn",
127
  "requestSlower",
128
  "unlockAudio",
 
 
 
129
  ],
130
  )
131
  def test_surface_member_is_declared_once(member):
@@ -467,3 +480,55 @@ def test_audio_state_is_published(name):
467
  """audioState is a published field on getDebug() from both transports and the
468
  standalone harness, so the strict-policy tests assert a number, not a silence."""
469
  assert "audioState" in src(name), f"{name} does not publish audioState"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20
  GRADIO_TOKENS = ["gradio", "gradio_api", "server.", "trigger("]
21
  AMPLITUDE_TOKENS = ["AnalyserNode", "getByteFrequencyData", "getFloatTimeDomainData"]
22
  TURN_SURFACE = ["startListening", "stopListening", "dispatchTurn", "requestSlower"]
23
+ # Plan 02-06. The language bridge: tokens for learner lines, CPU translation, asset facts.
24
+ # Host round trips implemented once in turn-loop.js so both transports get them.
25
+ LANGUAGE_SURFACE = ["analyze", "translate", "languageInfo"]
26
+ LANGUAGE_DEBUG_KEYS = [
27
+ "analyzeCount",
28
+ "lastTokens",
29
+ "translateCount",
30
+ "lastTranslateMs",
31
+ "languageInfo",
32
+ ]
33
  # Plan 01-07. Mic capture and ASR run in the PARENT document under both transports -
34
  # only rendering and audio playback live inside the iframe - so these modules must hang
35
  # off the shared turn loop, never off a transport.
 
136
  "dispatchTurn",
137
  "requestSlower",
138
  "unlockAudio",
139
+ "analyze",
140
+ "translate",
141
+ "languageInfo",
142
  ],
143
  )
144
  def test_surface_member_is_declared_once(member):
 
480
  """audioState is a published field on getDebug() from both transports and the
481
  standalone harness, so the strict-policy tests assert a number, not a silence."""
482
  assert "audioState" in src(name), f"{name} does not publish audioState"
483
+
484
+
485
+ # --------------------------------------------------------------- plan 02-06: the language bridge
486
+ #
487
+ # Three new host round trips. Same seam, same rule: declared once in AVATAR_SURFACE,
488
+ # implemented once in turn-loop.js, absent from both transports, and every bridge call
489
+ # packs ONE payload object because the host's bridge would turn two arguments into a list.
490
+
491
+
492
+ @pytest.mark.parametrize("member", LANGUAGE_SURFACE)
493
+ def test_language_surface_declared_once(member):
494
+ assert avatar_surface().count(member) == 1, (
495
+ f"{member} must appear exactly once in AVATAR_SURFACE"
496
+ )
497
+
498
+
499
+ @pytest.mark.parametrize("member", LANGUAGE_SURFACE)
500
+ def test_language_surface_lives_in_the_shared_module(member):
501
+ assert f"async {member}(" in src("turn-loop.js"), (
502
+ f"{member} must be implemented in avatar/turn-loop.js so BOTH transports get it"
503
+ )
504
+ for transport in TRANSPORTS:
505
+ assert member not in src(transport), (
506
+ f"{transport} mentions {member!r}; the language bridge belongs in avatar/turn-loop.js"
507
+ )
508
+
509
+
510
+ @pytest.mark.parametrize("key", LANGUAGE_DEBUG_KEYS)
511
+ def test_language_debug_keys_present(key):
512
+ """Initialised at construction, not on first use: the parity suite compares key sets."""
513
+ assert f"{key}:" in src("turn-loop.js"), (
514
+ f"__debug.{key} is not initialised in avatar/turn-loop.js"
515
+ )
516
+
517
+
518
+ def test_turn_event_carries_tokens():
519
+ """`tokens:` is inside the emit('turn', {...}) object literal, so the host page renders
520
+ the avatar's line from the same event it always used - no second round trip."""
521
+ s = src("turn-loop.js")
522
+ start = s.index("emit('turn', {")
523
+ end = s.index("});", start)
524
+ assert "tokens:" in s[start:end], "the turn event does not carry tokens"
525
+
526
+
527
+ def test_bridge_calls_take_one_payload():
528
+ """01-08's bridge trap: server.fn(a, b) arrives as fn([a, b]). One object per call."""
529
+ s = src("turn-loop.js")
530
+ assert "bridge.analyze({" in s
531
+ assert "bridge.translate({" in s
532
+ assert "bridge.language_info(" in s
533
+ assert "bridge.analyze(text" not in s, "analyze must pack its arguments into one object"
534
+ assert "bridge.translate(text" not in s, "translate must pack its arguments into one object"