source/scripts/verify_playground.cjs

/* Real-browser smoke test against an already-running playground; no fixtures. */
const assert = require('node:assert/strict');
const fs = require('node:fs/promises');
const path = require('node:path');
const { chromium } = require(process.env.PLAYWRIGHT_MODULE || 'playwright-core');

async function main() {
  const url = process.argv[2] || 'http://127.0.0.1:7466';
  const output = process.argv[3];
  if (!output || !process.env.CHROMIUM_PATH) {
    throw new Error('Provide URL, a new output directory and CHROMIUM_PATH.');
  }
  await fs.mkdir(output, { recursive: false });
  const browser = await chromium.launch({
    executablePath: process.env.CHROMIUM_PATH,
    headless: true,
    args: ['--disable-gpu', '--no-sandbox'],
  });
  const report = { started_utc: new Date().toISOString(), url, real_model_responses: [], checks: [] };
  try {
    const context = await browser.newContext({ viewport: { width: 1440, height: 1000 } });
    const page = await context.newPage();
    const errors = [];
    page.on('pageerror', error => errors.push(error.message));
    page.on('console', message => {
      if (message.type() === 'error' && /Content Security Policy|Refused to/.test(message.text())) {
        errors.push(message.text());
      }
    });
    await page.goto(url);
    await page.waitForFunction(() => !document.getElementById('model').disabled);
    const ids = await page.locator('#model option').evaluateAll(options => options.map(option => option.value));
    assert(ids.length >= 2, 'Expected multiple real models');
    const catalog = await (await page.request.get(new URL('/api/models', url).href)).json();
    const selectedId = await page.locator('#model').inputValue();
    const selectedModel = catalog.models.find(model => model.id === selectedId);
    assert(selectedModel, 'Selected model must exist in the live catalog');
    assert.equal(await page.locator('#calibration-badge').innerText(),
      selectedModel.calibrated ? 'Calibrated checkpoint' : 'Uncalibrated checkpoint');
    const revealPane = async (name) => {
      if (await page.locator('#workspace-tabs').isVisible()) {
        await page.locator(`#tab-${name}`).click();
      }
    };
    const assertNoDocumentOverflow = async (layout) => {
      const dimensions = await page.evaluate(() => ({
        width: document.documentElement.scrollWidth,
        height: document.documentElement.scrollHeight,
        viewportWidth: window.innerWidth,
        viewportHeight: window.innerHeight,
      }));
      assert(dimensions.width <= dimensions.viewportWidth, `${layout}: horizontal document overflow`);
      assert(dimensions.height <= dimensions.viewportHeight, `${layout}: vertical document overflow`);
    };
    const score = async (model) => {
      await revealPane('context');
      await page.selectOption('#model', model);
      await page.waitForFunction(() => !document.getElementById('submit').disabled);
      const responsePromise = page.waitForResponse(response =>
        new URL(response.url()).pathname === '/api/score' && response.request().method() === 'POST',
      { timeout: 300000 });
      await page.locator('#submit').click();
      const response = await responsePromise;
      const body = await response.json();
      assert.equal(response.status(), 200, JSON.stringify(body));
      assert.equal(body.model.id, model);
      assert.equal(body.probabilities.length, 3);
      assert(Math.abs(body.probabilities.reduce((sum, row) => sum + row.probability, 0) - 1) < 1e-5);
      assert(body.probabilities.every(row => Number.isFinite(row.probability) && row.probability >= 0));
      await revealPane('results');
      await page.locator('#results-content').waitFor({ state: 'visible' });
      assert.equal(await page.locator('.probability-item').count(), 3);
      assert.equal(await page.locator('#stale-notice').isVisible(), false);
      report.real_model_responses.push(body);
      return body;
    };
    for (const [index, model] of ids.entries()) {
      await score(model);
      if (!index) {
        await assertNoDocumentOverflow('Desktop');
        await page.screenshot({ path: path.join(output, 'desktop.png'), fullPage: true });
      }
    }
    report.checks.push('Real browser scoring and normalized distributions for every configured model');
    report.checks.push('Desktop layout has no horizontal or vertical document overflow');
    await revealPane('choices');
    await page.locator('#question').fill('Which choice fits the context best?');
    await revealPane('results');
    assert.equal(await page.locator('#stale-notice').isVisible(), true);
    report.checks.push('Input edits visibly mark old results as stale');
    await revealPane('choices');
    await page.locator('#options').fill('Duplicate\nDuplicate');
    await revealPane('context');
    await page.locator('#submit').click();
    assert.match(await page.locator('#error-message').innerText(), /duplicate|different/i);
    report.checks.push('Duplicate options rejected without scoring');
    await revealPane('context');
    await page.locator('#reset').click();
    await score(ids[0]);
    await context.grantPermissions(['clipboard-read', 'clipboard-write']);
    await page.locator('#copy-results').click();
    await page.waitForFunction(() => document.getElementById('copy-label').textContent === 'Copied');
    assert.equal(await page.locator('#copy-label').innerText(), 'Copied');
    const copied = JSON.parse(await page.evaluate(() => navigator.clipboard.readText()));
    assert.equal(copied.model.id, ids[0]);
    report.checks.push('Copy JSON returns the actual selected model result');
    const tokenError = await page.evaluate(async () => {
      const response = await fetch('/api/score', { method: 'POST', headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({ model: document.getElementById('model').value,
          text: 'A very long invented context. '.repeat(1500), question: 'Which option?', options: ['First', 'Second'] }) });
      return { status: response.status, body: await response.json() };
    });
    assert.equal(tokenError.status, 422);
    assert.equal(tokenError.body.code, 'token_limit');
    report.checks.push('Actual tokenizer rejects oversized input without truncation');
    await page.setViewportSize({ width: 390, height: 844 });
    await revealPane('context');
    await page.locator('#reset').click();
    await score(ids[0]);
    await assertNoDocumentOverflow('Mobile');
    await page.screenshot({ path: path.join(output, 'mobile.png'), fullPage: true });
    report.checks.push('Mobile layout has no horizontal or vertical document overflow and real results remain usable');
    assert.deepEqual(errors, []);
    report.checks.push('No browser script or Content Security Policy errors');
    report.status = 'passed';
  } catch (error) {
    report.status = 'failed';
    report.error = error.message;
    throw error;
  } finally {
    report.finished_utc = new Date().toISOString();
    await fs.writeFile(path.join(output, 'browser-verification.json'), JSON.stringify(report, null, 2) + '\n');
    await browser.close();
  }
  console.log(JSON.stringify({ status: report.status, models: report.real_model_responses.map(row => row.model.id), output }));
}

main().catch(error => { console.error(error.message); process.exitCode = 1; });