"""Browser regression checks for the static Space. Install playwright and Chrome/Chromium, then run: python3 -m unittest discover -s tests -v Set CHROME_PATH to override the installed browser executable. """ import csv import io import json import os from functools import partial from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path import shutil import threading import unittest from playwright.sync_api import sync_playwright ROOT = Path(__file__).resolve().parents[1] SNAPSHOT = json.loads((ROOT / 'front/data/benchmark.json').read_text()) class QuietHandler(SimpleHTTPRequestHandler): def log_message(self, *args): pass class StaticSpaceTests(unittest.TestCase): @classmethod def setUpClass(cls): cls.server = ThreadingHTTPServer( ('127.0.0.1', 0), partial(QuietHandler, directory=str(ROOT))) cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True) cls.thread.start() cls.base = 'http://127.0.0.1:' + str(cls.server.server_port) cls.playwright = sync_playwright().start() cls.browser = cls.playwright.chromium.launch( executable_path=os.environ.get('CHROME_PATH') or shutil.which('google-chrome') or shutil.which('chromium'), headless=True, timeout=20000) @classmethod def tearDownClass(cls): cls.browser.close() cls.playwright.stop() cls.server.shutdown() cls.server.server_close() cls.thread.join() def setUp(self): self.context = self.browser.new_context(viewport={'width': 1440, 'height': 1000}) self.page = self.context.new_page() self.errors = [] self.requests = [] self.page.on('pageerror', lambda error: self.errors.append(str(error))) self.page.on('request', lambda request: self.requests.append((request.method, request.url))) def tearDown(self): self.context.close() self.assertEqual(self.errors, []) def open(self, state='ready'): self.page.goto(self.base, wait_until='domcontentloaded') self.page.wait_for_selector('#dataStatus[data-state="' + state + '"]') def refresh(self, state='ready', mobile=False): with self.page.expect_response('**/front/data/benchmark.json'): self.page.locator('#refreshMobileBtn' if mobile else '#refreshBtn').click() self.page.wait_for_selector('#dataStatus[data-state="' + state + '"]') def test_published_scores_filters_chart_and_export(self): self.open() self.assertEqual(self.page.locator('#kpiModels').inner_text(), str(len(SNAPSHOT['models']))) self.assertEqual(self.page.locator('#datasetFilters input').count(), len(SNAPSHOT['datasets'])) self.assertEqual(self.page.locator('[data-tab="submit"], #submitForm').count(), 0) self.assertFalse(any('/api/' in url or method != 'GET' for method, url in self.requests)) # Check rendered means against the preserved scoring rules, independently in Python. excluded = {'멀티 모달', '팩트 체크', 'K-Prism (Text, Intact)', 'K-Prism (Text, Perturbed)', 'K-Prism (Image, Intact)', 'K-Prism (Image, Perturbed)'} rows = self.page.locator('#tableBody tr').all() for model in SNAPSHOT['models']: cells = next(row.locator('td').all() for row in rows if row.locator('td').nth(2).locator('.font-semibold').inner_text() == model['name']) scores = {score['dataset_name']: score['score'] for score in model['scores']} datasets = [d for d in SNAPSHOT['datasets'] if model['is_multimodal'] or d not in excluded] expected = sum(scores[d] for d in datasets) / len(datasets) self.assertAlmostEqual(float(cells[3].inner_text()), expected, delta=0.006) self.page.locator('button[data-fav]').first.click() self.page.wait_for_function("window.Chart && Chart.getChart('scoresChart').data.datasets.length === 2") self.page.locator('#searchInput').fill('qwen3-32b') self.assertEqual(self.page.locator('#kpiModels').inner_text(), '1') with self.page.expect_download() as download: self.page.locator('#exportCsvBtn').click() rows = list(csv.reader(io.StringIO(Path(download.value.path()).read_text()))) self.assertEqual(len(rows), 2) self.assertEqual(rows[1][2], 'qwen3-32b') self.page.locator('#searchInput').fill('') self.page.locator('#providerSelect').select_option('OpenAI') self.assertEqual(self.page.locator('#kpiModels').inner_text(), '1') self.page.locator('#searchInput').fill('no-model-matches-this') self.assertIn('No models match', self.page.locator('#tableBody').inner_text()) def test_preferences_and_repeated_refresh(self): self.open() self.page.locator('#clearAllBtn').click() self.refresh() self.refresh() self.assertEqual(self.page.locator('#datasetFilters input:checked').count(), 0) self.page.reload(wait_until='domcontentloaded') self.page.wait_for_selector('#dataStatus[data-state="ready"]') self.assertEqual(self.page.locator('#datasetFilters input:checked').count(), 0) self.page.locator('#selectAllBtn').click() self.page.locator('#searchInput').fill('qwen3-32b') self.page.reload(wait_until='domcontentloaded') self.page.wait_for_selector('#dataStatus[data-state="ready"]') self.assertEqual(self.page.locator('#searchInput').input_value(), 'qwen3-32b') self.assertEqual(self.page.locator('#kpiModels').inner_text(), '1') def test_initial_failure_never_invents_results_and_retry_recovers(self): self.page.route('**/front/data/benchmark.json', lambda route: route.fulfill(status=503, body='Unavailable')) self.open('error') self.assertEqual(self.page.locator('#kpiModels').inner_text(), '—') self.assertEqual(self.page.locator('button[data-fav]').count(), 0) self.assertTrue(self.page.locator('#exportCsvBtn').is_disabled()) self.page.unroute('**/front/data/benchmark.json') self.refresh() self.assertEqual(self.page.locator('#kpiModels').inner_text(), str(len(SNAPSHOT['models']))) def test_malformed_refresh_preserves_previous_results(self): self.open() before = self.page.locator('#tableBody').inner_text() self.page.route('**/front/data/benchmark.json', lambda route: route.fulfill( content_type='application/json', body='{"models":[{"provider":"Broken"}]}')) self.refresh('error') self.assertEqual(self.page.locator('#tableBody').inner_text(), before) self.assertIn('Previously loaded', self.page.locator('#dataStatus').inner_text()) def test_mobile_controls(self): self.page.set_viewport_size({'width': 390, 'height': 844}) self.open() self.assertTrue(self.page.locator('#refreshMobileBtn').is_visible()) self.assertLessEqual(self.page.evaluate('document.documentElement.scrollWidth'), 390) mean = self.page.locator('#tableBody tr').first.locator('td.mean-col').bounding_box() self.assertLessEqual(mean['x'] + mean['width'], 390) before = self.page.locator('html').get_attribute('class') or '' self.page.locator('#themeToggleMobile').click() self.assertNotEqual(self.page.locator('html').get_attribute('class') or '', before) self.refresh(mobile=True) self.assertEqual(self.page.locator('#kpiModels').inner_text(), str(len(SNAPSHOT['models']))) def test_blocked_storage_and_unavailable_chart_keep_table_usable(self): self.page.add_init_script("Object.defineProperty(window, 'localStorage', {get() { throw new Error('blocked'); }});") self.page.route('https://cdn.jsdelivr.net/npm/chart.js', lambda route: route.abort()) self.open() self.assertTrue(self.page.locator('#chartUnavailable').is_visible()) self.page.locator('#searchInput').fill('qwen3-32b') self.assertEqual(self.page.locator('#kpiModels').inner_text(), '1') self.page.locator('#themeToggle').click() def test_empty_results_and_untrusted_labels(self): self.page.route('**/front/data/benchmark.json', lambda route: route.fulfill( content_type='application/json', body='{"models":[],"datasets":[]}')) self.open() self.assertEqual(self.page.locator('#kpiModels').inner_text(), '0') self.assertIn('No results have been published', self.page.locator('#tableBody').inner_text()) self.page.unroute('**/front/data/benchmark.json') label = '' payload = {'datasets': [label], 'models': [{ 'provider': label, 'name': label, 'repo': label, 'is_multimodal': True, 'scores': [{'dataset_name': label, 'score': 42, 'metric_type': 'raw'}]}]} self.page.route('**/front/data/benchmark.json', lambda route: route.fulfill( content_type='application/json', body=json.dumps(payload))) self.refresh() self.page.locator('#selectAllBtn').click() self.assertEqual(self.page.locator('#tableBody img, #datasetFilters img').count(), 0) self.assertFalse(self.page.evaluate('Boolean(window.injected)')) self.assertIn(label, self.page.locator('#tableBody').inner_text()) def test_legacy_array_validation_and_request_timeout(self): self.open() result = self.page.evaluate('''async () => { const {normalizeBenchmark, loadBenchmark} = await import('/front/data.mjs'); const legacy = normalizeBenchmark([{provider: 'P', name: 'M', scores: [ {dataset_name: 'Additional', score: 0}]}]); let invalid = 0; for (const payload of [null, {}, {models: [{provider:'P', name:'M', scores:[ {dataset_name:'A', score:'99'}]}]}, {models:[], datasets:'invalid'}]) { try { normalizeBenchmark(payload); } catch { invalid++; } } window.fetch = (_url, options) => new Promise((_resolve, reject) => { options.signal.addEventListener('abort', () => reject(new DOMException('Aborted', 'AbortError'))); }); let timedOut = false; try { await loadBenchmark('/never-responds', 20); } catch (error) { timedOut = error.name === 'AbortError'; } return {datasets: legacy.datasets, score: legacy.models[0].scores[0].score, invalid, timedOut}; }''') self.assertIn('Additional', result['datasets']) self.assertEqual(result['score'], 0) self.assertEqual(result['invalid'], 4) self.assertTrue(result['timedOut']) if __name__ == '__main__': unittest.main()