tests/test_static_space.py

"""Browser regression checks for the static Space.

Install playwright and Chrome/Chromium, then run:
    python3 -m unittest discover -s tests -v
Set CHROME_PATH to override the installed browser executable.
"""
import csv
import io
import json
import os
from functools import partial
from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
import shutil
import threading
import unittest

from playwright.sync_api import sync_playwright

ROOT = Path(__file__).resolve().parents[1]
SNAPSHOT = json.loads((ROOT / 'front/data/benchmark.json').read_text())


class QuietHandler(SimpleHTTPRequestHandler):
    def log_message(self, *args):
        pass


class StaticSpaceTests(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.server = ThreadingHTTPServer(
            ('127.0.0.1', 0), partial(QuietHandler, directory=str(ROOT)))
        cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True)
        cls.thread.start()
        cls.base = 'http://127.0.0.1:' + str(cls.server.server_port)
        cls.playwright = sync_playwright().start()
        cls.browser = cls.playwright.chromium.launch(
            executable_path=os.environ.get('CHROME_PATH') or shutil.which('google-chrome')
            or shutil.which('chromium'), headless=True, timeout=20000)

    @classmethod
    def tearDownClass(cls):
        cls.browser.close()
        cls.playwright.stop()
        cls.server.shutdown()
        cls.server.server_close()
        cls.thread.join()

    def setUp(self):
        self.context = self.browser.new_context(viewport={'width': 1440, 'height': 1000})
        self.page = self.context.new_page()
        self.errors = []
        self.requests = []
        self.page.on('pageerror', lambda error: self.errors.append(str(error)))
        self.page.on('request', lambda request: self.requests.append((request.method, request.url)))

    def tearDown(self):
        self.context.close()
        self.assertEqual(self.errors, [])

    def open(self, state='ready'):
        self.page.goto(self.base, wait_until='domcontentloaded')
        self.page.wait_for_selector('#dataStatus[data-state="' + state + '"]')

    def refresh(self, state='ready', mobile=False):
        with self.page.expect_response('**/front/data/benchmark.json'):
            self.page.locator('#refreshMobileBtn' if mobile else '#refreshBtn').click()
        self.page.wait_for_selector('#dataStatus[data-state="' + state + '"]')

    def test_published_scores_filters_chart_and_export(self):
        self.open()
        self.assertEqual(self.page.locator('#kpiModels').inner_text(), str(len(SNAPSHOT['models'])))
        self.assertEqual(self.page.locator('#datasetFilters input').count(), len(SNAPSHOT['datasets']))
        self.assertEqual(self.page.locator('[data-tab="submit"], #submitForm').count(), 0)
        self.assertFalse(any('/api/' in url or method != 'GET' for method, url in self.requests))
        # Check rendered means against the preserved scoring rules, independently in Python.
        excluded = {'멀티 모달', '팩트 체크', 'K-Prism (Text, Intact)',
                    'K-Prism (Text, Perturbed)', 'K-Prism (Image, Intact)', 'K-Prism (Image, Perturbed)'}
        rows = self.page.locator('#tableBody tr').all()
        for model in SNAPSHOT['models']:
            cells = next(row.locator('td').all() for row in rows
                         if row.locator('td').nth(2).locator('.font-semibold').inner_text() == model['name'])
            scores = {score['dataset_name']: score['score'] for score in model['scores']}
            datasets = [d for d in SNAPSHOT['datasets'] if model['is_multimodal'] or d not in excluded]
            expected = sum(scores[d] for d in datasets) / len(datasets)
            self.assertAlmostEqual(float(cells[3].inner_text()), expected, delta=0.006)
        self.page.locator('button[data-fav]').first.click()
        self.page.wait_for_function("window.Chart && Chart.getChart('scoresChart').data.datasets.length === 2")
        self.page.locator('#searchInput').fill('qwen3-32b')
        self.assertEqual(self.page.locator('#kpiModels').inner_text(), '1')
        with self.page.expect_download() as download:
            self.page.locator('#exportCsvBtn').click()
        rows = list(csv.reader(io.StringIO(Path(download.value.path()).read_text())))
        self.assertEqual(len(rows), 2)
        self.assertEqual(rows[1][2], 'qwen3-32b')
        self.page.locator('#searchInput').fill('')
        self.page.locator('#providerSelect').select_option('OpenAI')
        self.assertEqual(self.page.locator('#kpiModels').inner_text(), '1')
        self.page.locator('#searchInput').fill('no-model-matches-this')
        self.assertIn('No models match', self.page.locator('#tableBody').inner_text())

    def test_preferences_and_repeated_refresh(self):
        self.open()
        self.page.locator('#clearAllBtn').click()
        self.refresh()
        self.refresh()
        self.assertEqual(self.page.locator('#datasetFilters input:checked').count(), 0)
        self.page.reload(wait_until='domcontentloaded')
        self.page.wait_for_selector('#dataStatus[data-state="ready"]')
        self.assertEqual(self.page.locator('#datasetFilters input:checked').count(), 0)
        self.page.locator('#selectAllBtn').click()
        self.page.locator('#searchInput').fill('qwen3-32b')
        self.page.reload(wait_until='domcontentloaded')
        self.page.wait_for_selector('#dataStatus[data-state="ready"]')
        self.assertEqual(self.page.locator('#searchInput').input_value(), 'qwen3-32b')
        self.assertEqual(self.page.locator('#kpiModels').inner_text(), '1')

    def test_initial_failure_never_invents_results_and_retry_recovers(self):
        self.page.route('**/front/data/benchmark.json', lambda route: route.fulfill(status=503, body='Unavailable'))
        self.open('error')
        self.assertEqual(self.page.locator('#kpiModels').inner_text(), '—')
        self.assertEqual(self.page.locator('button[data-fav]').count(), 0)
        self.assertTrue(self.page.locator('#exportCsvBtn').is_disabled())
        self.page.unroute('**/front/data/benchmark.json')
        self.refresh()
        self.assertEqual(self.page.locator('#kpiModels').inner_text(), str(len(SNAPSHOT['models'])))

    def test_malformed_refresh_preserves_previous_results(self):
        self.open()
        before = self.page.locator('#tableBody').inner_text()
        self.page.route('**/front/data/benchmark.json', lambda route: route.fulfill(
            content_type='application/json', body='{"models":[{"provider":"Broken"}]}'))
        self.refresh('error')
        self.assertEqual(self.page.locator('#tableBody').inner_text(), before)
        self.assertIn('Previously loaded', self.page.locator('#dataStatus').inner_text())

    def test_mobile_controls(self):
        self.page.set_viewport_size({'width': 390, 'height': 844})
        self.open()
        self.assertTrue(self.page.locator('#refreshMobileBtn').is_visible())
        self.assertLessEqual(self.page.evaluate('document.documentElement.scrollWidth'), 390)
        mean = self.page.locator('#tableBody tr').first.locator('td.mean-col').bounding_box()
        self.assertLessEqual(mean['x'] + mean['width'], 390)
        before = self.page.locator('html').get_attribute('class') or ''
        self.page.locator('#themeToggleMobile').click()
        self.assertNotEqual(self.page.locator('html').get_attribute('class') or '', before)
        self.refresh(mobile=True)
        self.assertEqual(self.page.locator('#kpiModels').inner_text(), str(len(SNAPSHOT['models'])))

    def test_blocked_storage_and_unavailable_chart_keep_table_usable(self):
        self.page.add_init_script("Object.defineProperty(window, 'localStorage', {get() { throw new Error('blocked'); }});")
        self.page.route('https://cdn.jsdelivr.net/npm/chart.js', lambda route: route.abort())
        self.open()
        self.assertTrue(self.page.locator('#chartUnavailable').is_visible())
        self.page.locator('#searchInput').fill('qwen3-32b')
        self.assertEqual(self.page.locator('#kpiModels').inner_text(), '1')
        self.page.locator('#themeToggle').click()

    def test_empty_results_and_untrusted_labels(self):
        self.page.route('**/front/data/benchmark.json', lambda route: route.fulfill(
            content_type='application/json', body='{"models":[],"datasets":[]}'))
        self.open()
        self.assertEqual(self.page.locator('#kpiModels').inner_text(), '0')
        self.assertIn('No results have been published', self.page.locator('#tableBody').inner_text())
        self.page.unroute('**/front/data/benchmark.json')
        label = '<img src=x onerror="window.injected=true">'
        payload = {'datasets': [label], 'models': [{
            'provider': label, 'name': label, 'repo': label, 'is_multimodal': True,
            'scores': [{'dataset_name': label, 'score': 42, 'metric_type': 'raw'}]}]}
        self.page.route('**/front/data/benchmark.json', lambda route: route.fulfill(
            content_type='application/json', body=json.dumps(payload)))
        self.refresh()
        self.page.locator('#selectAllBtn').click()
        self.assertEqual(self.page.locator('#tableBody img, #datasetFilters img').count(), 0)
        self.assertFalse(self.page.evaluate('Boolean(window.injected)'))
        self.assertIn(label, self.page.locator('#tableBody').inner_text())

    def test_legacy_array_validation_and_request_timeout(self):
        self.open()
        result = self.page.evaluate('''async () => {
          const {normalizeBenchmark, loadBenchmark} = await import('/front/data.mjs');
          const legacy = normalizeBenchmark([{provider: 'P', name: 'M', scores: [
            {dataset_name: 'Additional', score: 0}]}]);
          let invalid = 0;
          for (const payload of [null, {}, {models: [{provider:'P', name:'M', scores:[
            {dataset_name:'A', score:'99'}]}]}, {models:[], datasets:'invalid'}]) {
            try { normalizeBenchmark(payload); } catch { invalid++; }
          }
          window.fetch = (_url, options) => new Promise((_resolve, reject) => {
            options.signal.addEventListener('abort', () => reject(new DOMException('Aborted', 'AbortError')));
          });
          let timedOut = false;
          try { await loadBenchmark('/never-responds', 20); } catch (error) { timedOut = error.name === 'AbortError'; }
          return {datasets: legacy.datasets, score: legacy.models[0].scores[0].score, invalid, timedOut};
        }''')
        self.assertIn('Additional', result['datasets'])
        self.assertEqual(result['score'], 0)
        self.assertEqual(result['invalid'], 4)
        self.assertTrue(result['timedOut'])


if __name__ == '__main__':
    unittest.main()