import pytest from pydantic import ValidationError from app.catalogue import Catalogue, build_record from app.events import COLUMNS, ClientEvent, EventBatch, clean_notes, to_row SID = "0123456789abcdef" class FakeCatalogue: def __init__(self, recs): self.by_id = {r.id: r for r in recs} CAT = FakeCatalogue([ build_record({"id": "mlx-community/Qwen3-8B-4bit", "downloads": 42, "likes": 3, "safetensors": {"total": 8.19e9}, "tags": ["4-bit"], "pipeline_tag": "text-generation"}), build_record({"id": "mlx-community/Llama-3.3-70B-Instruct-4bit", "safetensors": {"total": 70e9}, "tags": ["4-bit"], "pipeline_tag": "text-generation"}), ]) def ev(**kw): return ClientEvent(**{"event_type": "search", "session_id": SID, **kw}) def test_session_row_has_all_columns_and_no_pii(): row = to_row(ev(event_type="session", model_family="Qwen", parameter_bucket="8-15B", quantization="4-bit", target_context=32768, priority="balanced", hardware_source="confirmed", hardware_memory_class=36, families_searched=["Qwen", "Gemma", "Qwen"], quants_searched=["4-bit"], distinct_queries=3, top_model="mlx-community/Qwen3-8B-4bit", top_model_score=88.2, top_model_fit="Comfortable", models_viewed=["mlx-community/Qwen3-8B-4bit"]), CAT) assert set(row) == set(COLUMNS) assert row["schema_version"] == 2 and row["event_type"] == "session" assert row["families_searched"] == ["Qwen", "Gemma"] # de-duplicated assert row["hardware_confirmed"] is True and row["suspicious_flags"] == [] assert row["timestamp"].endswith("Z") and "." not in row["timestamp"] for forbidden in ("ip", "user_agent", "email", "cookie"): assert forbidden not in row def test_feedback_gets_catalogue_facts(): row = to_row(ev(event_type="feedback", selected_model="mlx-community/Qwen3-8B-4bit", tried="yes", quality_rating="good"), CAT) assert row["hf_downloads_at_selection"] == 42 and row["quant_bits"] == 4 and row["outcome"] == "worked" assert row["model_family"] == "Qwen" and row["parameter_bucket"] == "8-15B" def test_legacy_clickstream_events_are_dropped_and_old_bench_name_is_accepted(): assert to_row(ev(event_type="search", model_family="Qwen"), CAT) is None assert to_row(ev(event_type="model_view", selected_model="mlx-community/Qwen3-8B-4bit"), CAT) is None row = to_row(ev(event_type="mlx_benchmark", selected_model="mlx-community/Qwen3-8B-4bit", generation_tps=40, benchmark_version="mlxbench-1"), CAT) assert row["event_type"] == "mlx_benchmark" and row["suspicious_flags"] == [] def test_session_lists_are_capped_and_checked(): with pytest.raises(ValidationError): ev(event_type="session", models_viewed=[f"mlx-community/m{i}" for i in range(11)]) with pytest.raises(ValidationError): ev(event_type="session", models_compared=["../etc/passwd"]) with pytest.raises(ValidationError): ev(event_type="session", families_searched=["a", "b", "c", "d", "e", "f"]) row = to_row(ev(event_type="session", models_clicked=["mlx-community/Nope-1B"]), CAT) assert "unknown_model_in_session" in row["suspicious_flags"] def test_latest_sessions_keeps_newest_row_per_visit(): from app.events import latest_sessions rows = [ {"event_type": "session", "session_id": "a" * 16, "timestamp": "2026-09-14T01:00:00Z", "distinct_queries": 1}, {"event_type": "feedback", "session_id": "a" * 16, "timestamp": "2026-09-14T01:00:30Z"}, {"event_type": "session", "session_id": "a" * 16, "timestamp": "2026-09-14T01:05:00Z", "distinct_queries": 4}, {"event_type": "session", "session_id": "b" * 16, "timestamp": "2026-09-14T01:05:00Z", "distinct_queries": 1}, {"event_type": "session", "session_id": "b" * 16, "timestamp": "2026-09-14T01:05:00Z", "distinct_queries": 2}, ] out = latest_sessions(rows) sessions = {r["session_id"]: r["distinct_queries"] for r in out if r["event_type"] == "session"} assert sessions == {"a" * 16: 4, "b" * 16: 2} # later arrival wins a same-second tie assert sum(1 for r in out if r["event_type"] == "feedback") == 1 and len(out) == 3 @pytest.mark.parametrize("bad", [ {"event_type": "delete_everything"}, {"event_type": "session", "compare_models": ["a/b"]}, {"event_type": "search", "session_id": "NOT-HEX"}, {"event_type": "search", "target_context": 12345}, {"event_type": "search", "hardware_memory_class": 17}, {"event_type": "search", "quantization": "7-bit"}, {"event_type": "feedback", "reported_tokens_per_second": -1}, {"event_type": "feedback", "reported_tokens_per_second": 99999}, {"event_type": "search", "selected_model": "../../etc/passwd"}, {"event_type": "search", "selected_model": "") assert e.notes == "" # stored as inert text; UI renders textContent only def test_flags_unknown_model_and_implausible_tps(): row = to_row(ev(event_type="mlx_benchmark", selected_model="mlx-community/Nope-1B", generation_tps=10, benchmark_version="1", benchmark_type="mlx_lm"), CAT) assert "unknown_model" in row["suspicious_flags"] row = to_row(ev(event_type="mlx_benchmark", selected_model="mlx-community/Llama-3.3-70B-Instruct-4bit", generation_tps=1500, benchmark_version="1", benchmark_type="mlx_lm"), CAT) assert "implausible_tps" in row["suspicious_flags"] row = to_row(ev(event_type="mlx_benchmark", selected_model="mlx-community/Qwen3-8B-4bit", generation_tps=45, benchmark_version="1", benchmark_type="mlx_lm", peak_memory_gb=5.1, reported_ram_gb=36), CAT) assert row["suspicious_flags"] == [] def test_flags_other_inconsistencies(): row = to_row(ev(event_type="mlx_benchmark", peak_memory_gb=40, reported_ram_gb=16), CAT) assert {"peak_memory_exceeds_ram", "incomplete_benchmark"} <= set(row["suspicious_flags"]) row = to_row(ev(event_type="feedback", tried="yes", quality_rating="good", failure_reason="too_slow"), CAT) assert "conflicting_feedback" in row["suspicious_flags"] row = to_row(ev(event_type="browser_benchmark", benchmark_type="mlx_lm"), CAT) assert "benchmark_type_mismatch" in row["suspicious_flags"] def test_quality_submission_validation(): ok = to_row(ev(event_type="mlx_benchmark", selected_model="mlx-community/Qwen3-8B-4bit", benchmark_type="mlx_lm", benchmark_version="mlxbench-1", perplexity=9.8, perplexity_stderr=0.2, eval_dataset="wikitext2-test-v1", eval_tokens=16368), CAT) assert ok["suspicious_flags"] == [] and ok["perplexity"] == 9.8 bad = to_row(ev(event_type="mlx_benchmark", selected_model="mlx-community/Qwen3-8B-4bit", benchmark_version="mlxbench-1", perplexity=5000, eval_dataset="wikitext2-test-v1", eval_tokens=16368), CAT) assert "implausible_perplexity" in bad["suspicious_flags"] inc = to_row(ev(event_type="mlx_benchmark", selected_model="mlx-community/Qwen3-8B-4bit", benchmark_version="mlxbench-1", perplexity=9.8), CAT) assert "incomplete_quality" in inc["suspicious_flags"] with pytest.raises(ValidationError): ev(event_type="mlx_benchmark", perplexity=0.5) with pytest.raises(ValidationError): ev(event_type="mlx_benchmark", perplexity=9.8, eval_dataset="my-own-text") def test_perplexity_from_old_bench_version_is_ignored(): from app.stats import community_signals row = dict(event_type="mlx_benchmark", selected_model="mlx-community/gemma-4-e2b-it-4bit", perplexity=5268.9, eval_dataset="wikitext2-test-v1", suspicious_flags=[]) sig = community_signals([{**row, "benchmark_version": "mlxbench-1"}, {**row, "benchmark_version": "mlxbench-2", "perplexity": 21.4}]) assert sig[row["selected_model"]].median_perplexity == 21.4 assert sig[row["selected_model"]].perplexity_count == 1 def test_quality_only_benchmark_is_clean(): row = to_row(ev(event_type="mlx_benchmark", selected_model="mlx-community/Qwen3-8B-4bit", benchmark_type="mlx_lm", benchmark_version="mlxbench-2", perplexity=9.8, perplexity_stderr=0.2, eval_dataset="wikitext2-test-v1", eval_tokens=16368, peak_memory_gb=5.1, reported_ram_gb=36, chip="Apple M3 Max"), CAT) assert row["suspicious_flags"] == [] and row["generation_tps"] is None