import re from dataclasses import replace from datetime import datetime, timezone from pathlib import Path from app.catalogue import build_record from app.model_detail import ModelDetail from app.recommend import CommunitySignal, HeuristicEngine, Query, sort_results from .test_parsing import QWEN3_8B_CONFIG NOW = datetime(2026, 9, 1, tzinfo=timezone.utc) ENGINE = HeuristicEngine(now=NOW) def rec(name, total, bits, downloads=1000, modified="2026-08-01T00:00:00Z", tags=None): return build_record({ "id": f"mlx-community/{name}", "downloads": downloads, "likes": 1, "lastModified": modified, "pipeline_tag": "text-generation", "safetensors": {"total": total}, "tags": (tags or []) + [f"{bits}-bit"], }) def family(): return [ rec("Fam-8B-2bit", 8e9, 2), rec("Fam-8B-3bit", 8e9, 3), rec("Fam-8B-4bit", 8e9, 4), rec("Fam-8B-6bit", 8e9, 6), rec("Fam-8B-8bit", 8e9, 8), ] def ranked(models, q, details=None, community=None): return [s.model.name for s in sort_results(ENGINE.score(models, q, details or {}, community or {}), "recommended")] def test_priority_changes_order(): ms = family() quality = ranked(ms, Query(ram_gb=64, priority="quality")) memory = ranked(ms, Query(ram_gb=64, priority="memory")) assert quality[0] == "Fam-8B-8bit" assert memory[0] == "Fam-8B-2bit" assert ranked(ms, Query(ram_gb=64, priority="balanced"))[0] == "Fam-8B-4bit" def test_ram_changes_order(): ms = family() big = ranked(ms, Query(ram_gb=64, priority="quality")) small = ranked(ms, Query(ram_gb=8, priority="quality")) assert big[0] == "Fam-8B-8bit" assert small[0] != "Fam-8B-8bit" # 8-bit 8B (~8.5 GB) can't load on an 8 GB Mac scored = {s.model.name: s for s in ENGINE.score(ms, Query(ram_gb=8, priority="quality"))} assert scored["Fam-8B-8bit"].memory.fit == "Unlikely" def test_context_changes_score(): m = rec("Fam-8B-4bit", 8e9, 4) d = {m.id: ModelDetail(m.id, QWEN3_8B_CONFIG, [], None, 0)} short = ENGINE.score([m], Query(ram_gb=16, context=4096), d)[0] long = ENGINE.score([m], Query(ram_gb=16, context=131072), d)[0] assert long.memory.total_gb > short.memory.total_gb assert long.score < short.score assert any("shorter maximum context" in r for r in long.reasons) def test_quant_changes_score_for_same_model(): a, b = rec("Fam-8B-4bit", 8e9, 4), rec("Fam-8B-3bit", 8e9, 3) q = Query(ram_gb=64, priority="balanced") s = {x.model.name: x.score for x in ENGINE.score([a, b], q)} assert s["Fam-8B-4bit"] > s["Fam-8B-3bit"] def test_reasons_present_and_no_perf_claims(): for s in ENGINE.score(family(), Query(ram_gb=16, priority="speed")): assert s.reasons text = " ".join(s.reasons).lower() assert "tok/s" not in text # no invented throughput without community data assert "depends" in text def test_unknown_ram_and_unknown_memory_degrade_gracefully(): m = build_record({"id": "mlx-community/mystery", "tags": []}) s = ENGINE.score([m], Query())[0] assert s.memory.total_gb is None and s.memory.fit is None assert any("isn't available" in r for r in s.reasons) def test_neutral_to_names_and_brands(): base = rec("Qwen3-8B-4bit", 8e9, 4, downloads=5000) renamed = replace(base, id="mlx-community/Qwen3-8B-OptiQ-4bit", name="Qwen3-8B-OptiQ-4bit") other_org = replace(base, id="someone/Qwen3-8B-4bit", name="Qwen3-8B-4bit") q = Query(ram_gb=24, priority="balanced", context=32768) scores = [ENGINE.score([m], q, {}, {})[0].score for m in (base, renamed, other_org)] assert scores[0] == scores[1] == scores[2] def test_engine_source_mentions_no_product_names(): src = (Path(__file__).resolve().parent.parent / "app" / "recommend.py").read_text().lower() code = re.sub(r'"""[\s\S]*?"""', "", src) # the module docstring may explain the rule for word in ("optiq", "unsloth", "lmstudio", "bartowski", "dwq"): assert word not in code def test_popularity_and_community_break_ties(): a = rec("A-8B-4bit", 8e9, 4, downloads=100) b = rec("B-8B-4bit", 8e9, 4, downloads=100_000) assert ranked([a, b], Query(ram_gb=32))[0] == "B-8B-4bit" comm = {a.id: CommunitySignal(feedback_count=30, positive_share=1.0, benchmark_count=20, median_generation_tps=40)} s = {x.model.name: x for x in ENGINE.score([a, b], Query(ram_gb=32), {}, comm)} assert s["A-8B-4bit"].components["community"] > 0 assert any("MLX benchmark" in r for r in s["A-8B-4bit"].reasons) def test_sorts(): old = rec("Old-8B-4bit", 8e9, 4, downloads=10**6, modified="2024-01-01T00:00:00Z") new = rec("New-8B-4bit", 8e9, 4, downloads=10, modified="2026-08-30T00:00:00Z") scored = ENGINE.score([old, new], Query(ram_gb=32)) assert sort_results(scored, "popular")[0].model.name == "Old-8B-4bit" assert sort_results(scored, "recent")[0].model.name == "New-8B-4bit" assert [s.model.name for s in sort_results(scored, "all")] == ["New-8B-4bit", "Old-8B-4bit"] def test_config_quant_overrides_misleading_name(): m = rec("Fam-9B-4bit", 9e9, 4) cfg = {"model_type": "x", "num_hidden_layers": 4, "num_attention_heads": 8, "head_dim": 64} d = {m.id: ModelDetail(m.id, cfg, [], 18_000_000_000, 0)} s = ENGINE.score([m], Query(ram_gb=36), d)[0] assert s.quant["bits"] == 16 and s.to_dict()["quant"]["label"] == "16-bit" assert any("config.json says 16-bit" in r for r in s.reasons) def test_likes_count_toward_popularity(): a = rec("A-8B-4bit", 8e9, 4, downloads=20_000) b = replace(rec("B-8B-4bit", 8e9, 4, downloads=20_000), likes=150) assert ranked([a, b], Query(ram_gb=32))[0] == "B-8B-4bit" def test_capacity_prefers_larger_models_that_fit(): small = rec("Fam-1B-4bit", 1e9, 4, downloads=5000) mid = rec("Fam-8B-4bit", 8e9, 4, downloads=5000) huge = rec("Fam-70B-4bit", 70e9, 4, downloads=5000) order = ranked([small, mid, huge], Query(ram_gb=36, priority="balanced")) assert order[0] == "Fam-8B-4bit" and order[-1] == "Fam-70B-4bit" # 70B doesn't fit 36 GB assert ranked([small, mid, huge], Query(ram_gb=128, priority="quality"))[0] == "Fam-70B-4bit" def test_norms_make_scores_page_independent(): ms = [rec(f"M{i}-8B-4bit", 8e9, 4, downloads=10 ** i) for i in range(1, 6)] q = Query(ram_gb=32) norms = ENGINE.norms(ms, q) full = {s.model.id: s.score for s in ENGINE.score(ms, q, {}, {}, norms)} page = {s.model.id: s.score for s in ENGINE.score(ms[:2], q, {}, {}, norms)} assert all(abs(full[k] - v) < 1e-9 for k, v in page.items()) def _refs(comm, models): from app.recommend import perplexity_refs return perplexity_refs(comm, {m.id: m.base_model for m in models}, {m.id: m.quant_bucket for m in models}) def test_measured_perplexity_compares_same_width_peers_only(): a = rec("Base-4B-4bit", 4e9, 4, downloads=20_000, tags=["base_model:Org/Base-4B"]) b = rec("Base-4B-Mixed-4bit", 4e9, 4, downloads=20_000, tags=["base_model:Org/Base-4B"]) c = rec("Base-4B-8bit", 4e9, 8, downloads=20_000, tags=["base_model:Org/Base-4B"]) u = rec("Base-4B-Unmeasured-4bit", 4e9, 4, downloads=20_000, tags=["base_model:Org/Base-4B"]) comm = {a.id: CommunitySignal(median_perplexity=20.5, perplexity_count=1), b.id: CommunitySignal(median_perplexity=18.5, perplexity_count=1), c.id: CommunitySignal(median_perplexity=17.2, perplexity_count=1)} q = Query(ram_gb=36, priority="balanced") norms = ENGINE.norms([a, b, c, u], q) norms["ppl_refs"] = _refs(comm, [a, b, c, u]) s = {x.model.name: x for x in ENGINE.score([a, b, c, u], q, {}, comm, norms)} assert s["Base-4B-Mixed-4bit"].components["measured_quality"] > 0 > s["Base-4B-4bit"].components["measured_quality"] # measuring is neutral in expectation: the better 4-bit rises above the unmeasured one, the worse falls below assert s["Base-4B-Mixed-4bit"].score > s["Base-4B-Unmeasured-4bit"].score > s["Base-4B-4bit"].score # the 8-bit has no measured 8-bit peer, so its score isn't moved by the 4-bit measurements assert "measured_quality" not in s["Base-4B-8bit"].components assert any("lower than the 1 other measured 4-bit" in r for r in s["Base-4B-Mixed-4bit"].reasons) def test_single_measurement_does_not_change_score(): a = rec("Solo-4B-4bit", 4e9, 4, tags=["base_model:Org/Solo"]) comm = {a.id: CommunitySignal(median_perplexity=12.0, perplexity_count=1)} q = Query(ram_gb=36) norms = ENGINE.norms([a], q) base = ENGINE.score([a], q, {}, {}, dict(norms, ppl_refs={}))[0] norms["ppl_refs"] = _refs(comm, [a]) s = ENGINE.score([a], q, {}, comm, norms)[0] assert "measured_quality" not in s.components assert any("no other 4-bit quantization" in r for r in s.reasons) assert abs(s.components["quant"] - base.components["quant"]) < 1e-9