recipe/pipeline/nex_judge.py

#!/usr/bin/env python3
"""Write judgments.json for the Nex-N2.5-mini cards. Sentence structure is authored; every number is
computed from summary.json. A judgment whose inputs are missing is NOT emitted (the renderer then
shows JUDGMENT PENDING).  usage: nex_judge.py <summary.json> <judgments.json>"""
import json, math, statistics, sys

S = json.load(open(sys.argv[1])); T = S.get("tiers") or {}; J = {}
MiB = 1024 ** 2
UB = S.get("n_ubatch") if S.get("n_ubatch") is not None else 1024
SPEED_EQ = 3.0  # % — tiers closer than this are called speed-equivalent
TWINS = (("q106i", "q106"), ("q102i", "q102"), ("q103i", "q103"))
STD_TAGS = ("q106", "q102", "q103")
IMAT_TAGS = ("q106i", "q102i", "q103i")
NAMES = {"q106": "STRIX_LEAN", "q102": "COHERENT", "q103": "FAST",
         "q106i": "STRIX_LEAN", "q102i": "COHERENT", "q103i": "FAST"}


def pct(a, b):
    return (b - a) / a * 100


def sig(a, ea, b, eb):
    return abs(b - a) / math.sqrt(ea * ea + eb * eb)


def have(*v):
    return all(x is not None for x in v)


def bench(label):   # last row wins: a re-run supersedes an earlier row with the same label
    return next((x for x in reversed(S.get("bench") or []) if x["label"] == label), None)


def gate(label):
    return next((x for x in reversed(S.get("gates") or []) if x.get("label") == label), None)


def bn(tag, dev):
    return bench(f"n-{tag}-{dev}")


def twin_spread(key):
    """Largest |gap| between an imatrix file and its standard twin: a measured noise floor."""
    if not all(T.get(i, {}).get("same_tensor_types_as_standard") is True for i, _ in TWINS):
        return None
    vals = []
    for ti, ts in TWINS:
        for dev in ("rocm", "vk"):
            x, y = bn(ti, dev), bn(ts, dev)
            if not (x and y and x.get(key) is not None and y.get(key) is not None):
                return None
            vals.append(abs(pct(y[key], x[key])))
    return max(vals) if vals else None


TG_NOISE, PP_NOISE = twin_spread("tg_median"), twin_spread("pp_median")
DEC_EQ = max(SPEED_EQ, TG_NOISE) if TG_NOISE is not None else None
PRE_EQ = max(SPEED_EQ, PP_NOISE) if PP_NOISE is not None else None


def rel(g, what, eq):
    if abs(g) < eq:
        return f"{what} within {abs(g):.1f} % of"
    return f"{what} {abs(g):.1f} % {'faster' if g > 0 else 'slower'} than"


def cmp_line(a_tag, b_tag, a_nm, b_nm):
    """-> (text, a_advantage, a_disadvantage) or (None, None, None)."""
    ra, rb, va, vb = bn(a_tag, "rocm"), bn(b_tag, "rocm"), bn(a_tag, "vk"), bn(b_tag, "vk")
    if not all((ra, rb, va, vb)) or DEC_EQ is None or PRE_EQ is None:
        return None, None, None
    if not have(ra.get("tg_median"), rb.get("tg_median"), va.get("tg_median"), vb.get("tg_median"),
                ra.get("pp_median"), rb.get("pp_median"),
                ra.get("tg_min"), ra.get("tg_max"), rb.get("tg_min"), rb.get("tg_max")):
        return None, None, None
    g_r, g_v = pct(rb["tg_median"], ra["tg_median"]), pct(vb["tg_median"], va["tg_median"])
    g_p = pct(rb["pp_median"], ra["pp_median"])
    txt = (f"`{a_nm}` {rel(g_r, 'decodes', DEC_EQ)} `{b_nm}` on ROCm0 ({ra['tg_median']:.2f} vs "
           f"{rb['tg_median']:.2f} tok/s; per-run ranges {ra['tg_min']:.2f}–{ra['tg_max']:.2f} and "
           f"{rb['tg_min']:.2f}–{rb['tg_max']:.2f}) and {rel(g_v, 'decodes', DEC_EQ)[len('decodes '):]} it on Vulkan0 "
           f"({va['tg_median']:.2f} vs {vb['tg_median']:.2f}), and {rel(g_p, 'prefills', PRE_EQ)} it on ROCm0 "
           f"({ra['pp_median']:.0f} vs {rb['pp_median']:.0f} tok/s)")
    adv = g_p >= PRE_EQ or g_r >= DEC_EQ or g_v >= DEC_EQ
    dis = g_p <= -PRE_EQ or g_r <= -DEC_EQ or g_v <= -DEC_EQ
    return txt, adv, dis


NOISE_NOTE = ("" if DEC_EQ is None or PRE_EQ is None else
              f"Speed gaps below {DEC_EQ:.1f} % (decode) and {PRE_EQ:.1f} % (prefill) are called a tie: the larger of "
              f"{SPEED_EQ:.0f} % and the widest gap measured between files that do identical work per token (each imatrix "
              f"file and its standard twin: decode {TG_NOISE:.1f} %, prefill {PP_NOISE:.1f} %).")


def kld_cmp(a, b):
    """KLD of tag a against tag b -> (pct change b->a, sigma, word). 'lower'/'higher' only at >= 2 sigma."""
    x, y = T[a], T[b]
    d = pct(y["kld_mean"], x["kld_mean"])
    sg = sig(y["kld_mean"], y["kld_err"], x["kld_mean"], x["kld_err"])
    word = "within noise of" if sg < 2 else ("lower than" if d < 0 else "higher than")
    return d, sg, word


def kld_ready(tags):
    return all(have(T.get(k, {}).get("kld_mean"), T.get(k, {}).get("kld_err"),
                    T.get(k, {}).get("size_bytes")) for k in tags)


def kld_rank_text(tags):
    items = sorted(tags, key=lambda t: T[t]["kld_mean"])
    parts = []
    for i, tag in enumerate(items):
        x = T[tag]
        bit = f"`{NAMES[tag]}` {x['kld_mean']:.4f}"
        if i > 0:
            lo = T[items[0]]["kld_mean"]
            parts.append(f"{bit} ({pct(lo, x['kld_mean']):+.1f} % vs `{NAMES[items[0]]}`)")
        else:
            parts.append(bit)
    return ", ".join(parts)


def size_vs(a, b):
    da, db = T[a]["size_bytes"], T[b]["size_bytes"]
    if da == db:
        return f"`{NAMES[a]}` and `{NAMES[b]}` are the same size ({da / MiB:.0f} MiB)"
    smaller, larger = (a, b) if da < db else (b, a)
    return (f"`{NAMES[smaller]}` is {(T[larger]['size_bytes'] - T[smaller]['size_bytes']) / MiB:.0f} MiB "
            f"smaller than `{NAMES[larger]}`")


# ---------- quality provenance ----------
rep, rep_v, ref = S.get("repeat") or {}, S.get("repeat_vk") or {}, S.get("reference") or {}
if rep.get("result") == "MATCH" and rep.get("rows") is not None and have(ref.get("cpu_chunk1"), ref.get("vulkan0_chunk1")):
    both = rep_v.get("result") == "MATCH" and rep_v.get("rows") is not None
    J["quality_provenance"] = (
        f"Measured directly on these files, against BF16 logits computed **on the CPU** in the same session "
        f"(first-window perplexity {ref['cpu_chunk1']:.4f}; Vulkan0 gave {ref['vulkan0_chunk1']:.4f} for the same "
        f"window). Every file was graded on ROCm0 (the columns above) and again on Vulkan0. The STRIX_LEAN grade was run "
        f"twice {'on each backend' if both else 'on ROCm0'} and every per-chunk row matched: {rep['rows']} of "
        f"{rep['rows']} on ROCm0" + (f", {rep_v['rows']} of {rep_v['rows']} on Vulkan0." if both else ".")
        + " Why not the GPU for the reference: see [Known issues](#known-issues-and-limits).")

# ---------- quality by backend ----------
vk_rows = [(t, T[t]["vk"]) for t in STD_TAGS + IMAT_TAGS if (T.get(t) or {}).get("vk") and have(
    T[t].get("kld_mean"), T[t].get("kld_err"), T[t]["vk"].get("kld_mean"), T[t]["vk"].get("kld_err"))]
if len(vk_rows) == 6:
    parts_b, lower_vk, lower_rocm = [], 0, 0
    for t, v in vk_rows:
        a_, b_ = T[t]["kld_mean"], v["kld_mean"]
        sg_ = sig(a_, T[t]["kld_err"], b_, v["kld_err"])
        if sg_ >= 2:
            lower_vk += b_ < a_; lower_rocm += a_ < b_
        pc_ = f"{pct(a_, b_):+.1f}"
        pc_ = "0.0" if pc_ in ("+0.0", "-0.0") else pc_
        parts_b.append(f"{'imatrix ' if t.endswith('i') else ''}{NAMES[t]} {a_:.4f} / {b_:.4f} ({pc_} %, {sg_:.1f}σ)")
    verdict = ("The two backends agree within noise on every file." if not (lower_vk or lower_rocm) else
               f"Vulkan0's output is measurably closer to BF16 on {lower_vk} of 6 files and ROCm0's on {lower_rocm}.")
    J["backend_quality_note"] = (
        f"**Same files, same reference, graded on each backend** — KLD ROCm0 / Vulkan0: " + "; ".join(parts_b) + f". {verdict}")


# ---------- imatrix verdict ----------
def twin_kld_ok(st, im):
    keys = ("kld_mean", "kld_err", "kld_median", "kld_p99", "same_top_p")
    return have(*(T.get(st, {}).get(k) for k in keys), *(T.get(im, {}).get(k) for k in keys))


if all(twin_kld_ok(st, im) for im, st in TWINS):
    bits = []
    improved, worse = [], []
    for im, st in TWINS:
        nm = NAMES[st]
        a, b = T[st], T[im]
        dk = pct(a["kld_mean"], b["kld_mean"])
        sg = sig(a["kld_mean"], a["kld_err"], b["kld_mean"], b["kld_err"])
        dmed = pct(a["kld_median"], b["kld_median"])
        dp99 = pct(a["kld_p99"], b["kld_p99"])
        dtop = b["same_top_p"] - a["same_top_p"]
        if dk < 0 and sg >= 2:
            verb = "improves"
            improved.append(nm)
        elif dk > 0 and sg >= 2:
            verb = "is measurably worse than"
            worse.append(nm)
        else:
            verb = "is within noise of"
        bits.append(
            f"**{nm}** {verb} the standard file on mean KLD "
            f"({dk:+.1f} %, {sg:.1f}σ; {a['kld_mean']:.4f} → {b['kld_mean']:.4f}); "
            f"median {dmed:+.1f} %, 99th-pct {dp99:+.1f} %, top-1 {dtop:+.2f} pp")
    if len(improved) == 3:
        head = "**The imatrix measurably improves all three tiers.** "
    elif improved:
        head = f"**The imatrix measurably improves {', '.join(improved)}.** "
    else:
        head = "**The imatrix does not measurably improve any of the three tiers on this corpus.** "
    if worse:
        head += f"**It is measurably worse on {', '.join(worse)}.** "
    J["imat_verdict"] = head + "; ".join(bits) + "."


# ---------- speed_note (standard card) ----------
def backend_line(tag):
    r, v = bn(tag, "rocm"), bn(tag, "vk")
    if not have(r, v) or DEC_EQ is None or PRE_EQ is None:
        return None
    if not have(r.get("tg_median"), v.get("tg_median"), r.get("pp_median"), v.get("pp_median"),
                r.get("tg_min"), r.get("tg_max"), v.get("tg_min"), v.get("tg_max")):
        return None
    g_t, g_p = pct(v["tg_median"], r["tg_median"]), pct(v["pp_median"], r["pp_median"])
    return (f"`{NAMES[tag]}`: ROCm0 {rel(g_t, 'decodes', DEC_EQ)} Vulkan0 "
            f"({r['tg_median']:.2f} vs {v['tg_median']:.2f} tok/s; ranges "
            f"{r['tg_min']:.2f}–{r['tg_max']:.2f} / {v['tg_min']:.2f}–{v['tg_max']:.2f}) and "
            f"{rel(g_p, 'prefills', PRE_EQ)} it ({r['pp_median']:.0f} vs {v['pp_median']:.0f} tok/s)")


if DEC_EQ is not None and PRE_EQ is not None:
    lines = [backend_line(t) for t in STD_TAGS]
    cr, pr = bench("n-q106-rocm"), bench("n-q106-rocm-prose")
    cv, pv = bench("n-q106-vk"), bench("n-q106-vk-prose")
    prose = None
    if have(cr, pr, cr and cr.get("tg_median"), pr and pr.get("tg_median")):
        g = pct(cr["tg_median"], pr["tg_median"])
        prose = (f"STRIX_LEAN workload range on ROCm0: code {cr['tg_median']:.2f} tok/s vs prose "
                 f"{pr['tg_median']:.2f} ({rel(g, 'prose decodes', DEC_EQ)} code)")
        if have(cv, pv, cv and cv.get("tg_median"), pv and pv.get("tg_median")):
            gv = pct(cv["tg_median"], pv["tg_median"])
            prose += (f"; Vulkan0 code {cv['tg_median']:.2f} vs prose {pv['tg_median']:.2f} "
                      f"({rel(gv, 'prose decodes', DEC_EQ)} code)")
        prose += "."

        def _span(wl):
            v = [(b_.get("prompt_n_min") if b_.get("prompt_n_min") is not None else b_.get("prompt_n"),
                  b_.get("prompt_n_max") if b_.get("prompt_n_max") is not None else b_.get("prompt_n"))
                 for b_ in S.get("bench") or [] if b_.get("workload") == wl]
            v = [x for x in v if None not in x]
            if not v:
                return None
            lo, hi = min(a for a, _ in v), max(b for _, b in v)
            return f"{lo:,}" if lo == hi else f"{lo:,}–{hi:,}"
        sc, sp = _span("code"), _span("prose")
        if sc and sp:
            prose += (f" Prompt lengths: code {sc} tokens (the first 30,000 characters of `convert_hf_to_gguf.py` plus "
                      f"an instruction), prose {sp} tokens (the first 34,000 characters of wikitext-2 *train* plus a "
                      f"writing instruction).")
    if all(lines) and prose:
        J["speed_note"] = " ".join(x + "." for x in lines) + " " + prose + (f" {NOISE_NOTE}" if NOISE_NOTE else "")


# ---------- cache ----------
d = gate("n-c3-q106")
if d and d.get("all_reused") and d.get("all_aligned") and have(d.get("n"), d.get("L"), d.get("expect_cache_n"),
                                                                d.get("identical")):
    wm = [r_["warm_prompt_ms"] for r_ in d.get("rows") or [] if r_.get("warm_prompt_ms") is not None]
    cm = [r_["cold_prompt_ms"] for r_ in d.get("rows") or [] if r_.get("cold_prompt_ms") is not None]
    t_line = ""
    if wm and cm:
        t_line = (f" — median prefill **{statistics.median(wm) / 1000:.1f} s instead of "
                  f"{statistics.median(cm) / 1000:.1f} s** cold "
                  f"({statistics.median(cm) / statistics.median(wm):.1f}× faster)")
    J["cache_note"] = (
        f"In {d['n']} request pairs sharing a long prefix at one fixed prompt length of {d['L']:,} tokens, every "
        f"second request resumed from the checkpoint the first one left {4 + UB:,} tokens before its end — "
        f"**{d['expect_cache_n']:,} tokens reused "
        f"({d['expect_cache_n'] / d['L'] * 100:.0f} %), {d['L'] - d['expect_cache_n']:,} processed**{t_line}. "
        f"Each warm reply was byte-identical to a cold run of the same prompt in **{d['identical']}/{d['n']}** "
        f"exchanges.\n\n"
        f"llama-server processes the last `n_ubatch` + 4 tokens of every prompt as two batches so it can checkpoint "
        f"there ({UB:,} + 4 = {4 + UB:,} tokens with the `-ub {UB}` used in these measurements, where the server "
        f"default is `-ub 512`; [upstream PR #20288](https://github.com/ggml-org/llama.cpp/pull/20288)). A turn that "
        f"resumes from a checkpoint left by a prompt of a *different* length therefore splits its tail differently "
        f"from a cold run, and float rounding can flip a greedy token.")


# ---------- tools ----------
def _tools_ok(x):
    return bool(x) and have(x.get("passed"), x.get("total"), x.get("detail"))


tl = gate("n-tools-q106")                                                    # stock template
FX = [gate(l) for l in ("n-tools-q106-roff", "n-tools-q106-roff-r2", "n-tools-q106-roff-r3")]   # quick-start config
TD = S.get("tools_diag") or {}
TF = S.get("template_fix") or {}
if _tools_ok(tl) and all(_tools_ok(x) for x in FX) and have(
        TD.get("stock_on_leaks"), TD.get("stock_on_replies"), TD.get("stock_on_reasoning_extracted"),
        TD.get("nested_off_http500"), TD.get("nested_off_attempts")):
    misses, on_n, on_ok, off_n, off_ok = [], 0, 0, 0, 0
    for i, x in enumerate(FX, 1):
        for k, v in x["detail"].items():
            name, think = k.split("|think=")
            if think == "True":
                on_n += 1; on_ok += bool(v)
            else:
                off_n += 1; off_ok += bool(v)
            if not v:
                misses.append("`%s` with thinking %s (pass %d)" % (name, "on" if think == "True" else "off", i))
    fp = TD.get("flag_probes") or {}
    flag_txt = "; ".join("%s: %d of %d replies still had reasoning in `content`" % (lab, fp[key]["leaks"], fp[key]["n"])
                         for key, lab in (("fmt-deepseek", "`--reasoning-format deepseek`"),
                                          ("srv-kwargs-high", "`--chat-template-kwargs` with `reasoning_effort`"),
                                          ("reasoning-on", "`--reasoning on`")) if key in fp)
    J["tools_note"] = (
        "**Stock chat template: %d/%d.** Every thinking-on check failed. Re-run with the raw replies kept, %d of %d "
        "thinking-on replies carried the reasoning and a `</think>` in `content`, and %d had any "
        "`reasoning_content`. llama-server builds its reasoning parser by rendering the template with "
        "`enable_thinking` on and off; this template ignores `enable_thinking` (it switches on `reasoning_effort`), "
        "so the parser finds no reasoning markers and extracts nothing. Server switches did not help (%s). With "
        "thinking off, `nested-object` failed on an HTTP 500 — see [Known issues](#known-issues-and-limits).\n\n"
        "**With the included `%s` and `--reasoning off` (the quick start): %s over three passes of the same suite "
        "(%d/%d)** — thinking off %d/%d, thinking on %d/%d; the misses were %s. A pass requires a native "
        "`tool_calls` entry with the right arguments and no think tags in `content`. Each check is a single sample at "
        "the recommended temperature 0.7." % (
            tl["passed"], tl["total"], TD["stock_on_leaks"], TD["stock_on_replies"],
            TD["stock_on_reasoning_extracted"], flag_txt or "not measured", TF.get("file") or "—",
            ", ".join("%d/%d" % (x["passed"], x["total"]) for x in FX),
            sum(x["passed"] for x in FX), sum(x["total"] for x in FX), off_ok, off_n, on_ok, on_n,
            ", ".join(misses) if misses else "none"))


# ---------- the template fix: what each request option does (recommended configuration) ----------
PR = TF.get("probes_roff") or {}
if PR and all(_tools_ok(x) for x in FX):
    def _probe(prefix):
        rows_ = [v for k, v in PR.items() if k.split("|")[0] == prefix]
        return len(rows_), sum(1 for v in rows_ if v.get("leaks")), sum(1 for v in rows_ if v.get("reasoning_len"))
    on_ok = sum(bool(v) for x in FX for k, v in x["detail"].items() if k.endswith("|think=True"))
    on_n = sum(1 for x in FX for k in x["detail"] if k.endswith("|think=True"))
    off_ok = sum(bool(v) for x in FX for k, v in x["detail"].items() if k.endswith("|think=False"))
    off_n = sum(1 for x in FX for k in x["detail"] if k.endswith("|think=False"))
    lines = []
    for label, prefix, think, dest in (
            ("no `chat_template_kwargs`", "no-kwargs", "off (the server default with `--reasoning off`)", "—"),
            ("`\"enable_thinking\": false`", "enable_thinking=false", "off", "—"),
            ("`\"reasoning_effort\": \"none\"`", "reasoning_effort=none", "off", "—"),
            ("`\"reasoning_effort\": \"high\"`", "reasoning_effort=high", "on", "**`content`** — do not use"),
            ("`\"reasoning_effort\": \"medium\"`", "reasoning_effort=medium", "adaptive", "**`content`** — do not use")):
        n, leaks, _ = _probe(prefix)
        if n:
            lines.append("| %s | %s | %s | %d of %d replies with think tags in `content` |" % (label, think, dest, leaks, n))
    lines.insert(1 if lines else 0,
                 "| `\"enable_thinking\": true` | on | `reasoning_content` | tool suite with thinking on: %d/%d "
                 "(a pass requires no think tags in `content`) |" % (on_ok, on_n))
    J["template_note"] = (
        "Measured on the standard STRIX_LEAN file with the included template file and `--reasoning off` (greedy "
        "probes: a direct question, a "
        "one-word instruction and a tool request, each with the tool schema attached; tool suite: 3 passes):\n\n"
        "| request | thinking | reasoning ends up in | measured |\n| --- | --- | --- | --- |\n" + "\n".join(lines)
        + "\n\nSo: switch thinking with `enable_thinking` only. Thinking-off tool checks: %d/%d." % (off_ok, off_n))


# ---------- vision ----------
def vwhy(x):
    if x.get("server_died"):
        import re as _re
        errs = [_re.sub(r"^[0-9.]+ [IWE] (srv +)?", "", e).strip() for e in x.get("server_log_errors") or []]
        first = next((e for e in errs if "failed" in e.lower() or "error" in e.lower() or "abort" in e.lower()),
                     None) or (errs[0] if errs else None)
        return ("the server aborted" + (f" (`{first[:90]}`)" if first else ""))
    if x.get("error"):
        return f"the request failed ({x['error'][:80]})"
    exp = x.get("expected") or ""
    nexp = len(exp.split(",")) if exp else None
    hits = x.get("hits") or []
    if nexp:
        return f"the reply named {len(hits)} of {nexp} expected terms"
    return "the reply did not pass"


on, off = gate("n-vision-q106-faon"), gate("n-vision-q106-faoff")
if on and off:
    def vok(x):
        return x.get("result") == "PASS"
    if vok(on) and vok(off):
        J["vision_note"] = "✅ **Images work with `-fa on` and `-fa off`.**"
    elif vok(off) and not vok(on):
        J["vision_note"] = (f"⛔ **With `-fa on`, image requests fail** — {vwhy(on)}. With `-fa off` they work. "
                            f"**For image input, serve with `-fa off`.**")
        J["vision_quickstart_warning"] = ("> ⛔ **Images:** with `-fa on`, image requests fail on this build "
                                          "([measured](#vision)). If you send images, use `-fa off`.")
    elif vok(on) and not vok(off):
        J["vision_note"] = (f"⛔ **With `-fa off`, image requests fail** — {vwhy(off)}. With `-fa on` they work. "
                            f"**For image input, keep `-fa on` (the quick-start default).**")
    else:
        J["vision_note"] = (f"⛔ **Images failed in both `-fa` settings.** `-fa on`: {vwhy(on)}; "
                            f"`-fa off`: {vwhy(off)}.")
    vfx = gate("n-vision-q106-roff-faon")
    if vfx and "vision_note" in J:
        J["vision_note"] += (" The `-fa on` image test was repeated with the included template file and "
                             f"`--reasoning off` (the quick start): {'✅ passed' if vok(vfx) else '❌ ' + vwhy(vfx)}.")


# ---------- memory ----------
rows_sz = [r for r in (S.get("sizing") or []) if r.get("label") == "strix-lean"]
if rows_sz:
    def mem_cell(r, key):
        if r.get("result") == "LOAD_FAIL":
            return "did not load"
        v = r.get(key)
        return f"{v:.2f} GiB" if v is not None else "—"
    body = "\n".join(
        f"| {r['ctx']:,} | {mem_cell(r, 'footprint_loaded_gib')} | {mem_cell(r, 'footprint_after_8k_gib')} |"
        for r in rows_sz if r.get("ctx") is not None)
    J["memory_note"] = (
        "Measured footprint (drop in `MemAvailable`) of STRIX_LEAN with the vision projector, q8_0 KV cache, "
        "`-cram 512`, one slot, no draft head:\n\n"
        "| context | after load | after one request (30,000-character code prompt) |\n"
        "| ---: | ---: | ---: |\n" + body +
        "\n\nNo row was decode-benchmarked beyond that one request. Nothing beyond these rows was measured.")


# ---------- recommendations ----------
def recommend(lean, coh, fast):
    """STRIX_LEAN is the flagship tier; the data decides whether COHERENT's quality or FAST's speed is worth taking
    instead. -> (default tag, markdown) or (None, None) when an input is missing."""
    lc, lc_adv, lc_dis = cmp_line(lean, coh, "STRIX_LEAN", "COHERENT")
    fl, fl_adv, fl_dis = cmp_line(fast, lean, "FAST", "STRIX_LEAN")
    if lc is None or fl is None:
        return None, None
    L, C, F = T[lean], T[coh], T[fast]
    d_c, s_c, w_c = kld_cmp(coh, lean)
    more = (C["size_bytes"] - L["size_bytes"]) / MiB
    size_c = f"{abs(more):.0f} MiB {'more' if more > 0 else 'less'}"
    if w_c == "lower than" and lc_adv and not lc_dis:
        default = lean
        head = (f"**Start with `STRIX_LEAN`; take `COHERENT` if quality matters more than speed.** {lc}. `COHERENT`'s "
                f"KLD is {abs(d_c):.1f} % lower ({s_c:.1f}σ) for {size_c}.")
    elif w_c == "lower than":
        default = coh
        head = (f"**Start with `COHERENT`.** Its KLD is {abs(d_c):.1f} % lower than `STRIX_LEAN`'s ({s_c:.1f}σ) for "
                f"{size_c}, and the speed comparison does not clearly favour `STRIX_LEAN`: {lc}.")
    elif w_c == "within noise of" and lc_dis and not lc_adv:
        default = coh
        head = (f"**Start with `COHERENT`.** Its KLD is {w_c} `STRIX_LEAN`'s ({d_c:+.1f} %, {s_c:.1f}σ) and it is "
                f"measurably faster: {lc}. It costs {size_c}.")
    else:
        default = lean
        head = (f"**Start with `STRIX_LEAN`.** `COHERENT`'s KLD is {w_c} it ({d_c:+.1f} %, {s_c:.1f}σ) for "
                f"{size_c}; {lc}.")
    d_f, s_f, w_f = kld_cmp(fast, lean)
    kf = f"its KLD is {w_f} `STRIX_LEAN`'s ({F['kld_mean']:.4f} vs {L['kld_mean']:.4f}, {d_f:+.1f} %, {s_f:.1f}σ)"
    if fl_adv and not fl_dis:
        fast_txt = f"**Take `FAST` for speed:** {fl}; {kf}."
    else:
        fast_txt = f"`FAST` does not buy a clear speed gain here: {fl}; {kf}."
    sizes = f"{size_vs(lean, coh)}; {size_vs(lean, fast)}."
    body = (f"{head}\n\n{fast_txt}\n\nKLD order (lower is closer to BF16): {kld_rank_text((lean, coh, fast))}. "
            f"{sizes} {NOISE_NOTE}")
    return default, body


def imat_pointer():
    """Point the standard card at the imatrix repo; name only measurable (>= 2 sigma) twin improvements."""
    repo = S.get("model_repo_imat")
    if not repo:
        return ""
    link = f"[imatrix build](https://huggingface.co/{repo})"
    bits = []
    for im, st in TWINS:
        if not twin_kld_ok(st, im):
            return ""
        d, sg, w = kld_cmp(im, st)
        if w == "lower than":
            bits.append(f"`{NAMES[st]}` {T[st]['kld_mean']:.4f} → {T[im]['kld_mean']:.4f} ({d:+.1f} %, {sg:.1f}σ)")
    if bits:
        return f"**The {link} is measurably closer to BF16 at the same size:** " + "; ".join(bits) + "."
    return f"Importance-matrix twins of all three files: **{link}** (not measurably closer to BF16 on this corpus)."


if kld_ready(STD_TAGS) and DEC_EQ is not None and PRE_EQ is not None:
    default, body = recommend("q106", "q102", "q103")
    if default:
        ptr = imat_pointer()
        J["std_recommendation"] = body + (f"\n\n{ptr}" if ptr else "")
        J["std_default"] = T[default]["file"]

if kld_ready(IMAT_TAGS) and DEC_EQ is not None and PRE_EQ is not None:
    default_i, body_i = recommend("q106i", "q102i", "q103i")
    if default_i:
        cross = ""
        if kld_ready(("q102",)):
            d, sg, w = kld_cmp("q106i", "q102")
            cross = (f"\n\nFor scale: the imatrix `STRIX_LEAN`'s KLD is {w} the *standard* `COHERENT`'s "
                     f"({T['q106i']['kld_mean']:.4f} vs {T['q102']['kld_mean']:.4f}, {d:+.1f} %, {sg:.1f}σ), "
                     f"{size_vs('q106i', 'q102')}.")
        J["imat_recommendation"] = body_i + cross
        J["imat_default"] = T[default_i]["file"]

parts = []
for nm, ti, ts in (("STRIX_LEAN", "q106i", "q106"), ("COHERENT", "q102i", "q102"), ("FAST", "q103i", "q103")):
    for dev, dn in (("rocm", "ROCm0"), ("vk", "Vulkan0")):
        a_, b_ = bn(ti, dev), bn(ts, dev)
        if a_ and b_ and have(a_.get("tg_median"), b_.get("tg_median")):
            parts.append(f"{nm} on {dn} {a_['tg_median']:.2f} vs {b_['tg_median']:.2f} tok/s")
same = [T.get(i, {}).get("same_tensor_types_as_standard") for i, _ in TWINS]
_hk = [T.get(i, {}).get("header_keys_only_in_imatrix") for i, _ in TWINS]
_hs = [T.get(i, {}).get("header_keys_only_in_standard") for i, _ in TWINS]
HDR = ""
if all(x is not None for x in _hk + _hs) and not any(_hs) and len({tuple(x) for x in _hk}) == 1 and _hk[0]:
    HDR = ", all in the GGUF header, which carries %d extra keys in each imatrix file: %s" % (
        len(_hk[0]), ", ".join("`%s`" % k for k in _hk[0]))
deltas = [T.get(i, {}).get("file_size_delta_bytes") for i, _ in TWINS]
if len(parts) == 6 and all(x is True for x in same) and all(d is not None for d in deltas):
    J["imat_speed_note"] = (
        f"The imatrix changes scale values, not tensor types or sizes: tensor by tensor, each imatrix file has the "
        f"same names, types and byte sizes as its standard twin, so the work per token is the same (the files differ "
        f"in size by {deltas[0]} / {deltas[1]} / {deltas[2]} bytes{HDR})." + " "
        f"Measured speed still differs — imatrix vs standard: " + "; ".join(parts) + "."
        + (f" The largest of these decode gaps is {TG_NOISE:.1f} %; both cards call decode gaps below {DEC_EQ:.1f} % "
           f"a tie." if TG_NOISE is not None and DEC_EQ is not None else ""))


# ---------- known issues (model facts + protocol; always emit) ----------
ki = [
    "- **No MTP head.** The checkpoint's `config.json` declares `mtp_num_hidden_layers: 1`, but the weights contain "
    "**no** `mtp.*` tensors (1,026 tensors total). There is no multi-token-prediction head and no speculative decoding "
    "on these files — do not pass `--spec-type draft-mtp`.",
    "- **The stock chat template ignores `enable_thinking`, and llama-server cannot separate its reasoning** "
    "(see [Tool calling](#tool-calling)). Serve with the included `chat_template_enable_thinking.jinja` and "
    "`--reasoning off`, and switch thinking per request with `enable_thinking` — see "
    "[Reasoning controls](#reasoning-controls).",
    "- **Earlier assistant turns are re-rendered with their reasoning**, so multi-turn contexts grow faster than with "
    "templates that drop it.",
    "- **`llama-server`'s host-RAM prompt cache defaults to 8 GiB** (`-cram 8192`). On a shared box, set `-cram` explicitly.",
    "- Measured on Linux only (Ryzen AI Max+ 395, ROCm 7.2.4, unpatched `d3ca537`).",
]
_pn_vals = [b_.get("prompt_n_max") if b_.get("prompt_n_max") is not None else b_.get("prompt_n")
            for b_ in (S.get("bench") or [])]
_pn = max((v for v in _pn_vals if v is not None), default=None)   # longest prompt any timed decode followed
_sz = {r_.get("ctx"): r_ for r_ in (S.get("sizing") or []) if r_.get("label") == "strix-lean"}
_big = max(_sz) if _sz else None
_ctx_txt = ("" if _big is None else
            f" (a {_big:,}-token context was loaded in the memory test, not benchmarked)"
            if _sz[_big].get("result") != "LOAD_FAIL" else "")
for _c, _r in sorted(_sz.items()):
    if _r.get("result") == "LOAD_FAIL":
        ki.append(f"- **A {_c:,}-token context did not load** in the memory test (STRIX_LEAN + vision projector, "
                  f"q8_0 KV cache, one slot) on this box.")
ki.append(f"- **Not measured:** decode beyond a {_pn:,}-token prompt{_ctx_txt}, " if _pn is not None else
          "- **Not measured:** decode at long context, ")
ki[-1] += ("long-context quality, video input, concurrency above 1, and task-level accuracy. Perplexity/KLD measure "
           "next-token fidelity to BF16 on prose, not reasoning or code correctness.")
if have(ref.get("hip_rocm0_chunk1"), ref.get("cpu_chunk1"), ref.get("hip_rocm0_final")) and have(S.get("bf16", {}).get("ppl")):
    ki.insert(0, f"- **Do not run the BF16 GGUF of this model on ROCm0 with this build.** `d3ca537` computes it wrong on "
                 f"that path: wikitext-2 perplexity {ref['hip_rocm0_final']:.1f} on ROCm0 vs {S['bf16']['ppl']:.2f} on the CPU "
                 f"(first window {ref['hip_rocm0_chunk1']:.1f} vs {ref['cpu_chunk1']:.2f}; also wrong with `-fa off`). "
                 f"The 4-bit files are not affected — their ROCm0 grades are in the table — and the BF16 file is not "
                 f"published here; the quality reference was computed on the CPU instead.")
if have(TD.get("nested_off_http500"), TD.get("nested_off_attempts")):
    ki.insert(1, "- **llama-server rejects a tool call whose required arguments are not in the order the schema lists "
                 "them** — HTTP 500, *The model produced output that does not match the expected peg-native format*: "
                 "this build's parser for the XML tool-call format expects required arguments in definition order. "
                 "On the standard STRIX_LEAN file with the stock template and thinking off, the `nested-object` "
                 "request (three required arguments) "
                 "hit it in the tool suite%s and in %d of %d repeats; the model had written a well-formed call with "
                 "the arguments reordered. Be ready to retry on this error." % (
                     "" if TD.get("gate_http500_logged") else " (not confirmed in the suite's server log)",
                     TD["nested_off_http500"], TD["nested_off_attempts"]))
_PR = (S.get("template_fix") or {}).get("probes_roff") or {}
_lk = [v for k, v in _PR.items() if k.split("|")[0] in ("reasoning_effort=high", "reasoning_effort=medium")]
if _lk:
    ki.insert(2, "- **With the included template, do not set `reasoning_effort` to `high` or `medium`** — the reasoning "
                 "goes back into `content` (%d of %d probe replies). `enable_thinking: true` is the way to turn "
                 "thinking on." % (sum(1 for v in _lk if v.get("leaks")), len(_lk)))
_seat = [r_ for r_ in (S.get("seats") or {}).values() if r_.get("thinking_reasoning_len") is not None]
_short = [r_ for r_ in _seat if not r_.get("thinking_reply") and r_.get("thinking_reasoning_len")]
if _short:
    ki.insert(3, "- **With thinking on, a very short answer can stay inside the think block.** Served as in the quick "
                 "start, `Reply with the single word: ready` (no tools, `enable_thinking: true`, temperature 0) came "
                 "back with the word in `reasoning_content` and an empty `content` on %d of %d FAST files tested.%s" % (
                     len(_short), len(_seat),
                     " With thinking off the same request returned `ready` in `content`."
                     if all((r_.get("default_reply") or "").strip().lower().startswith("ready") for r_ in _seat) else ""))
J["std_known_issues"] = "\n".join(ki)
J["imat_known_issues"] = "\n".join([
    "- Calibration text is general-purpose English/code (bartowski `calibration_datav3.txt`); an imatrix built from "
    "your own domain can do better on that domain.",
    "- Graded on wikitext-2 *test*, a different corpus from the calibration text. Improvements on reasoning/code "
    "tasks were not measured.",
] + ki)

json.dump(J, open(sys.argv[2], "w"), indent=2)
print("judgments written:", sorted(J))
for k, v in J.items():
    print(f"\n[{k}]\n{v}")