#!/usr/bin/env python3 """Write judgments.json for the Nex-N2.5-mini cards. Sentence structure is authored; every number is computed from summary.json. A judgment whose inputs are missing is NOT emitted (the renderer then shows JUDGMENT PENDING). usage: nex_judge.py """ import json, math, statistics, sys S = json.load(open(sys.argv[1])); T = S.get("tiers") or {}; J = {} MiB = 1024 ** 2 UB = S.get("n_ubatch") if S.get("n_ubatch") is not None else 1024 SPEED_EQ = 3.0 # % — tiers closer than this are called speed-equivalent TWINS = (("q106i", "q106"), ("q102i", "q102"), ("q103i", "q103")) STD_TAGS = ("q106", "q102", "q103") IMAT_TAGS = ("q106i", "q102i", "q103i") NAMES = {"q106": "STRIX_LEAN", "q102": "COHERENT", "q103": "FAST", "q106i": "STRIX_LEAN", "q102i": "COHERENT", "q103i": "FAST"} def pct(a, b): return (b - a) / a * 100 def sig(a, ea, b, eb): return abs(b - a) / math.sqrt(ea * ea + eb * eb) def have(*v): return all(x is not None for x in v) def bench(label): # last row wins: a re-run supersedes an earlier row with the same label return next((x for x in reversed(S.get("bench") or []) if x["label"] == label), None) def gate(label): return next((x for x in reversed(S.get("gates") or []) if x.get("label") == label), None) def bn(tag, dev): return bench(f"n-{tag}-{dev}") def twin_spread(key): """Largest |gap| between an imatrix file and its standard twin: a measured noise floor.""" if not all(T.get(i, {}).get("same_tensor_types_as_standard") is True for i, _ in TWINS): return None vals = [] for ti, ts in TWINS: for dev in ("rocm", "vk"): x, y = bn(ti, dev), bn(ts, dev) if not (x and y and x.get(key) is not None and y.get(key) is not None): return None vals.append(abs(pct(y[key], x[key]))) return max(vals) if vals else None TG_NOISE, PP_NOISE = twin_spread("tg_median"), twin_spread("pp_median") DEC_EQ = max(SPEED_EQ, TG_NOISE) if TG_NOISE is not None else None PRE_EQ = max(SPEED_EQ, PP_NOISE) if PP_NOISE is not None else None def rel(g, what, eq): if abs(g) < eq: return f"{what} within {abs(g):.1f} % of" return f"{what} {abs(g):.1f} % {'faster' if g > 0 else 'slower'} than" def cmp_line(a_tag, b_tag, a_nm, b_nm): """-> (text, a_advantage, a_disadvantage) or (None, None, None).""" ra, rb, va, vb = bn(a_tag, "rocm"), bn(b_tag, "rocm"), bn(a_tag, "vk"), bn(b_tag, "vk") if not all((ra, rb, va, vb)) or DEC_EQ is None or PRE_EQ is None: return None, None, None if not have(ra.get("tg_median"), rb.get("tg_median"), va.get("tg_median"), vb.get("tg_median"), ra.get("pp_median"), rb.get("pp_median"), ra.get("tg_min"), ra.get("tg_max"), rb.get("tg_min"), rb.get("tg_max")): return None, None, None g_r, g_v = pct(rb["tg_median"], ra["tg_median"]), pct(vb["tg_median"], va["tg_median"]) g_p = pct(rb["pp_median"], ra["pp_median"]) txt = (f"`{a_nm}` {rel(g_r, 'decodes', DEC_EQ)} `{b_nm}` on ROCm0 ({ra['tg_median']:.2f} vs " f"{rb['tg_median']:.2f} tok/s; per-run ranges {ra['tg_min']:.2f}–{ra['tg_max']:.2f} and " f"{rb['tg_min']:.2f}–{rb['tg_max']:.2f}) and {rel(g_v, 'decodes', DEC_EQ)[len('decodes '):]} it on Vulkan0 " f"({va['tg_median']:.2f} vs {vb['tg_median']:.2f}), and {rel(g_p, 'prefills', PRE_EQ)} it on ROCm0 " f"({ra['pp_median']:.0f} vs {rb['pp_median']:.0f} tok/s)") adv = g_p >= PRE_EQ or g_r >= DEC_EQ or g_v >= DEC_EQ dis = g_p <= -PRE_EQ or g_r <= -DEC_EQ or g_v <= -DEC_EQ return txt, adv, dis NOISE_NOTE = ("" if DEC_EQ is None or PRE_EQ is None else f"Speed gaps below {DEC_EQ:.1f} % (decode) and {PRE_EQ:.1f} % (prefill) are called a tie: the larger of " f"{SPEED_EQ:.0f} % and the widest gap measured between files that do identical work per token (each imatrix " f"file and its standard twin: decode {TG_NOISE:.1f} %, prefill {PP_NOISE:.1f} %).") def kld_cmp(a, b): """KLD of tag a against tag b -> (pct change b->a, sigma, word). 'lower'/'higher' only at >= 2 sigma.""" x, y = T[a], T[b] d = pct(y["kld_mean"], x["kld_mean"]) sg = sig(y["kld_mean"], y["kld_err"], x["kld_mean"], x["kld_err"]) word = "within noise of" if sg < 2 else ("lower than" if d < 0 else "higher than") return d, sg, word def kld_ready(tags): return all(have(T.get(k, {}).get("kld_mean"), T.get(k, {}).get("kld_err"), T.get(k, {}).get("size_bytes")) for k in tags) def kld_rank_text(tags): items = sorted(tags, key=lambda t: T[t]["kld_mean"]) parts = [] for i, tag in enumerate(items): x = T[tag] bit = f"`{NAMES[tag]}` {x['kld_mean']:.4f}" if i > 0: lo = T[items[0]]["kld_mean"] parts.append(f"{bit} ({pct(lo, x['kld_mean']):+.1f} % vs `{NAMES[items[0]]}`)") else: parts.append(bit) return ", ".join(parts) def size_vs(a, b): da, db = T[a]["size_bytes"], T[b]["size_bytes"] if da == db: return f"`{NAMES[a]}` and `{NAMES[b]}` are the same size ({da / MiB:.0f} MiB)" smaller, larger = (a, b) if da < db else (b, a) return (f"`{NAMES[smaller]}` is {(T[larger]['size_bytes'] - T[smaller]['size_bytes']) / MiB:.0f} MiB " f"smaller than `{NAMES[larger]}`") # ---------- quality provenance ---------- rep, rep_v, ref = S.get("repeat") or {}, S.get("repeat_vk") or {}, S.get("reference") or {} if rep.get("result") == "MATCH" and rep.get("rows") is not None and have(ref.get("cpu_chunk1"), ref.get("vulkan0_chunk1")): both = rep_v.get("result") == "MATCH" and rep_v.get("rows") is not None J["quality_provenance"] = ( f"Measured directly on these files, against BF16 logits computed **on the CPU** in the same session " f"(first-window perplexity {ref['cpu_chunk1']:.4f}; Vulkan0 gave {ref['vulkan0_chunk1']:.4f} for the same " f"window). Every file was graded on ROCm0 (the columns above) and again on Vulkan0. The STRIX_LEAN grade was run " f"twice {'on each backend' if both else 'on ROCm0'} and every per-chunk row matched: {rep['rows']} of " f"{rep['rows']} on ROCm0" + (f", {rep_v['rows']} of {rep_v['rows']} on Vulkan0." if both else ".") + " Why not the GPU for the reference: see [Known issues](#known-issues-and-limits).") # ---------- quality by backend ---------- vk_rows = [(t, T[t]["vk"]) for t in STD_TAGS + IMAT_TAGS if (T.get(t) or {}).get("vk") and have( T[t].get("kld_mean"), T[t].get("kld_err"), T[t]["vk"].get("kld_mean"), T[t]["vk"].get("kld_err"))] if len(vk_rows) == 6: parts_b, lower_vk, lower_rocm = [], 0, 0 for t, v in vk_rows: a_, b_ = T[t]["kld_mean"], v["kld_mean"] sg_ = sig(a_, T[t]["kld_err"], b_, v["kld_err"]) if sg_ >= 2: lower_vk += b_ < a_; lower_rocm += a_ < b_ pc_ = f"{pct(a_, b_):+.1f}" pc_ = "0.0" if pc_ in ("+0.0", "-0.0") else pc_ parts_b.append(f"{'imatrix ' if t.endswith('i') else ''}{NAMES[t]} {a_:.4f} / {b_:.4f} ({pc_} %, {sg_:.1f}σ)") verdict = ("The two backends agree within noise on every file." if not (lower_vk or lower_rocm) else f"Vulkan0's output is measurably closer to BF16 on {lower_vk} of 6 files and ROCm0's on {lower_rocm}.") J["backend_quality_note"] = ( f"**Same files, same reference, graded on each backend** — KLD ROCm0 / Vulkan0: " + "; ".join(parts_b) + f". {verdict}") # ---------- imatrix verdict ---------- def twin_kld_ok(st, im): keys = ("kld_mean", "kld_err", "kld_median", "kld_p99", "same_top_p") return have(*(T.get(st, {}).get(k) for k in keys), *(T.get(im, {}).get(k) for k in keys)) if all(twin_kld_ok(st, im) for im, st in TWINS): bits = [] improved, worse = [], [] for im, st in TWINS: nm = NAMES[st] a, b = T[st], T[im] dk = pct(a["kld_mean"], b["kld_mean"]) sg = sig(a["kld_mean"], a["kld_err"], b["kld_mean"], b["kld_err"]) dmed = pct(a["kld_median"], b["kld_median"]) dp99 = pct(a["kld_p99"], b["kld_p99"]) dtop = b["same_top_p"] - a["same_top_p"] if dk < 0 and sg >= 2: verb = "improves" improved.append(nm) elif dk > 0 and sg >= 2: verb = "is measurably worse than" worse.append(nm) else: verb = "is within noise of" bits.append( f"**{nm}** {verb} the standard file on mean KLD " f"({dk:+.1f} %, {sg:.1f}σ; {a['kld_mean']:.4f} → {b['kld_mean']:.4f}); " f"median {dmed:+.1f} %, 99th-pct {dp99:+.1f} %, top-1 {dtop:+.2f} pp") if len(improved) == 3: head = "**The imatrix measurably improves all three tiers.** " elif improved: head = f"**The imatrix measurably improves {', '.join(improved)}.** " else: head = "**The imatrix does not measurably improve any of the three tiers on this corpus.** " if worse: head += f"**It is measurably worse on {', '.join(worse)}.** " J["imat_verdict"] = head + "; ".join(bits) + "." # ---------- speed_note (standard card) ---------- def backend_line(tag): r, v = bn(tag, "rocm"), bn(tag, "vk") if not have(r, v) or DEC_EQ is None or PRE_EQ is None: return None if not have(r.get("tg_median"), v.get("tg_median"), r.get("pp_median"), v.get("pp_median"), r.get("tg_min"), r.get("tg_max"), v.get("tg_min"), v.get("tg_max")): return None g_t, g_p = pct(v["tg_median"], r["tg_median"]), pct(v["pp_median"], r["pp_median"]) return (f"`{NAMES[tag]}`: ROCm0 {rel(g_t, 'decodes', DEC_EQ)} Vulkan0 " f"({r['tg_median']:.2f} vs {v['tg_median']:.2f} tok/s; ranges " f"{r['tg_min']:.2f}–{r['tg_max']:.2f} / {v['tg_min']:.2f}–{v['tg_max']:.2f}) and " f"{rel(g_p, 'prefills', PRE_EQ)} it ({r['pp_median']:.0f} vs {v['pp_median']:.0f} tok/s)") if DEC_EQ is not None and PRE_EQ is not None: lines = [backend_line(t) for t in STD_TAGS] cr, pr = bench("n-q106-rocm"), bench("n-q106-rocm-prose") cv, pv = bench("n-q106-vk"), bench("n-q106-vk-prose") prose = None if have(cr, pr, cr and cr.get("tg_median"), pr and pr.get("tg_median")): g = pct(cr["tg_median"], pr["tg_median"]) prose = (f"STRIX_LEAN workload range on ROCm0: code {cr['tg_median']:.2f} tok/s vs prose " f"{pr['tg_median']:.2f} ({rel(g, 'prose decodes', DEC_EQ)} code)") if have(cv, pv, cv and cv.get("tg_median"), pv and pv.get("tg_median")): gv = pct(cv["tg_median"], pv["tg_median"]) prose += (f"; Vulkan0 code {cv['tg_median']:.2f} vs prose {pv['tg_median']:.2f} " f"({rel(gv, 'prose decodes', DEC_EQ)} code)") prose += "." def _span(wl): v = [(b_.get("prompt_n_min") if b_.get("prompt_n_min") is not None else b_.get("prompt_n"), b_.get("prompt_n_max") if b_.get("prompt_n_max") is not None else b_.get("prompt_n")) for b_ in S.get("bench") or [] if b_.get("workload") == wl] v = [x for x in v if None not in x] if not v: return None lo, hi = min(a for a, _ in v), max(b for _, b in v) return f"{lo:,}" if lo == hi else f"{lo:,}–{hi:,}" sc, sp = _span("code"), _span("prose") if sc and sp: prose += (f" Prompt lengths: code {sc} tokens (the first 30,000 characters of `convert_hf_to_gguf.py` plus " f"an instruction), prose {sp} tokens (the first 34,000 characters of wikitext-2 *train* plus a " f"writing instruction).") if all(lines) and prose: J["speed_note"] = " ".join(x + "." for x in lines) + " " + prose + (f" {NOISE_NOTE}" if NOISE_NOTE else "") # ---------- cache ---------- d = gate("n-c3-q106") if d and d.get("all_reused") and d.get("all_aligned") and have(d.get("n"), d.get("L"), d.get("expect_cache_n"), d.get("identical")): wm = [r_["warm_prompt_ms"] for r_ in d.get("rows") or [] if r_.get("warm_prompt_ms") is not None] cm = [r_["cold_prompt_ms"] for r_ in d.get("rows") or [] if r_.get("cold_prompt_ms") is not None] t_line = "" if wm and cm: t_line = (f" — median prefill **{statistics.median(wm) / 1000:.1f} s instead of " f"{statistics.median(cm) / 1000:.1f} s** cold " f"({statistics.median(cm) / statistics.median(wm):.1f}× faster)") J["cache_note"] = ( f"In {d['n']} request pairs sharing a long prefix at one fixed prompt length of {d['L']:,} tokens, every " f"second request resumed from the checkpoint the first one left {4 + UB:,} tokens before its end — " f"**{d['expect_cache_n']:,} tokens reused " f"({d['expect_cache_n'] / d['L'] * 100:.0f} %), {d['L'] - d['expect_cache_n']:,} processed**{t_line}. " f"Each warm reply was byte-identical to a cold run of the same prompt in **{d['identical']}/{d['n']}** " f"exchanges.\n\n" f"llama-server processes the last `n_ubatch` + 4 tokens of every prompt as two batches so it can checkpoint " f"there ({UB:,} + 4 = {4 + UB:,} tokens with the `-ub {UB}` used in these measurements, where the server " f"default is `-ub 512`; [upstream PR #20288](https://github.com/ggml-org/llama.cpp/pull/20288)). A turn that " f"resumes from a checkpoint left by a prompt of a *different* length therefore splits its tail differently " f"from a cold run, and float rounding can flip a greedy token.") # ---------- tools ---------- def _tools_ok(x): return bool(x) and have(x.get("passed"), x.get("total"), x.get("detail")) tl = gate("n-tools-q106") # stock template FX = [gate(l) for l in ("n-tools-q106-roff", "n-tools-q106-roff-r2", "n-tools-q106-roff-r3")] # quick-start config TD = S.get("tools_diag") or {} TF = S.get("template_fix") or {} if _tools_ok(tl) and all(_tools_ok(x) for x in FX) and have( TD.get("stock_on_leaks"), TD.get("stock_on_replies"), TD.get("stock_on_reasoning_extracted"), TD.get("nested_off_http500"), TD.get("nested_off_attempts")): misses, on_n, on_ok, off_n, off_ok = [], 0, 0, 0, 0 for i, x in enumerate(FX, 1): for k, v in x["detail"].items(): name, think = k.split("|think=") if think == "True": on_n += 1; on_ok += bool(v) else: off_n += 1; off_ok += bool(v) if not v: misses.append("`%s` with thinking %s (pass %d)" % (name, "on" if think == "True" else "off", i)) fp = TD.get("flag_probes") or {} flag_txt = "; ".join("%s: %d of %d replies still had reasoning in `content`" % (lab, fp[key]["leaks"], fp[key]["n"]) for key, lab in (("fmt-deepseek", "`--reasoning-format deepseek`"), ("srv-kwargs-high", "`--chat-template-kwargs` with `reasoning_effort`"), ("reasoning-on", "`--reasoning on`")) if key in fp) J["tools_note"] = ( "**Stock chat template: %d/%d.** Every thinking-on check failed. Re-run with the raw replies kept, %d of %d " "thinking-on replies carried the reasoning and a `` in `content`, and %d had any " "`reasoning_content`. llama-server builds its reasoning parser by rendering the template with " "`enable_thinking` on and off; this template ignores `enable_thinking` (it switches on `reasoning_effort`), " "so the parser finds no reasoning markers and extracts nothing. Server switches did not help (%s). With " "thinking off, `nested-object` failed on an HTTP 500 — see [Known issues](#known-issues-and-limits).\n\n" "**With the included `%s` and `--reasoning off` (the quick start): %s over three passes of the same suite " "(%d/%d)** — thinking off %d/%d, thinking on %d/%d; the misses were %s. A pass requires a native " "`tool_calls` entry with the right arguments and no think tags in `content`. Each check is a single sample at " "the recommended temperature 0.7." % ( tl["passed"], tl["total"], TD["stock_on_leaks"], TD["stock_on_replies"], TD["stock_on_reasoning_extracted"], flag_txt or "not measured", TF.get("file") or "—", ", ".join("%d/%d" % (x["passed"], x["total"]) for x in FX), sum(x["passed"] for x in FX), sum(x["total"] for x in FX), off_ok, off_n, on_ok, on_n, ", ".join(misses) if misses else "none")) # ---------- the template fix: what each request option does (recommended configuration) ---------- PR = TF.get("probes_roff") or {} if PR and all(_tools_ok(x) for x in FX): def _probe(prefix): rows_ = [v for k, v in PR.items() if k.split("|")[0] == prefix] return len(rows_), sum(1 for v in rows_ if v.get("leaks")), sum(1 for v in rows_ if v.get("reasoning_len")) on_ok = sum(bool(v) for x in FX for k, v in x["detail"].items() if k.endswith("|think=True")) on_n = sum(1 for x in FX for k in x["detail"] if k.endswith("|think=True")) off_ok = sum(bool(v) for x in FX for k, v in x["detail"].items() if k.endswith("|think=False")) off_n = sum(1 for x in FX for k in x["detail"] if k.endswith("|think=False")) lines = [] for label, prefix, think, dest in ( ("no `chat_template_kwargs`", "no-kwargs", "off (the server default with `--reasoning off`)", "—"), ("`\"enable_thinking\": false`", "enable_thinking=false", "off", "—"), ("`\"reasoning_effort\": \"none\"`", "reasoning_effort=none", "off", "—"), ("`\"reasoning_effort\": \"high\"`", "reasoning_effort=high", "on", "**`content`** — do not use"), ("`\"reasoning_effort\": \"medium\"`", "reasoning_effort=medium", "adaptive", "**`content`** — do not use")): n, leaks, _ = _probe(prefix) if n: lines.append("| %s | %s | %s | %d of %d replies with think tags in `content` |" % (label, think, dest, leaks, n)) lines.insert(1 if lines else 0, "| `\"enable_thinking\": true` | on | `reasoning_content` | tool suite with thinking on: %d/%d " "(a pass requires no think tags in `content`) |" % (on_ok, on_n)) J["template_note"] = ( "Measured on the standard STRIX_LEAN file with the included template file and `--reasoning off` (greedy " "probes: a direct question, a " "one-word instruction and a tool request, each with the tool schema attached; tool suite: 3 passes):\n\n" "| request | thinking | reasoning ends up in | measured |\n| --- | --- | --- | --- |\n" + "\n".join(lines) + "\n\nSo: switch thinking with `enable_thinking` only. Thinking-off tool checks: %d/%d." % (off_ok, off_n)) # ---------- vision ---------- def vwhy(x): if x.get("server_died"): import re as _re errs = [_re.sub(r"^[0-9.]+ [IWE] (srv +)?", "", e).strip() for e in x.get("server_log_errors") or []] first = next((e for e in errs if "failed" in e.lower() or "error" in e.lower() or "abort" in e.lower()), None) or (errs[0] if errs else None) return ("the server aborted" + (f" (`{first[:90]}`)" if first else "")) if x.get("error"): return f"the request failed ({x['error'][:80]})" exp = x.get("expected") or "" nexp = len(exp.split(",")) if exp else None hits = x.get("hits") or [] if nexp: return f"the reply named {len(hits)} of {nexp} expected terms" return "the reply did not pass" on, off = gate("n-vision-q106-faon"), gate("n-vision-q106-faoff") if on and off: def vok(x): return x.get("result") == "PASS" if vok(on) and vok(off): J["vision_note"] = "✅ **Images work with `-fa on` and `-fa off`.**" elif vok(off) and not vok(on): J["vision_note"] = (f"⛔ **With `-fa on`, image requests fail** — {vwhy(on)}. With `-fa off` they work. " f"**For image input, serve with `-fa off`.**") J["vision_quickstart_warning"] = ("> ⛔ **Images:** with `-fa on`, image requests fail on this build " "([measured](#vision)). If you send images, use `-fa off`.") elif vok(on) and not vok(off): J["vision_note"] = (f"⛔ **With `-fa off`, image requests fail** — {vwhy(off)}. With `-fa on` they work. " f"**For image input, keep `-fa on` (the quick-start default).**") else: J["vision_note"] = (f"⛔ **Images failed in both `-fa` settings.** `-fa on`: {vwhy(on)}; " f"`-fa off`: {vwhy(off)}.") vfx = gate("n-vision-q106-roff-faon") if vfx and "vision_note" in J: J["vision_note"] += (" The `-fa on` image test was repeated with the included template file and " f"`--reasoning off` (the quick start): {'✅ passed' if vok(vfx) else '❌ ' + vwhy(vfx)}.") # ---------- memory ---------- rows_sz = [r for r in (S.get("sizing") or []) if r.get("label") == "strix-lean"] if rows_sz: def mem_cell(r, key): if r.get("result") == "LOAD_FAIL": return "did not load" v = r.get(key) return f"{v:.2f} GiB" if v is not None else "—" body = "\n".join( f"| {r['ctx']:,} | {mem_cell(r, 'footprint_loaded_gib')} | {mem_cell(r, 'footprint_after_8k_gib')} |" for r in rows_sz if r.get("ctx") is not None) J["memory_note"] = ( "Measured footprint (drop in `MemAvailable`) of STRIX_LEAN with the vision projector, q8_0 KV cache, " "`-cram 512`, one slot, no draft head:\n\n" "| context | after load | after one request (30,000-character code prompt) |\n" "| ---: | ---: | ---: |\n" + body + "\n\nNo row was decode-benchmarked beyond that one request. Nothing beyond these rows was measured.") # ---------- recommendations ---------- def recommend(lean, coh, fast): """STRIX_LEAN is the flagship tier; the data decides whether COHERENT's quality or FAST's speed is worth taking instead. -> (default tag, markdown) or (None, None) when an input is missing.""" lc, lc_adv, lc_dis = cmp_line(lean, coh, "STRIX_LEAN", "COHERENT") fl, fl_adv, fl_dis = cmp_line(fast, lean, "FAST", "STRIX_LEAN") if lc is None or fl is None: return None, None L, C, F = T[lean], T[coh], T[fast] d_c, s_c, w_c = kld_cmp(coh, lean) more = (C["size_bytes"] - L["size_bytes"]) / MiB size_c = f"{abs(more):.0f} MiB {'more' if more > 0 else 'less'}" if w_c == "lower than" and lc_adv and not lc_dis: default = lean head = (f"**Start with `STRIX_LEAN`; take `COHERENT` if quality matters more than speed.** {lc}. `COHERENT`'s " f"KLD is {abs(d_c):.1f} % lower ({s_c:.1f}σ) for {size_c}.") elif w_c == "lower than": default = coh head = (f"**Start with `COHERENT`.** Its KLD is {abs(d_c):.1f} % lower than `STRIX_LEAN`'s ({s_c:.1f}σ) for " f"{size_c}, and the speed comparison does not clearly favour `STRIX_LEAN`: {lc}.") elif w_c == "within noise of" and lc_dis and not lc_adv: default = coh head = (f"**Start with `COHERENT`.** Its KLD is {w_c} `STRIX_LEAN`'s ({d_c:+.1f} %, {s_c:.1f}σ) and it is " f"measurably faster: {lc}. It costs {size_c}.") else: default = lean head = (f"**Start with `STRIX_LEAN`.** `COHERENT`'s KLD is {w_c} it ({d_c:+.1f} %, {s_c:.1f}σ) for " f"{size_c}; {lc}.") d_f, s_f, w_f = kld_cmp(fast, lean) kf = f"its KLD is {w_f} `STRIX_LEAN`'s ({F['kld_mean']:.4f} vs {L['kld_mean']:.4f}, {d_f:+.1f} %, {s_f:.1f}σ)" if fl_adv and not fl_dis: fast_txt = f"**Take `FAST` for speed:** {fl}; {kf}." else: fast_txt = f"`FAST` does not buy a clear speed gain here: {fl}; {kf}." sizes = f"{size_vs(lean, coh)}; {size_vs(lean, fast)}." body = (f"{head}\n\n{fast_txt}\n\nKLD order (lower is closer to BF16): {kld_rank_text((lean, coh, fast))}. " f"{sizes} {NOISE_NOTE}") return default, body def imat_pointer(): """Point the standard card at the imatrix repo; name only measurable (>= 2 sigma) twin improvements.""" repo = S.get("model_repo_imat") if not repo: return "" link = f"[imatrix build](https://huggingface.co/{repo})" bits = [] for im, st in TWINS: if not twin_kld_ok(st, im): return "" d, sg, w = kld_cmp(im, st) if w == "lower than": bits.append(f"`{NAMES[st]}` {T[st]['kld_mean']:.4f} → {T[im]['kld_mean']:.4f} ({d:+.1f} %, {sg:.1f}σ)") if bits: return f"**The {link} is measurably closer to BF16 at the same size:** " + "; ".join(bits) + "." return f"Importance-matrix twins of all three files: **{link}** (not measurably closer to BF16 on this corpus)." if kld_ready(STD_TAGS) and DEC_EQ is not None and PRE_EQ is not None: default, body = recommend("q106", "q102", "q103") if default: ptr = imat_pointer() J["std_recommendation"] = body + (f"\n\n{ptr}" if ptr else "") J["std_default"] = T[default]["file"] if kld_ready(IMAT_TAGS) and DEC_EQ is not None and PRE_EQ is not None: default_i, body_i = recommend("q106i", "q102i", "q103i") if default_i: cross = "" if kld_ready(("q102",)): d, sg, w = kld_cmp("q106i", "q102") cross = (f"\n\nFor scale: the imatrix `STRIX_LEAN`'s KLD is {w} the *standard* `COHERENT`'s " f"({T['q106i']['kld_mean']:.4f} vs {T['q102']['kld_mean']:.4f}, {d:+.1f} %, {sg:.1f}σ), " f"{size_vs('q106i', 'q102')}.") J["imat_recommendation"] = body_i + cross J["imat_default"] = T[default_i]["file"] parts = [] for nm, ti, ts in (("STRIX_LEAN", "q106i", "q106"), ("COHERENT", "q102i", "q102"), ("FAST", "q103i", "q103")): for dev, dn in (("rocm", "ROCm0"), ("vk", "Vulkan0")): a_, b_ = bn(ti, dev), bn(ts, dev) if a_ and b_ and have(a_.get("tg_median"), b_.get("tg_median")): parts.append(f"{nm} on {dn} {a_['tg_median']:.2f} vs {b_['tg_median']:.2f} tok/s") same = [T.get(i, {}).get("same_tensor_types_as_standard") for i, _ in TWINS] _hk = [T.get(i, {}).get("header_keys_only_in_imatrix") for i, _ in TWINS] _hs = [T.get(i, {}).get("header_keys_only_in_standard") for i, _ in TWINS] HDR = "" if all(x is not None for x in _hk + _hs) and not any(_hs) and len({tuple(x) for x in _hk}) == 1 and _hk[0]: HDR = ", all in the GGUF header, which carries %d extra keys in each imatrix file: %s" % ( len(_hk[0]), ", ".join("`%s`" % k for k in _hk[0])) deltas = [T.get(i, {}).get("file_size_delta_bytes") for i, _ in TWINS] if len(parts) == 6 and all(x is True for x in same) and all(d is not None for d in deltas): J["imat_speed_note"] = ( f"The imatrix changes scale values, not tensor types or sizes: tensor by tensor, each imatrix file has the " f"same names, types and byte sizes as its standard twin, so the work per token is the same (the files differ " f"in size by {deltas[0]} / {deltas[1]} / {deltas[2]} bytes{HDR})." + " " f"Measured speed still differs — imatrix vs standard: " + "; ".join(parts) + "." + (f" The largest of these decode gaps is {TG_NOISE:.1f} %; both cards call decode gaps below {DEC_EQ:.1f} % " f"a tie." if TG_NOISE is not None and DEC_EQ is not None else "")) # ---------- known issues (model facts + protocol; always emit) ---------- ki = [ "- **No MTP head.** The checkpoint's `config.json` declares `mtp_num_hidden_layers: 1`, but the weights contain " "**no** `mtp.*` tensors (1,026 tensors total). There is no multi-token-prediction head and no speculative decoding " "on these files — do not pass `--spec-type draft-mtp`.", "- **The stock chat template ignores `enable_thinking`, and llama-server cannot separate its reasoning** " "(see [Tool calling](#tool-calling)). Serve with the included `chat_template_enable_thinking.jinja` and " "`--reasoning off`, and switch thinking per request with `enable_thinking` — see " "[Reasoning controls](#reasoning-controls).", "- **Earlier assistant turns are re-rendered with their reasoning**, so multi-turn contexts grow faster than with " "templates that drop it.", "- **`llama-server`'s host-RAM prompt cache defaults to 8 GiB** (`-cram 8192`). On a shared box, set `-cram` explicitly.", "- Measured on Linux only (Ryzen AI Max+ 395, ROCm 7.2.4, unpatched `d3ca537`).", ] _pn_vals = [b_.get("prompt_n_max") if b_.get("prompt_n_max") is not None else b_.get("prompt_n") for b_ in (S.get("bench") or [])] _pn = max((v for v in _pn_vals if v is not None), default=None) # longest prompt any timed decode followed _sz = {r_.get("ctx"): r_ for r_ in (S.get("sizing") or []) if r_.get("label") == "strix-lean"} _big = max(_sz) if _sz else None _ctx_txt = ("" if _big is None else f" (a {_big:,}-token context was loaded in the memory test, not benchmarked)" if _sz[_big].get("result") != "LOAD_FAIL" else "") for _c, _r in sorted(_sz.items()): if _r.get("result") == "LOAD_FAIL": ki.append(f"- **A {_c:,}-token context did not load** in the memory test (STRIX_LEAN + vision projector, " f"q8_0 KV cache, one slot) on this box.") ki.append(f"- **Not measured:** decode beyond a {_pn:,}-token prompt{_ctx_txt}, " if _pn is not None else "- **Not measured:** decode at long context, ") ki[-1] += ("long-context quality, video input, concurrency above 1, and task-level accuracy. Perplexity/KLD measure " "next-token fidelity to BF16 on prose, not reasoning or code correctness.") if have(ref.get("hip_rocm0_chunk1"), ref.get("cpu_chunk1"), ref.get("hip_rocm0_final")) and have(S.get("bf16", {}).get("ppl")): ki.insert(0, f"- **Do not run the BF16 GGUF of this model on ROCm0 with this build.** `d3ca537` computes it wrong on " f"that path: wikitext-2 perplexity {ref['hip_rocm0_final']:.1f} on ROCm0 vs {S['bf16']['ppl']:.2f} on the CPU " f"(first window {ref['hip_rocm0_chunk1']:.1f} vs {ref['cpu_chunk1']:.2f}; also wrong with `-fa off`). " f"The 4-bit files are not affected — their ROCm0 grades are in the table — and the BF16 file is not " f"published here; the quality reference was computed on the CPU instead.") if have(TD.get("nested_off_http500"), TD.get("nested_off_attempts")): ki.insert(1, "- **llama-server rejects a tool call whose required arguments are not in the order the schema lists " "them** — HTTP 500, *The model produced output that does not match the expected peg-native format*: " "this build's parser for the XML tool-call format expects required arguments in definition order. " "On the standard STRIX_LEAN file with the stock template and thinking off, the `nested-object` " "request (three required arguments) " "hit it in the tool suite%s and in %d of %d repeats; the model had written a well-formed call with " "the arguments reordered. Be ready to retry on this error." % ( "" if TD.get("gate_http500_logged") else " (not confirmed in the suite's server log)", TD["nested_off_http500"], TD["nested_off_attempts"])) _PR = (S.get("template_fix") or {}).get("probes_roff") or {} _lk = [v for k, v in _PR.items() if k.split("|")[0] in ("reasoning_effort=high", "reasoning_effort=medium")] if _lk: ki.insert(2, "- **With the included template, do not set `reasoning_effort` to `high` or `medium`** — the reasoning " "goes back into `content` (%d of %d probe replies). `enable_thinking: true` is the way to turn " "thinking on." % (sum(1 for v in _lk if v.get("leaks")), len(_lk))) _seat = [r_ for r_ in (S.get("seats") or {}).values() if r_.get("thinking_reasoning_len") is not None] _short = [r_ for r_ in _seat if not r_.get("thinking_reply") and r_.get("thinking_reasoning_len")] if _short: ki.insert(3, "- **With thinking on, a very short answer can stay inside the think block.** Served as in the quick " "start, `Reply with the single word: ready` (no tools, `enable_thinking: true`, temperature 0) came " "back with the word in `reasoning_content` and an empty `content` on %d of %d FAST files tested.%s" % ( len(_short), len(_seat), " With thinking off the same request returned `ready` in `content`." if all((r_.get("default_reply") or "").strip().lower().startswith("ready") for r_ in _seat) else "")) J["std_known_issues"] = "\n".join(ki) J["imat_known_issues"] = "\n".join([ "- Calibration text is general-purpose English/code (bartowski `calibration_datav3.txt`); an imatrix built from " "your own domain can do better on that domain.", "- Graded on wikitext-2 *test*, a different corpus from the calibration text. Improvements on reasoning/code " "tasks were not measured.", ] + ki) json.dump(J, open(sys.argv[2], "w"), indent=2) print("judgments written:", sorted(J)) for k, v in J.items(): print(f"\n[{k}]\n{v}")