Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+Agnes-3.0-Flash-Preview-MTP-imatrix-Q4_0-ROCmFP4-STRIX_LEAN.gguf filter=lfs diff=lfs merge=lfs -text
+Agnes-3.0-Flash-Preview.imatrix filter=lfs diff=lfs merge=lfs -text
+mmproj-Agnes-3.0-Flash-Preview-BF16.gguf filter=lfs diff=lfs merge=lfs -text

Agnes-3.0-Flash-Preview-MTP-imatrix-Q4_0-ROCmFP4-STRIX_LEAN.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6aae32cf31652a509044288c31e2e38e5d4b3f8fb42e9531ef1498a48d1fc8cd
+size 18061413504

Agnes-3.0-Flash-Preview.imatrix

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb66b219d8bb057b78b933e5b78042fc47e028415d626690883c8c8f72707a09
+size 15937792

SHA256SUMS

@@ -0,0 +1,4 @@
+643d7a6f601826c7ee2ff3cc2b8092ef971d1dec1e6f1fff6c36ab6c4d875f30  Agnes-3.0-Flash-Preview-MTP-imatrix-Q4_0-ROCmFP4-COHERENT.gguf
+6aae32cf31652a509044288c31e2e38e5d4b3f8fb42e9531ef1498a48d1fc8cd  Agnes-3.0-Flash-Preview-MTP-imatrix-Q4_0-ROCmFP4-STRIX_LEAN.gguf
+eb66b219d8bb057b78b933e5b78042fc47e028415d626690883c8c8f72707a09  Agnes-3.0-Flash-Preview.imatrix
+d35b469bce95e80dfa5928e7547089ceefe8e9bb77452349a3af02c5679d7026  mmproj-Agnes-3.0-Flash-Preview-BF16.gguf

mmproj-Agnes-3.0-Flash-Preview-BF16.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d35b469bce95e80dfa5928e7547089ceefe8e9bb77452349a3af02c5679d7026
+size 931146048

recipe/FOLD_PROVENANCE.json

@@ -0,0 +1,15 @@
+{
+  "source_repo": "Agnes-AI/Agnes-3.0-Flash",
+  "source_dir": "hf",
+  "main_ffn": 17408,
+  "parallel_ffn": 2048,
+  "folded_ffn": 19456,
+  "stats": {
+    "fold": 216,
+    "pad": 3,
+    "pass": 1086
+  },
+  "tensors_out": 1305,
+  "bytes_out": 66243917920,
+  "fold_script_sha256": "b1e6dbcf16bbac48fbc95b763efeb85efb21ce1d1fdfa825a6ae01187f044486"
+}

recipe/fold_agnes.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+"""Agnes-3.0-Flash Preview -> stock Qwen3.5 HF checkpoint for convert_hf_to_gguf.py.
+
+Every transform is exact (verified separately by verify_fold.py):
+  1. *.delta_attn.*  -> *.linear_attn.*   the converter V-head-reorders ONLY linear_attn.* names
+  2. *.global_attn.* -> *.self_attn.*
+  3. mlp.parallel_ffn folded into mlp     reference: y = down(act(gate x)*up x) + parallel_ffn(x)
+       gate/up: cat(main, par, dim=0)       plain sum, same activation => concatenation is exact
+       down   : cat(main, par, dim=1)
+  4. mtp.layers.N.mlp zero-padded to the folded width   zero SwiGLU rows contribute exactly 0
+  5. config: qwen3_5 model/arch ids, layer_types mapped, full_attention_interval,
+     intermediate_size = main + parallel
+usage: fold_agnes.py <agnes_hf_dir> <out_dir>
+"""
+import hashlib, json, os, re, shutil, sys, time
+import torch
+from safetensors import safe_open
+from safetensors.torch import save_file
+
+SRC, DST = sys.argv[1], sys.argv[2]
+os.makedirs(DST, exist_ok=True)
+
+cfg = json.load(open(os.path.join(SRC, "config.json")))
+tc = cfg["text_config"]
+MAIN = int(tc["intermediate_size"])
+PAR = int(tc["parallel_ffn_intermediate_size"])
+FOLD = MAIN + PAR
+N_LAYERS = int(tc["num_hidden_layers"])
+INTERVAL = int(tc["global_attention_interval"])
+LT = {"agnes_delta_attention": "linear_attention", "agnes_global_attention": "full_attention"}
+
+# layer plan must be the pattern the loader will re-derive from full_attention_interval
+plan = tc["layer_types"]
+assert len(plan) == N_LAYERS, (len(plan), N_LAYERS)
+for i, t in enumerate(plan):
+    want = "agnes_global_attention" if (i + 1) % INTERVAL == 0 else "agnes_delta_attention"
+    assert t == want, f"layer {i}: {t} != {want} (loader derives the pattern from the interval)"
+
+wm = json.load(open(os.path.join(SRC, "model.safetensors.index.json")))["weight_map"]
+par_names = {k for k in wm if ".mlp.parallel_ffn." in k}
+par_files = sorted({wm[k] for k in par_names})
+main_files = sorted({f for f in wm.values()} - set(par_files))
+assert len(par_names) == 3 * N_LAYERS, len(par_names)
+assert not any(".mlp.parallel_ffn." not in k for k in wm if wm[k] in par_files), "parallel file holds other tensors"
+
+par_handles = {f: safe_open(os.path.join(SRC, f), "pt") for f in par_files}
+def par(name):
+    return par_handles[wm[name]].get_tensor(name)
+
+RE_MAIN_MLP = re.compile(r"^(model\.language_model\.layers\.(\d+)\.mlp)\.(gate_proj|up_proj|down_proj)\.weight$")
+RE_MTP_MLP = re.compile(r"^mtp\.layers\.\d+\.mlp\.(gate_proj|up_proj|down_proj)\.weight$")
+
+def rename(n):
+    n = n.replace(".delta_attn.", ".linear_attn.")
+    n = n.replace(".global_attn.", ".self_attn.")
+    return n
+
+new_map, total, stats = {}, 0, {"fold": 0, "pad": 0, "pass": 0}
+t0 = time.time()
+for f in main_files:
+    out = {}
+    with safe_open(os.path.join(SRC, f), "pt") as h:
+        for name in h.keys():
+            t = h.get_tensor(name)
+            m = RE_MAIN_MLP.match(name)
+            if m:
+                prefix, _, kind = m.groups()
+                p = par(f"{prefix}.parallel_ffn.{kind}.weight")
+                assert t.dtype == p.dtype, (name, t.dtype, p.dtype)
+                if kind in ("gate_proj", "up_proj"):
+                    assert tuple(t.shape) == (MAIN, t.shape[1]) and tuple(p.shape) == (PAR, t.shape[1]), (name, t.shape, p.shape)
+                    t = torch.cat([t, p], dim=0)
+                else:
+                    assert tuple(t.shape) == (t.shape[0], MAIN) and tuple(p.shape) == (t.shape[0], PAR), (name, t.shape, p.shape)
+                    t = torch.cat([t, p], dim=1)
+                stats["fold"] += 1
+            elif RE_MTP_MLP.match(name):
+                kind = RE_MTP_MLP.match(name).group(1)
+                if kind in ("gate_proj", "up_proj"):
+                    assert t.shape[0] == MAIN, (name, t.shape)
+                    t = torch.cat([t, torch.zeros((PAR, t.shape[1]), dtype=t.dtype)], dim=0)
+                else:
+                    assert t.shape[1] == MAIN, (name, t.shape)
+                    t = torch.cat([t, torch.zeros((t.shape[0], PAR), dtype=t.dtype)], dim=1)
+                stats["pad"] += 1
+            else:
+                stats["pass"] += 1
+            nn_ = rename(name)
+            assert nn_ not in out and nn_ not in new_map, f"name collision {nn_}"
+            out[nn_] = t.contiguous()
+    save_file(out, os.path.join(DST, f), metadata={"format": "pt"})
+    for k, v in out.items():
+        new_map[k] = f
+        total += v.numel() * v.element_size()
+    print(f"  wrote {f}: {len(out)} tensors  ({time.time()-t0:.0f}s)", flush=True)
+    del out
+
+json.dump({"metadata": {"total_size": total}, "weight_map": dict(sorted(new_map.items()))},
+          open(os.path.join(DST, "model.safetensors.index.json"), "w"), indent=2)
+
+# ---- config ----
+o = json.loads(json.dumps(cfg))
+o.pop("auto_map", None)
+o["architectures"] = ["Qwen3_5ForConditionalGeneration"]
+o["model_type"] = "qwen3_5"
+t = o["text_config"]
+t["model_type"] = "qwen3_5_text"
+t["layer_types"] = [LT[x] for x in t["layer_types"]]
+t["full_attention_interval"] = t.pop("global_attention_interval")
+t["intermediate_size"] = FOLD
+t.pop("parallel_ffn_intermediate_size")
+o["vision_config"]["model_type"] = "qwen3_5"
+json.dump(o, open(os.path.join(DST, "config.json"), "w"), indent=2)
+
+# ---- sidecar files the converter/tokenizer read (custom code + sglang patch deliberately NOT copied) ----
+KEEP = ["tokenizer.json", "tokenizer_config.json", "vocab.json", "merges.txt", "chat_template.jinja",
+        "generation_config.json", "preprocessor_config.json", "video_preprocessor_config.json",
+        "LICENSE", "README.md"]
+for k in KEEP:
+    if os.path.exists(os.path.join(SRC, k)):
+        shutil.copy2(os.path.join(SRC, k), os.path.join(DST, k))
+
+prov = {"source_repo": "Agnes-AI/Agnes-3.0-Flash", "source_dir": SRC, "main_ffn": MAIN, "parallel_ffn": PAR,
+        "folded_ffn": FOLD, "stats": stats, "tensors_out": len(new_map), "bytes_out": total,
+        "fold_script_sha256": hashlib.sha256(open(__file__, "rb").read()).hexdigest()}
+json.dump(prov, open(os.path.join(DST, "FOLD_PROVENANCE.json"), "w"), indent=2)
+print("DONE", json.dumps(prov), flush=True)

recipe/logs/B2_imatrix.log

@@ -0,0 +1,65 @@
+0.00.068.874 I common_init_result: fitting params to device memory ...
+0.00.068.880 I common_init_result: (for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)
+0.00.563.403 W llama_model_loader: direct I/O is enabled, disabling mmap
+1.20.388.130 W llama_context: n_ctx_seq (512) < n_ctx_train (262144) -- the full capacity of the model will not be utilized
+1.20.449.409 I 
+1.20.449.534 I system_info: n_threads = 16 (n_threads_batch = 16) / 32 | ROCm : NO_VMM = 1 | PEER_MAX_BATCH_SIZE = 128 | FA_ALL_QUANTS = 1 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+1.20.449.581 I compute_imatrix: tokenizing the input ..
+1.20.538.717 I compute_imatrix: tokenization took 89.171 ms
+1.20.538.743 I compute_imatrix: computing over 129 chunks, n_ctx=512, batch_size=512, n_seq=1
+1.26.097.413 I compute_imatrix: 5.56 seconds per pass - ETA 11.95 minutes
+[1]4.9076,[2]3.6939,[3]3.5023,[4]3.7237,[5]3.6719,[6]3.4578,[7]3.9055,[8]3.9478,[9]4.3632,2.10.119.574 W 
+save_imatrix: saving imatrix using GGUF format with a different suffix than .gguf
+2.10.119.575 W save_imatrix: if you want the previous imatrix format, use --output-format dat
+
+[10]4.4852,[11]4.1245,[12]4.5016,[13]5.0161,[14]5.2409,[15]5.6788,[16]5.8723,[17]6.1281,[18]6.4236,[19]6.1378,3.05.385.766 W 
+save_imatrix: saving imatrix using GGUF format with a different suffix than .gguf
+3.05.385.767 W save_imatrix: if you want the previous imatrix format, use --output-format dat
+
+[20]6.1804,[21]6.2324,[22]6.2199,[23]6.0845,[24]6.2482,[25]6.3968,[26]6.4389,[27]6.5903,[28]6.7527,[29]7.0604,4.04.930.247 W 
+save_imatrix: saving imatrix using GGUF format with a different suffix than .gguf
+4.04.930.249 W save_imatrix: if you want the previous imatrix format, use --output-format dat
+
+[30]7.1050,[31]6.8972,[32]6.6113,[33]6.4182,[34]6.2723,[35]6.2020,[36]6.2811,[37]6.3316,[38]6.4117,[39]6.5548,5.02.852.427 W 
+save_imatrix: saving imatrix using GGUF format with a different suffix than .gguf
+5.02.852.428 W save_imatrix: if you want the previous imatrix format, use --output-format dat
+
+[40]6.7426,[41]6.8424,[42]7.1064,[43]7.2836,[44]7.4628,[45]7.5687,[46]7.4849,[47]7.4479,[48]7.5905,[49]7.6776,5.48.238.271 W 
+save_imatrix: saving imatrix using GGUF format with a different suffix than .gguf
+5.48.238.272 W save_imatrix: if you want the previous imatrix format, use --output-format dat
+
+[50]7.6233,[51]7.5430,[52]7.6002,[53]7.7363,[54]7.8387,[55]7.9166,[56]7.9407,[57]7.9634,[58]7.9820,[59]7.9799,6.22.003.653 W 
+save_imatrix: saving imatrix using GGUF format with a different suffix than .gguf
+6.22.003.655 W save_imatrix: if you want the previous imatrix format, use --output-format dat
+
+[60]7.9475,[61]7.9024,[62]7.8557,[63]7.8903,[64]7.9190,[65]7.8691,[66]7.8520,[67]7.8486,[68]7.7677,[69]7.7436,6.55.927.232 W 
+save_imatrix: saving imatrix using GGUF format with a different suffix than .gguf
+6.55.927.234 W save_imatrix: if you want the previous imatrix format, use --output-format dat
+
+[70]7.7488,[71]7.7251,[72]7.6933,[73]7.6895,[74]7.6322,[75]7.5635,[76]7.5240,[77]7.4955,[78]7.4733,[79]7.4308,7.29.787.884 W 
+save_imatrix: saving imatrix using GGUF format with a different suffix than .gguf
+7.29.787.885 W save_imatrix: if you want the previous imatrix format, use --output-format dat
+
+[80]7.3803,[81]7.3938,[82]7.3937,[83]7.3608,[84]7.3803,[85]7.4070,[86]7.3312,[87]7.3247,[88]7.3076,[89]7.3213,8.03.945.340 W 
+save_imatrix: saving imatrix using GGUF format with a different suffix than .gguf
+8.03.945.341 W save_imatrix: if you want the previous imatrix format, use --output-format dat
+
+[90]7.3340,[91]7.3067,[92]7.2339,[93]7.1617,[94]7.0804,[95]7.0069,[96]6.9433,[97]6.8737,[98]6.8120,[99]6.7556,8.38.542.860 W 
+save_imatrix: saving imatrix using GGUF format with a different suffix than .gguf
+8.38.542.862 W save_imatrix: if you want the previous imatrix format, use --output-format dat
+
+[100]6.7740,[101]6.7927,[102]6.8860,[103]6.9718,[104]7.0452,[105]7.1568,[106]7.2470,[107]7.2711,[108]7.2896,[109]7.2935,9.13.159.314 W 
+save_imatrix: saving imatrix using GGUF format with a different suffix than .gguf
+9.13.159.315 W save_imatrix: if you want the previous imatrix format, use --output-format dat
+
+[110]7.2987,[111]7.2736,[112]7.2092,[113]7.1307,[114]7.1688,[115]7.2059,[116]7.2241,[117]7.2269,[118]7.2650,[119]7.2963,9.44.748.389 W 
+save_imatrix: saving imatrix using GGUF format with a different suffix than .gguf
+9.44.748.390 W save_imatrix: if you want the previous imatrix format, use --output-format dat
+
+[120]7.3067,[121]7.3198,[122]7.3255,[123]7.2919,[124]7.3443,[125]7.3959,[126]7.4370,[127]7.4892,[128]7.5278,[129]7.5714,
+Final estimate: PPL = 7.5714 +/- 0.10659
+10.18.344.914 W 
+save_imatrix: saving imatrix using GGUF format with a different suffix than .gguf
+10.18.344.914 W save_imatrix: if you want the previous imatrix format, use --output-format dat
+
+

recipe/logs/P1_apply_check.log

@@ -0,0 +1,9 @@
+Checking patch common/common.cpp...
+Checking patch common/common.h...
+Hunk #1 succeeded at 1074 (offset -1 lines).
+Checking patch tools/server/server-context.cpp...
+Hunk #1 succeeded at 2026 (offset -3 lines).
+Hunk #2 succeeded at 2794 (offset -3 lines).
+Hunk #3 succeeded at 2910 (offset -3 lines).
+Hunk #4 succeeded at 2927 (offset -3 lines).
+Checking patch tools/server/server-task.cpp...

recipe/logs/R1_kld4_old_q106_cpuburn.log

@@ -0,0 +1,15 @@
+0.00.051.544 I common_init_result: fitting params to device memory ...
+0.00.051.550 I common_init_result: (for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)
+0.00.520.939 W llama_model_loader: direct I/O is enabled, disabling mmap
+0.26.402.585 W llama_context: n_ctx_seq (2048) < n_ctx_train (262144) -- the full capacity of the model will not be utilized
+0.26.537.079 W common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+0.26.677.969 I 
+0.26.678.090 I system_info: n_threads = 16 (n_threads_batch = 16) / 32 | ROCm : NO_VMM = 1 | PEER_MAX_BATCH_SIZE = 128 | FA_ALL_QUANTS = 1 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+0.26.800.104 I kl_divergence: computing over 40 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+0.36.294.541 I kl_divergence: 9.49 seconds per pass - ETA 6.32 minutes
+
+chunk             PPL               ln(PPL(Q)/PPL(base))          KL Divergence              Δp RMS            Same top p
+   1       5.6619 ±    0.4178       0.01063 ±    0.00809       0.03262 ±    0.00202     5.271 ±  0.361 %    91.789 ±  0.859 %
+   2       6.8229 ±    0.3691       0.02962 ±    0.00697       0.04275 ±    0.00205     5.500 ±  0.293 %    90.371 ±  0.652 %
+   3       7.0007 ±    0.3084       0.02622 ±    0.00560       0.03988 ±    0.00158     5.443 ±  0.241 %    90.616 ±  0.526 %
+   4       7.2110 ±    0.2849       0.04157 ±    0.00522       0.04291 ±    0.00164     5.873 ±  0.280 %    90.689 ±  0.454 %

recipe/logs/R2_ppl_bf16.log

@@ -0,0 +1,15 @@
+0.00.040.938 I common_init_result: fitting params to device memory ...
+0.00.040.942 I common_init_result: (for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)
+0.00.393.366 W llama_model_loader: direct I/O is enabled, disabling mmap
+1.12.661.627 W llama_context: n_ctx_seq (2048) < n_ctx_train (262144) -- the full capacity of the model will not be utilized
+1.12.727.778 W common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+1.13.134.594 I 
+1.13.134.711 I system_info: n_threads = 16 (n_threads_batch = 16) / 32 | ROCm : NO_VMM = 1 | PEER_MAX_BATCH_SIZE = 128 | FA_ALL_QUANTS = 1 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+1.13.135.051 I perplexity: saving all logits to kld/bf16_r2.kld
+1.13.135.057 I perplexity: tokenizing the input ..
+1.13.433.291 I perplexity: tokenization took 298.224 ms
+1.13.433.382 I perplexity: calculating perplexity over 40 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+1.19.563.270 I perplexity: 5.99 seconds per pass - ETA 3.98 minutes
+[1]5.5906,[2]6.6182,[3]6.8133,[4]6.9122,[5]6.8296,[6]5.8746,[7]5.3850,[8]5.4021,[9]5.7141,[10]5.8085,[11]5.8798,[12]6.1251,[13]6.2179,[14]6.2829,[15]6.3333,[16]6.6341,[17]6.6898,[18]6.7989,[19]6.7409,[20]6.5603,[21]6.5673,[22]6.5897,[23]6.6337,[24]6.6470,[25]6.6946,[26]6.6705,[27]6.8066,[28]6.8874,[29]6.8912,[30]6.8584,[31]6.7830,[32]6.6822,[33]6.6543,[34]6.6494,[35]6.6636,[36]6.6672,[37]6.5522,[38]6.4879,[39]6.4201,[40]6.3314,
+5.19.083.910 I Final estimate: PPL = 6.3314 +/- 0.07632
+

recipe/logs/R3_kld_q102i.log

@@ -0,0 +1,92 @@
+0.00.040.422 I common_init_result: fitting params to device memory ...
+0.00.040.426 I common_init_result: (for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)
+0.00.474.454 W llama_model_loader: direct I/O is enabled, disabling mmap
+0.27.064.122 W llama_context: n_ctx_seq (2048) < n_ctx_train (262144) -- the full capacity of the model will not be utilized
+0.27.133.422 W common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+0.27.289.156 I 
+0.27.289.290 I system_info: n_threads = 16 (n_threads_batch = 16) / 32 | ROCm : NO_VMM = 1 | PEER_MAX_BATCH_SIZE = 128 | FA_ALL_QUANTS = 1 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+0.27.404.427 I kl_divergence: computing over 40 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+0.36.830.849 I kl_divergence: 9.43 seconds per pass - ETA 6.28 minutes
+
+chunk             PPL               ln(PPL(Q)/PPL(base))          KL Divergence              Δp RMS            Same top p
+   1       5.6709 ±    0.4138       0.01425 ±    0.00904       0.02960 ±    0.00288     5.636 ±  0.769 %    92.082 ±  0.845 %
+   2       6.7293 ±    0.3601       0.01663 ±    0.00684       0.02989 ±    0.00161     5.217 ±  0.431 %    90.714 ±  0.642 %
+   3       6.9169 ±    0.3016       0.01507 ±    0.00554       0.03014 ±    0.00149     5.374 ±  0.356 %    91.333 ±  0.508 %
+   4       7.0890 ±    0.2767       0.02524 ±    0.00551       0.03455 ±    0.00230     5.724 ±  0.377 %    91.422 ±  0.438 %
+   5       6.9977 ±    0.2459       0.02431 ±    0.00473       0.03510 ±    0.00197     5.756 ±  0.317 %    91.359 ±  0.393 %
+   6       6.0143 ±    0.1871       0.02349 ±    0.00446       0.03679 ±    0.00183     6.238 ±  0.301 %    91.365 ±  0.359 %
+   7       5.4692 ±    0.1536       0.01550 ±    0.00423       0.04573 ±    0.00356     6.706 ±  0.294 %    91.649 ±  0.327 %
+   8       5.4716 ±    0.1434       0.01278 ±    0.00396       0.04609 ±    0.00315     6.674 ±  0.266 %    91.508 ±  0.308 %
+   9       5.7707 ±    0.1434       0.00984 ±    0.00371       0.04562 ±    0.00282     6.516 ±  0.245 %    91.202 ±  0.295 %
+  10       5.8669 ±    0.1392       0.01034 ±    0.00343       0.04327 ±    0.00254     6.366 ±  0.228 %    91.310 ±  0.279 %
+  11       5.9368 ±    0.1343       0.00995 ±    0.00319       0.04136 ±    0.00232     6.217 ±  0.214 %    91.425 ±  0.264 %
+  12       6.1791 ±    0.1354       0.00904 ±    0.00301       0.04007 ±    0.00213     6.096 ±  0.203 %    91.373 ±  0.253 %
+  13       6.2710 ±    0.1322       0.00876 ±    0.00286       0.03942 ±    0.00197     6.006 ±  0.192 %    91.481 ±  0.242 %
+  14       6.3372 ±    0.1288       0.00883 ±    0.00271       0.03813 ±    0.00183     5.878 ±  0.182 %    91.482 ±  0.233 %
+  15       6.3906 ±    0.1257       0.00924 ±    0.00259       0.03703 ±    0.00171     5.762 ±  0.174 %    91.528 ±  0.225 %
+  16       6.6471 ±    0.1283       0.00226 ±    0.00269       0.03821 ±    0.00164     5.705 ±  0.166 %    91.416 ±  0.219 %
+  17       6.7047 ±    0.1250       0.00250 ±    0.00257       0.03722 ±    0.00155     5.611 ±  0.159 %    91.507 ±  0.211 %
+  18       6.8261 ±    0.1241       0.00442 ±    0.00247       0.03659 ±    0.00147     5.530 ±  0.152 %    91.512 ±  0.205 %
+  19       6.7763 ±    0.1204       0.00565 ±    0.00239       0.03618 ±    0.00140     5.532 ±  0.151 %    91.573 ±  0.199 %
+  20       6.5992 ±    0.1138       0.00630 ±    0.00233       0.03693 ±    0.00135     5.609 ±  0.144 %    91.579 ±  0.194 %
+  21       6.6019 ±    0.1107       0.00562 ±    0.00226       0.03658 ±    0.00128     5.590 ±  0.139 %    91.565 ±  0.190 %
+  22       6.6282 ±    0.1087       0.00624 ±    0.00218       0.03607 ±    0.00123     5.575 ±  0.136 %    91.562 ±  0.185 %
+  23       6.6712 ±    0.1071       0.00603 ±    0.00212       0.03556 ±    0.00118     5.532 ±  0.131 %    91.610 ±  0.181 %
+  24       6.6902 ±    0.1050       0.00685 ±    0.00206       0.03512 ±    0.00113     5.495 ±  0.127 %    91.663 ±  0.176 %
+  25       6.7396 ±    0.1038       0.00706 ±    0.00200       0.03489 ±    0.00109     5.454 ±  0.123 %    91.715 ±  0.172 %
+  26       6.7177 ±    0.1015       0.00740 ±    0.00196       0.03452 ±    0.00105     5.421 ±  0.120 %    91.766 ±  0.169 %
+  27       6.8568 ±    0.1021       0.00768 ±    0.00192       0.03400 ±    0.00101     5.375 ±  0.117 %    91.803 ±  0.165 %
+  28       6.9402 ±    0.1019       0.00795 ±    0.00186       0.03351 ±    0.00098     5.324 ±  0.114 %    91.862 ±  0.162 %
+  29       6.9426 ±    0.1001       0.00773 ±    0.00181       0.03310 ±    0.00094     5.288 ±  0.111 %    91.876 ±  0.159 %
+  30       6.9084 ±    0.0978       0.00755 ±    0.00178       0.03292 ±    0.00091     5.264 ±  0.109 %    91.906 ±  0.156 %
+  31       6.8359 ±    0.0951       0.00806 ±    0.00175       0.03280 ±    0.00089     5.250 ±  0.106 %    91.962 ±  0.153 %
+  32       6.7389 ±    0.0922       0.00873 ±    0.00172       0.03308 ±    0.00086     5.310 ±  0.103 %    91.932 ±  0.151 %
+  33       6.7122 ±    0.0905       0.00902 ±    0.00169       0.03300 ±    0.00086     5.299 ±  0.102 %    91.940 ±  0.148 %
+  34       6.7067 ±    0.0890       0.00891 ±    0.00166       0.03277 ±    0.00083     5.271 ±  0.099 %    91.950 ±  0.146 %
+  35       6.7286 ±    0.0881       0.01003 ±    0.00164       0.03268 ±    0.00081     5.277 ±  0.099 %    91.948 ±  0.144 %
+  36       6.7363 ±    0.0870       0.01063 ±    0.00161       0.03248 ±    0.00079     5.251 ±  0.097 %    92.017 ±  0.141 %
+  37       6.6250 ±    0.0840       0.01137 ±    0.00158       0.03228 ±    0.00078     5.245 ±  0.095 %    92.035 ±  0.139 %
+  38       6.5583 ±    0.0818       0.01109 ±    0.00155       0.03207 ±    0.00076     5.241 ±  0.094 %    92.077 ±  0.137 %
+  39       6.4896 ±    0.0797       0.01105 ±    0.00155       0.03231 ±    0.00078     5.254 ±  0.092 %    92.070 ±  0.135 %
+  40       6.4004 ±    0.0772       0.01112 ±    0.00152       0.03219 ±    0.00077     5.252 ±  0.090 %    92.107 ±  0.133 %
+
+====== Perplexity statistics ======
+Mean PPL(Q)                   :   6.400447 ±   0.077237
+Mean PPL(base)                :   6.329637 ±   0.076239
+Cor(ln(PPL(Q)), ln(PPL(base))):  99.20%
+Mean ln(PPL(Q)/PPL(base))     :   0.011125 ±   0.001523
+Mean PPL(Q)/PPL(base)         :   1.011187 ±   0.001540
+Mean PPL(Q)-PPL(base)         :   0.070810 ±   0.009741
+
+====== KL divergence statistics ======
+Mean    KLD:   0.032188 ±   0.000766
+Maximum KLD:  17.548290
+99.9%   KLD:   1.516570
+99.0%   KLD:   0.313102
+95.0%   KLD:   0.099888
+90.0%   KLD:   0.058733
+Median  KLD:   0.013397
+10.0%   KLD:   0.000221
+ 5.0%   KLD:   0.000052
+ 1.0%   KLD:   0.000005
+ 0.1%   KLD:  -0.000000
+Minimum KLD:  -0.000020
+
+====== Token probability statistics ======
+Mean    Δp: -0.221 ± 0.026 %
+Maximum Δp: 90.468%
+99.9%   Δp: 34.983%
+99.0%   Δp: 13.933%
+95.0%   Δp:  6.180%
+90.0%   Δp:  3.604%
+75.0%   Δp:  0.658%
+Median  Δp: -0.001%
+25.0%   Δp: -0.917%
+10.0%   Δp: -4.169%
+ 5.0%   Δp: -7.119%
+ 1.0%   Δp: -16.256%
+ 0.1%   Δp: -42.662%
+Minimum Δp: -98.257%
+RMS Δp    :  5.252 ± 0.090 %
+Same top p: 92.107 ± 0.133 %
+

recipe/logs/R3_kld_q106.log

@@ -0,0 +1,92 @@
+0.00.056.459 I common_init_result: fitting params to device memory ...
+0.00.056.463 I common_init_result: (for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)
+0.00.414.626 W llama_model_loader: direct I/O is enabled, disabling mmap
+0.31.123.455 W llama_context: n_ctx_seq (2048) < n_ctx_train (262144) -- the full capacity of the model will not be utilized
+0.31.189.690 W common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+0.31.333.670 I 
+0.31.333.809 I system_info: n_threads = 16 (n_threads_batch = 16) / 32 | ROCm : NO_VMM = 1 | PEER_MAX_BATCH_SIZE = 128 | FA_ALL_QUANTS = 1 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+0.31.445.087 I kl_divergence: computing over 40 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+0.38.577.429 I kl_divergence: 7.13 seconds per pass - ETA 4.75 minutes
+
+chunk             PPL               ln(PPL(Q)/PPL(base))          KL Divergence              Δp RMS            Same top p
+   1       5.6619 ±    0.4178       0.01265 ±    0.00809       0.03262 ±    0.00203     5.307 ±  0.369 %    91.593 ±  0.868 %
+   2       6.8229 ±    0.3691       0.03044 ±    0.00695       0.04270 ±    0.00205     5.525 ±  0.291 %    90.469 ±  0.649 %
+   3       7.0007 ±    0.3084       0.02712 ±    0.00560       0.03995 ±    0.00158     5.478 ±  0.239 %    90.616 ±  0.526 %
+   4       7.2110 ±    0.2849       0.04230 ±    0.00522       0.04298 ±    0.00164     5.902 ±  0.280 %    90.689 ±  0.454 %
+   5       7.1696 ±    0.2561       0.04858 ±    0.00492       0.04482 ±    0.00163     5.977 ±  0.239 %    90.635 ±  0.407 %
+   6       6.1208 ±    0.1933       0.04104 ±    0.00457       0.04600 ±    0.00166     6.514 ±  0.252 %    90.877 ±  0.368 %
+   7       5.5425 ±    0.1578       0.02881 ±    0.00441       0.05468 ±    0.00326     6.987 ±  0.263 %    90.951 ±  0.339 %
+   8       5.5348 ±    0.1471       0.02425 ±    0.00416       0.05443 ±    0.00290     6.941 ±  0.242 %    91.007 ±  0.316 %
+   9       5.8142 ±    0.1460       0.01736 ±    0.00406       0.05560 ±    0.00263     6.870 ±  0.222 %    90.638 ±  0.304 %
+  10       5.9193 ±    0.1421       0.01922 ±    0.00380       0.05317 ±    0.00238     6.733 ±  0.207 %    90.645 ±  0.288 %
+  11       6.0045 ±    0.1375       0.02128 ±    0.00356       0.05157 ±    0.00218     6.651 ±  0.196 %    90.731 ±  0.273 %
+  12       6.2508 ±    0.1385       0.02058 ±    0.00335       0.05003 ±    0.00200     6.500 ±  0.185 %    90.803 ±  0.261 %
+  13       6.3463 ±    0.1354       0.02070 ±    0.00317       0.04889 ±    0.00185     6.433 ±  0.177 %    90.917 ±  0.249 %
+  14       6.4064 ±    0.1316       0.01969 ±    0.00302       0.04772 ±    0.00173     6.353 ±  0.169 %    90.888 ±  0.240 %
+  15       6.4579 ±    0.1284       0.01972 ±    0.00289       0.04674 ±    0.00162     6.251 ±  0.161 %    90.948 ±  0.232 %
+  16       6.7292 ±    0.1315       0.01453 ±    0.00292       0.04759 ±    0.00159     6.185 ±  0.153 %    90.848 ±  0.225 %
+  17       6.7855 ±    0.1281       0.01448 ±    0.00280       0.04656 ±    0.00150     6.097 ±  0.146 %    90.903 ±  0.218 %
+  18       6.8961 ±    0.1267       0.01462 ±    0.00269       0.04590 ±    0.00142     6.026 ±  0.140 %    90.958 ±  0.211 %
+  19       6.8469 ±    0.1230       0.01601 ±    0.00260       0.04524 ±    0.00136     5.969 ±  0.137 %    91.033 ±  0.205 %
+  20       6.6773 ±    0.1165       0.01807 ±    0.00256       0.04666 ±    0.00132     6.169 ±  0.138 %    91.031 ±  0.200 %
+  21       6.6797 ±    0.1133       0.01734 ±    0.00248       0.04626 ±    0.00127     6.149 ±  0.134 %    91.035 ±  0.195 %
+  22       6.7091 ±    0.1113       0.01836 ±    0.00240       0.04573 ±    0.00121     6.112 ±  0.131 %    91.016 ±  0.191 %
+  23       6.7538 ±    0.1096       0.01833 ±    0.00233       0.04527 ±    0.00116     6.093 ±  0.126 %    91.075 ±  0.186 %
+  24       6.7698 ±    0.1074       0.01868 ±    0.00227       0.04457 ±    0.00112     6.032 ±  0.123 %    91.109 ±  0.182 %
+  25       6.8184 ±    0.1061       0.01868 ±    0.00221       0.04415 ±    0.00107     5.982 ±  0.119 %    91.112 ±  0.178 %
+  26       6.7958 ±    0.1037       0.01895 ±    0.00216       0.04391 ±    0.00104     5.995 ±  0.117 %    91.165 ±  0.174 %
+  27       6.9355 ±    0.1043       0.01909 ±    0.00210       0.04335 ±    0.00100     5.932 ±  0.114 %    91.181 ±  0.171 %
+  28       7.0181 ±    0.1040       0.01911 ±    0.00205       0.04287 ±    0.00097     5.886 ±  0.112 %    91.206 ±  0.167 %
+  29       7.0216 ±    0.1022       0.01904 ±    0.00200       0.04249 ±    0.00094     5.854 ±  0.109 %    91.229 ±  0.164 %
+  30       6.9873 ±    0.0999       0.01891 ±    0.00196       0.04236 ±    0.00091     5.830 ±  0.106 %    91.225 ±  0.162 %
+  31       6.9137 ±    0.0972       0.01937 ±    0.00193       0.04231 ±    0.00089     5.810 ±  0.104 %    91.272 ±  0.158 %
+  32       6.8167 ±    0.0942       0.02022 ±    0.00191       0.04265 ±    0.00087     5.853 ±  0.101 %    91.282 ±  0.156 %
+  33       6.8131 ±    0.0929       0.02393 ±    0.00194       0.04399 ±    0.00092     5.938 ±  0.104 %    91.256 ±  0.154 %
+  34       6.8092 ±    0.0914       0.02408 ±    0.00190       0.04371 ±    0.00089     5.916 ±  0.102 %    91.251 ±  0.152 %
+  35       6.8282 ±    0.0904       0.02472 ±    0.00188       0.04377 ±    0.00088     5.935 ±  0.103 %    91.250 ±  0.149 %
+  36       6.8368 ±    0.0893       0.02544 ±    0.00185       0.04362 ±    0.00087     5.903 ±  0.100 %    91.295 ±  0.147 %
+  37       6.7206 ±    0.0862       0.02570 ±    0.00182       0.04326 ±    0.00085     5.880 ±  0.098 %    91.284 ±  0.145 %
+  38       6.6518 ±    0.0839       0.02525 ±    0.00178       0.04302 ±    0.00083     5.883 ±  0.097 %    91.316 ±  0.143 %
+  39       6.5787 ±    0.0817       0.02469 ±    0.00178       0.04401 ±    0.00107     5.965 ±  0.098 %    91.265 ±  0.141 %
+  40       6.4877 ±    0.0792       0.02466 ±    0.00175       0.04379 ±    0.00105     5.960 ±  0.096 %    91.293 ±  0.139 %
+
+====== Perplexity statistics ======
+Mean PPL(Q)                   :   6.487656 ±   0.079193
+Mean PPL(base)                :   6.329637 ±   0.076239
+Cor(ln(PPL(Q)), ln(PPL(base))):  98.97%
+Mean ln(PPL(Q)/PPL(base))     :   0.024658 ±   0.001747
+Mean PPL(Q)/PPL(base)         :   1.024965 ±   0.001790
+Mean PPL(Q)-PPL(base)         :   0.158019 ±   0.011529
+
+====== KL divergence statistics ======
+Mean    KLD:   0.043788 ±   0.001045
+Maximum KLD:  27.035896
+99.9%   KLD:   1.728752
+99.0%   KLD:   0.426193
+95.0%   KLD:   0.138685
+90.0%   KLD:   0.084163
+Median  KLD:   0.018434
+10.0%   KLD:   0.000298
+ 5.0%   KLD:   0.000074
+ 1.0%   KLD:   0.000008
+ 0.1%   KLD:   0.000000
+Minimum KLD:  -0.000025
+
+====== Token probability statistics ======
+Mean    Δp: -0.150 ± 0.029 %
+Maximum Δp: 99.991%
+99.9%   Δp: 39.974%
+99.0%   Δp: 15.865%
+95.0%   Δp:  7.331%
+90.0%   Δp:  4.280%
+75.0%   Δp:  0.875%
+Median  Δp: -0.001%
+25.0%   Δp: -0.976%
+10.0%   Δp: -4.560%
+ 5.0%   Δp: -7.978%
+ 1.0%   Δp: -18.131%
+ 0.1%   Δp: -47.519%
+Minimum Δp: -99.992%
+RMS Δp    :  5.960 ± 0.096 %
+Same top p: 91.293 ± 0.139 %
+

recipe/logs/R3_kld_q106i.log

@@ -0,0 +1,92 @@
+0.00.038.696 I common_init_result: fitting params to device memory ...
+0.00.038.700 I common_init_result: (for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)
+0.00.498.231 W llama_model_loader: direct I/O is enabled, disabling mmap
+0.22.903.012 W llama_context: n_ctx_seq (2048) < n_ctx_train (262144) -- the full capacity of the model will not be utilized
+0.22.972.787 W common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+0.23.130.908 I 
+0.23.131.056 I system_info: n_threads = 16 (n_threads_batch = 16) / 32 | ROCm : NO_VMM = 1 | PEER_MAX_BATCH_SIZE = 128 | FA_ALL_QUANTS = 1 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+0.23.260.916 I kl_divergence: computing over 40 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+0.31.660.763 I kl_divergence: 8.40 seconds per pass - ETA 5.58 minutes
+
+chunk             PPL               ln(PPL(Q)/PPL(base))          KL Divergence              Δp RMS            Same top p
+   1       5.6408 ±    0.4130       0.00892 ±    0.00851       0.02884 ±    0.00223     5.006 ±  0.404 %    92.082 ±  0.845 %
+   2       6.7620 ±    0.3622       0.02148 ±    0.00672       0.03521 ±    0.00179     5.173 ±  0.314 %    91.447 ±  0.618 %
+   3       6.9615 ±    0.3040       0.02151 ±    0.00548       0.03504 ±    0.00172     5.651 ±  0.327 %    91.268 ±  0.510 %
+   4       7.1890 ±    0.2834       0.03924 ±    0.00539       0.03832 ±    0.00181     6.014 ±  0.333 %    91.105 ±  0.445 %
+   5       7.0853 ±    0.2513       0.03674 ±    0.00474       0.03905 ±    0.00161     5.949 ±  0.285 %    90.948 ±  0.401 %
+   6       6.0847 ±    0.1909       0.03514 ±    0.00433       0.03877 ±    0.00140     6.087 ±  0.247 %    91.170 ±  0.362 %
+   7       5.5234 ±    0.1564       0.02535 ±    0.00421       0.04894 ±    0.00389     6.737 ±  0.274 %    91.286 ±  0.333 %
+   8       5.5154 ±    0.1457       0.02076 ±    0.00401       0.04901 ±    0.00344     6.677 ±  0.248 %    91.178 ±  0.314 %
+   9       5.8123 ±    0.1454       0.01704 ±    0.00383       0.04897 ±    0.00308     6.620 ±  0.229 %    90.768 ±  0.302 %
+  10       5.9091 ±    0.1412       0.01750 ±    0.00354       0.04633 ±    0.00277     6.413 ±  0.213 %    90.938 ±  0.284 %
+  11       5.9785 ±    0.1361       0.01695 ±    0.00329       0.04449 ±    0.00253     6.286 ±  0.199 %    91.105 ±  0.268 %
+  12       6.2236 ±    0.1372       0.01623 ±    0.00310       0.04303 ±    0.00232     6.175 ±  0.188 %    91.096 ±  0.257 %
+  13       6.3097 ±    0.1337       0.01490 ±    0.00297       0.04242 ±    0.00215     6.107 ±  0.177 %    91.150 ±  0.246 %
+  14       6.3726 ±    0.1301       0.01440 ±    0.00282       0.04117 ±    0.00199     5.997 ±  0.168 %    91.063 ±  0.238 %
+  15       6.4245 ±    0.1269       0.01453 ±    0.00270       0.04008 ±    0.00186     5.886 ±  0.160 %    91.065 ±  0.230 %
+  16       6.6579 ±    0.1287       0.00389 ±    0.00295       0.04291 ±    0.00185     5.918 ±  0.153 %    90.805 ±  0.226 %
+  17       6.7177 ±    0.1255       0.00444 ±    0.00282       0.04183 ±    0.00174     5.817 ±  0.147 %    90.932 ±  0.218 %
+  18       6.8318 ±    0.1243       0.00525 ±    0.00270       0.04111 ±    0.00165     5.756 ±  0.141 %    90.936 ±  0.212 %
+  19       6.7788 ±    0.1205       0.00602 ±    0.00258       0.04025 ±    0.00156     5.692 ±  0.138 %    91.027 ±  0.205 %
+  20       6.6083 ±    0.1141       0.00768 ±    0.00253       0.04120 ±    0.00150     5.772 ±  0.132 %    91.051 ±  0.200 %
+  21       6.6069 ±    0.1109       0.00638 ±    0.00246       0.04124 ±    0.00147     5.771 ±  0.127 %    91.007 ±  0.195 %
+  22       6.6339 ±    0.1089       0.00710 ±    0.00238       0.04060 ±    0.00140     5.746 ±  0.125 %    90.989 ±  0.191 %
+  23       6.6839 ±    0.1073       0.00793 ±    0.00230       0.04010 ±    0.00134     5.717 ±  0.121 %    91.041 ±  0.186 %
+  24       6.7004 ±    0.1052       0.00837 ±    0.00223       0.03946 ±    0.00129     5.664 ±  0.117 %    91.109 ±  0.182 %
+  25       6.7488 ±    0.1040       0.00842 ±    0.00217       0.03909 ±    0.00124     5.618 ±  0.114 %    91.140 ±  0.178 %
+  26       6.7278 ±    0.1016       0.00891 ±    0.00212       0.03881 ±    0.00120     5.607 ±  0.111 %    91.221 ±  0.174 %
+  27       6.8677 ±    0.1023       0.00927 ±    0.00206       0.03825 ±    0.00115     5.549 ±  0.109 %    91.278 ±  0.170 %
+  28       6.9512 ±    0.1020       0.00954 ±    0.00201       0.03777 ±    0.00112     5.514 ±  0.107 %    91.342 ±  0.166 %
+  29       6.9561 ±    0.1003       0.00968 ±    0.00195       0.03732 ±    0.00108     5.479 ±  0.104 %    91.364 ±  0.163 %
+  30       6.9223 ±    0.0980       0.00957 ±    0.00192       0.03710 ±    0.00104     5.450 ±  0.102 %    91.398 ±  0.160 %
+  31       6.8485 ±    0.0953       0.00989 ±    0.00188       0.03699 ±    0.00101     5.434 ±  0.099 %    91.433 ±  0.157 %
+  32       6.7464 ±    0.0923       0.00984 ±    0.00185       0.03739 ±    0.00099     5.474 ±  0.097 %    91.419 ±  0.155 %
+  33       6.7483 ±    0.0912       0.01437 ±    0.00190       0.03958 ±    0.00108     5.645 ±  0.109 %    91.333 ±  0.153 %
+  34       6.7441 ±    0.0897       0.01448 ±    0.00187       0.03924 ±    0.00105     5.613 ±  0.107 %    91.383 ±  0.150 %
+  35       6.7648 ±    0.0888       0.01539 ±    0.00184       0.03917 ±    0.00103     5.611 ±  0.107 %    91.387 ±  0.148 %
+  36       6.7723 ±    0.0877       0.01597 ±    0.00181       0.03893 ±    0.00100     5.594 ±  0.104 %    91.439 ±  0.146 %
+  37       6.6580 ±    0.0846       0.01634 ±    0.00177       0.03863 ±    0.00098     5.581 ±  0.102 %    91.467 ±  0.144 %
+  38       6.5925 ±    0.0824       0.01630 ±    0.00174       0.03835 ±    0.00096     5.576 ±  0.101 %    91.508 ±  0.141 %
+  39       6.5182 ±    0.0802       0.01545 ±    0.00173       0.03894 ±    0.00101     5.623 ±  0.100 %    91.498 ±  0.140 %
+  40       6.4269 ±    0.0777       0.01525 ±    0.00170       0.03872 ±    0.00099     5.622 ±  0.098 %    91.522 ±  0.138 %
+
+====== Perplexity statistics ======
+Mean PPL(Q)                   :   6.426891 ±   0.077720
+Mean PPL(base)                :   6.329637 ±   0.076239
+Cor(ln(PPL(Q)), ln(PPL(base))):  99.01%
+Mean ln(PPL(Q)/PPL(base))     :   0.015248 ±   0.001701
+Mean PPL(Q)/PPL(base)         :   1.015365 ±   0.001727
+Mean PPL(Q)-PPL(base)         :   0.097253 ±   0.010943
+
+====== KL divergence statistics ======
+Mean    KLD:   0.038720 ±   0.000987
+Maximum KLD:  21.346630
+99.9%   KLD:   1.791168
+99.0%   KLD:   0.386349
+95.0%   KLD:   0.118341
+90.0%   KLD:   0.070348
+Median  KLD:   0.015312
+10.0%   KLD:   0.000257
+ 5.0%   KLD:   0.000059
+ 1.0%   KLD:   0.000006
+ 0.1%   KLD:   0.000000
+Minimum KLD:  -0.000033
+
+====== Token probability statistics ======
+Mean    Δp: -0.245 ± 0.028 %
+Maximum Δp: 94.355%
+99.9%   Δp: 39.023%
+99.0%   Δp: 14.874%
+95.0%   Δp:  6.653%
+90.0%   Δp:  3.760%
+75.0%   Δp:  0.671%
+Median  Δp: -0.002%
+25.0%   Δp: -1.033%
+10.0%   Δp: -4.485%
+ 5.0%   Δp: -7.481%
+ 1.0%   Δp: -17.039%
+ 0.1%   Δp: -44.703%
+Minimum Δp: -98.759%
+RMS Δp    :  5.622 ± 0.098 %
+Same top p: 91.522 ± 0.138 %
+

recipe/logs/R3_kld_q111.log

@@ -0,0 +1,92 @@
+0.00.033.573 I common_init_result: fitting params to device memory ...
+0.00.033.577 I common_init_result: (for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)
+0.00.407.709 W llama_model_loader: direct I/O is enabled, disabling mmap
+0.36.544.614 W llama_context: n_ctx_seq (2048) < n_ctx_train (262144) -- the full capacity of the model will not be utilized
+0.36.609.559 W common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+0.36.812.163 I 
+0.36.812.307 I system_info: n_threads = 16 (n_threads_batch = 16) / 32 | ROCm : NO_VMM = 1 | PEER_MAX_BATCH_SIZE = 128 | FA_ALL_QUANTS = 1 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+0.36.932.555 I kl_divergence: computing over 40 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+0.44.986.034 I kl_divergence: 8.05 seconds per pass - ETA 5.37 minutes
+
+chunk             PPL               ln(PPL(Q)/PPL(base))          KL Divergence              Δp RMS            Same top p
+   1       5.6059 ±    0.4093       0.00271 ±    0.00332       0.00457 ±    0.00030     1.953 ±  0.126 %    97.556 ±  0.483 %
+   2       6.6305 ±    0.3529       0.00184 ±    0.00264       0.00589 ±    0.00036     2.222 ±  0.169 %    96.432 ±  0.410 %
+   3       6.8224 ±    0.2960       0.00133 ±    0.00209       0.00587 ±    0.00032     2.374 ±  0.190 %    96.579 ±  0.328 %
+   4       6.8978 ±    0.2659      -0.00211 ±    0.00189       0.00618 ±    0.00037     2.424 ±  0.178 %    96.896 ±  0.271 %
+   5       6.8264 ±    0.2374      -0.00048 ±    0.00173       0.00648 ±    0.00036     2.454 ±  0.166 %    96.618 ±  0.253 %
+   6       5.8711 ±    0.1809      -0.00060 ±    0.00159       0.00674 ±    0.00032     2.581 ±  0.143 %    96.514 ±  0.234 %
+   7       5.3771 ±    0.1501      -0.00150 ±    0.00194       0.01538 ±    0.00307     3.622 ±  0.322 %    96.411 ±  0.220 %
+   8       5.3744 ±    0.1400      -0.00515 ±    0.00201       0.01627 ±    0.00271     3.732 ±  0.290 %    96.017 ±  0.216 %
+   9       5.6841 ±    0.1409      -0.00528 ±    0.00191       0.01570 ±    0.00241     3.648 ±  0.268 %    95.786 ±  0.209 %
+  10       5.7797 ±    0.1368      -0.00463 ±    0.00178       0.01478 ±    0.00218     3.529 ±  0.250 %    95.894 ±  0.196 %
+  11       5.8547 ±    0.1322      -0.00399 ±    0.00166       0.01394 ±    0.00198     3.437 ±  0.233 %    95.992 ±  0.185 %
+  12       6.1082 ±    0.1337      -0.00249 ±    0.00158       0.01362 ±    0.00182     3.345 ±  0.220 %    95.984 ±  0.177 %
+  13       6.1990 ±    0.1306      -0.00280 ±    0.00150       0.01316 ±    0.00168     3.307 ±  0.207 %    96.030 ±  0.169 %
+  14       6.2654 ±    0.1271      -0.00256 ±    0.00143       0.01272 ±    0.00156     3.246 ±  0.197 %    95.971 ±  0.164 %
+  15       6.3162 ±    0.1240      -0.00247 ±    0.00140       0.01234 ±    0.00146     3.214 ±  0.188 %    95.947 ±  0.159 %
+  16       6.6139 ±    0.1282      -0.00275 ±    0.00152       0.01298 ±    0.00143     3.260 ±  0.188 %    95.870 ±  0.156 %
+  17       6.6699 ±    0.1249      -0.00271 ±    0.00145       0.01261 ±    0.00134     3.203 ±  0.180 %    95.900 ±  0.150 %
+  18       6.7785 ±    0.1236      -0.00259 ±    0.00139       0.01226 ±    0.00127     3.156 ±  0.173 %    95.873 ±  0.147 %
+  19       6.7237 ±    0.1198      -0.00214 ±    0.00133       0.01187 ±    0.00120     3.102 ±  0.166 %    95.879 ±  0.143 %
+  20       6.5488 ±    0.1132      -0.00136 ±    0.00132       0.01205 ±    0.00116     3.201 ±  0.168 %    95.860 ±  0.139 %
+  21       6.5550 ±    0.1102      -0.00150 ±    0.00127       0.01203 ±    0.00112     3.206 ±  0.162 %    95.890 ±  0.135 %
+  22       6.5767 ±    0.1081      -0.00156 ±    0.00122       0.01174 ±    0.00107     3.165 ±  0.157 %    95.894 ±  0.132 %
+  23       6.6160 ±    0.1063      -0.00229 ±    0.00119       0.01167 ±    0.00103     3.177 ±  0.153 %    95.937 ±  0.129 %
+  24       6.6269 ±    0.1041      -0.00265 ±    0.00115       0.01151 ±    0.00098     3.149 ±  0.148 %    95.951 ±  0.126 %
+  25       6.6721 ±    0.1028      -0.00301 ±    0.00112       0.01137 ±    0.00094     3.126 ±  0.144 %    95.957 ±  0.123 %
+  26       6.6480 ±    0.1004      -0.00303 ±    0.00109       0.01114 ±    0.00091     3.100 ±  0.140 %    96.000 ±  0.120 %
+  27       6.7853 ±    0.1010      -0.00280 ±    0.00106       0.01089 ±    0.00088     3.058 ±  0.136 %    96.036 ±  0.117 %
+  28       6.8669 ±    0.1008      -0.00267 ±    0.00105       0.01107 ±    0.00087     3.023 ±  0.133 %    96.031 ±  0.115 %
+  29       6.8723 ±    0.0991      -0.00244 ±    0.00102       0.01088 ±    0.00084     2.993 ±  0.130 %    96.039 ±  0.113 %
+  30       6.8393 ±    0.0969      -0.00250 ±    0.00100       0.01073 ±    0.00082     2.968 ±  0.127 %    96.067 ±  0.111 %
+  31       6.7637 ±    0.0941      -0.00257 ±    0.00097       0.01061 ±    0.00079     2.947 ±  0.124 %    96.096 ±  0.109 %
+  32       6.6655 ±    0.0912      -0.00222 ±    0.00096       0.01054 ±    0.00076     2.964 ±  0.119 %    96.093 ±  0.107 %
+  33       6.6325 ±    0.0893      -0.00292 ±    0.00096       0.01058 ±    0.00074     2.953 ±  0.116 %    96.066 ±  0.106 %
+  34       6.6281 ±    0.0879      -0.00287 ±    0.00093       0.01044 ±    0.00072     2.932 ±  0.114 %    96.050 ±  0.104 %
+  35       6.6458 ±    0.0869      -0.00235 ±    0.00094       0.01053 ±    0.00073     2.969 ±  0.119 %    96.017 ±  0.103 %
+  36       6.6507 ±    0.0857      -0.00216 ±    0.00092       0.01043 ±    0.00071     2.951 ±  0.116 %    96.033 ±  0.102 %
+  37       6.5368 ±    0.0827      -0.00203 ±    0.00090       0.01028 ±    0.00069     2.933 ±  0.114 %    96.032 ±  0.100 %
+  38       6.4710 ±    0.0805      -0.00231 ±    0.00088       0.01017 ±    0.00067     2.915 ±  0.112 %    96.051 ±  0.099 %
+  39       6.4058 ±    0.0785      -0.00194 ±    0.00091       0.01088 ±    0.00082     2.993 ±  0.113 %    96.047 ±  0.098 %
+  40       6.3176 ±    0.0761      -0.00190 ±    0.00089       0.01081 ±    0.00080     2.996 ±  0.110 %    96.043 ±  0.096 %
+
+====== Perplexity statistics ======
+Mean PPL(Q)                   :   6.317616 ±   0.076081
+Mean PPL(base)                :   6.329637 ±   0.076239
+Cor(ln(PPL(Q)), ln(PPL(base))):  99.72%
+Mean ln(PPL(Q)/PPL(base))     :  -0.001901 ±   0.000893
+Mean PPL(Q)/PPL(base)         :   0.998101 ±   0.000892
+Mean PPL(Q)-PPL(base)         :  -0.012022 ±   0.005651
+
+====== KL divergence statistics ======
+Mean    KLD:   0.010810 ±   0.000803
+Maximum KLD:  19.487749
+99.9%   KLD:   0.624512
+99.0%   KLD:   0.085990
+95.0%   KLD:   0.025315
+90.0%   KLD:   0.015265
+Median  KLD:   0.003010
+10.0%   KLD:   0.000068
+ 5.0%   KLD:   0.000017
+ 1.0%   KLD:   0.000002
+ 0.1%   KLD:  -0.000002
+Minimum KLD:  -0.000031
+
+====== Token probability statistics ======
+Mean    Δp: -0.013 ± 0.015 %
+Maximum Δp: 90.262%
+99.9%   Δp: 20.093%
+99.0%   Δp:  7.391%
+95.0%   Δp:  3.201%
+90.0%   Δp:  1.823%
+75.0%   Δp:  0.376%
+Median  Δp: -0.000%
+25.0%   Δp: -0.395%
+10.0%   Δp: -1.832%
+ 5.0%   Δp: -3.218%
+ 1.0%   Δp: -7.366%
+ 0.1%   Δp: -21.522%
+Minimum Δp: -98.323%
+RMS Δp    :  2.996 ± 0.110 %
+Same top p: 96.043 ± 0.096 %
+

recipe/logs/R3_kld_q115.log

@@ -0,0 +1,92 @@
+0.00.046.645 I common_init_result: fitting params to device memory ...
+0.00.046.648 I common_init_result: (for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)
+0.00.521.234 W llama_model_loader: direct I/O is enabled, disabling mmap
+0.46.661.487 W llama_context: n_ctx_seq (2048) < n_ctx_train (262144) -- the full capacity of the model will not be utilized
+0.46.725.915 W common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+0.46.946.305 I 
+0.46.946.435 I system_info: n_threads = 16 (n_threads_batch = 16) / 32 | ROCm : NO_VMM = 1 | PEER_MAX_BATCH_SIZE = 128 | FA_ALL_QUANTS = 1 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+0.47.074.269 I kl_divergence: computing over 40 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+0.55.826.309 I kl_divergence: 8.75 seconds per pass - ETA 5.83 minutes
+
+chunk             PPL               ln(PPL(Q)/PPL(base))          KL Divergence              Δp RMS            Same top p
+   1       5.5566 ±    0.4036      -0.00612 ±    0.00269       0.00307 ±    0.00021     1.587 ±  0.124 %    97.361 ±  0.501 %
+   2       6.6107 ±    0.3509      -0.00116 ±    0.00204       0.00344 ±    0.00028     1.785 ±  0.224 %    96.725 ±  0.394 %
+   3       6.8101 ±    0.2950      -0.00048 ±    0.00157       0.00307 ±    0.00019     1.656 ±  0.165 %    96.905 ±  0.313 %
+   4       6.8838 ±    0.2650      -0.00414 ±    0.00214       0.00680 ±    0.00217     2.704 ±  0.512 %    97.092 ±  0.263 %
+   5       6.8057 ±    0.2361      -0.00351 ±    0.00188       0.00646 ±    0.00174     2.561 ±  0.433 %    96.950 ±  0.240 %
+   6       5.8506 ±    0.1799      -0.00410 ±    0.00168       0.00632 ±    0.00145     2.565 ±  0.364 %    97.019 ±  0.217 %
+   7       5.3534 ±    0.1491      -0.00591 ±    0.00178       0.01395 ±    0.00318     3.427 ±  0.353 %    96.914 ±  0.204 %
+   8       5.3640 ±    0.1397      -0.00709 ±    0.00177       0.01399 ±    0.00279     3.454 ±  0.318 %    96.615 ±  0.200 %
+   9       5.6737 ±    0.1405      -0.00711 ±    0.00168       0.01328 ±    0.00249     3.390 ±  0.293 %    96.448 ±  0.193 %
+  10       5.7712 ±    0.1365      -0.00611 ±    0.00157       0.01235 ±    0.00224     3.254 ±  0.275 %    96.549 ±  0.180 %
+  11       5.8449 ±    0.1319      -0.00566 ±    0.00145       0.01146 ±    0.00204     3.131 ±  0.260 %    96.632 ±  0.170 %
+  12       6.0885 ±    0.1330      -0.00572 ±    0.00135       0.01089 ±    0.00187     3.029 ±  0.246 %    96.636 ±  0.163 %
+  13       6.1823 ±    0.1300      -0.00550 ±    0.00128       0.01035 ±    0.00173     2.964 ±  0.233 %    96.737 ±  0.154 %
+  14       6.2493 ±    0.1266      -0.00513 ±    0.00121       0.00991 ±    0.00160     2.893 ±  0.222 %    96.676 ±  0.150 %
+  15       6.3012 ±    0.1236      -0.00484 ±    0.00118       0.00950 ±    0.00150     2.824 ±  0.212 %    96.624 ±  0.146 %
+  16       6.5894 ±    0.1274      -0.00646 ±    0.00132       0.00985 ±    0.00145     2.789 ±  0.202 %    96.573 ±  0.142 %
+  17       6.6478 ±    0.1242      -0.00603 ±    0.00125       0.00951 ±    0.00136     2.733 ±  0.194 %    96.607 ±  0.137 %
+  18       6.7575 ±    0.1230      -0.00569 ±    0.00120       0.00918 ±    0.00129     2.682 ±  0.187 %    96.584 ±  0.134 %
+  19       6.6994 ±    0.1190      -0.00576 ±    0.00116       0.00902 ±    0.00122     2.646 ±  0.179 %    96.568 ±  0.131 %
+  20       6.5206 ±    0.1124      -0.00569 ±    0.00113       0.00908 ±    0.00116     2.728 ±  0.179 %    96.559 ±  0.127 %
+  21       6.5276 ±    0.1095      -0.00569 ±    0.00109       0.00906 ±    0.00113     2.745 ±  0.173 %    96.607 ±  0.124 %
+  22       6.5501 ±    0.1074      -0.00563 ±    0.00105       0.00880 ±    0.00108     2.704 ±  0.168 %    96.632 ±  0.120 %
+  23       6.5926 ±    0.1057      -0.00582 ±    0.00102       0.00868 ±    0.00103     2.706 ±  0.164 %    96.651 ±  0.117 %
+  24       6.6052 ±    0.1035      -0.00594 ±    0.00098       0.00855 ±    0.00099     2.697 ±  0.160 %    96.656 ±  0.115 %
+  25       6.6526 ±    0.1023      -0.00594 ±    0.00095       0.00842 ±    0.00095     2.674 ±  0.156 %    96.669 ±  0.112 %
+  26       6.6305 ±    0.1000      -0.00567 ±    0.00092       0.00819 ±    0.00091     2.637 ±  0.152 %    96.691 ±  0.110 %
+  27       6.7675 ±    0.1006      -0.00543 ±    0.00089       0.00797 ±    0.00088     2.596 ±  0.149 %    96.727 ±  0.107 %
+  28       6.8491 ±    0.1004      -0.00526 ±    0.00090       0.00820 ±    0.00088     2.568 ±  0.145 %    96.711 ±  0.105 %
+  29       6.8537 ±    0.0986      -0.00515 ±    0.00087       0.00802 ±    0.00085     2.539 ±  0.142 %    96.703 ±  0.104 %
+  30       6.8219 ±    0.0964      -0.00505 ±    0.00085       0.00786 ±    0.00082     2.507 ±  0.139 %    96.729 ±  0.102 %
+  31       6.7475 ±    0.0937      -0.00497 ±    0.00082       0.00772 ±    0.00079     2.480 ±  0.136 %    96.771 ±  0.099 %
+  32       6.6475 ±    0.0908      -0.00492 ±    0.00081       0.00763 ±    0.00077     2.495 ±  0.135 %    96.774 ±  0.098 %
+  33       6.6137 ±    0.0889      -0.00577 ±    0.00082       0.00770 ±    0.00075     2.492 ±  0.131 %    96.750 ±  0.097 %
+  34       6.6095 ±    0.0875      -0.00569 ±    0.00080       0.00757 ±    0.00073     2.470 ±  0.129 %    96.748 ±  0.095 %
+  35       6.6264 ±    0.0865      -0.00527 ±    0.00080       0.00764 ±    0.00073     2.505 ±  0.134 %    96.738 ±  0.094 %
+  36       6.6315 ±    0.0853      -0.00505 ±    0.00078       0.00752 ±    0.00071     2.483 ±  0.132 %    96.742 ±  0.093 %
+  37       6.5180 ±    0.0823      -0.00492 ±    0.00076       0.00739 ±    0.00069     2.461 ±  0.129 %    96.735 ±  0.091 %
+  38       6.4529 ±    0.0802      -0.00512 ±    0.00075       0.00733 ±    0.00067     2.450 ±  0.127 %    96.738 ±  0.090 %
+  39       6.3883 ±    0.0781      -0.00467 ±    0.00074       0.00741 ±    0.00066     2.441 ±  0.124 %    96.742 ±  0.089 %
+  40       6.3014 ±    0.0758      -0.00447 ±    0.00073       0.00742 ±    0.00065     2.444 ±  0.121 %    96.735 ±  0.088 %
+
+====== Perplexity statistics ======
+Mean PPL(Q)                   :   6.301422 ±   0.075776
+Mean PPL(base)                :   6.329637 ±   0.076239
+Cor(ln(PPL(Q)), ln(PPL(base))):  99.82%
+Mean ln(PPL(Q)/PPL(base))     :  -0.004468 ±   0.000730
+Mean PPL(Q)/PPL(base)         :   0.995542 ±   0.000727
+Mean PPL(Q)-PPL(base)         :  -0.028216 ±   0.004630
+
+====== KL divergence statistics ======
+Mean    KLD:   0.007418 ±   0.000652
+Maximum KLD:  16.334919
+99.9%   KLD:   0.546826
+99.0%   KLD:   0.056239
+95.0%   KLD:   0.015157
+90.0%   KLD:   0.008705
+Median  KLD:   0.001653
+10.0%   KLD:   0.000034
+ 5.0%   KLD:   0.000008
+ 1.0%   KLD:   0.000000
+ 0.1%   KLD:  -0.000002
+Minimum KLD:  -0.000109
+
+====== Token probability statistics ======
+Mean    Δp:  0.043 ± 0.012 %
+Maximum Δp: 89.625%
+99.9%   Δp: 20.105%
+99.0%   Δp:  5.537%
+95.0%   Δp:  2.389%
+90.0%   Δp:  1.391%
+75.0%   Δp:  0.306%
+Median  Δp:  0.000%
+25.0%   Δp: -0.254%
+10.0%   Δp: -1.312%
+ 5.0%   Δp: -2.251%
+ 1.0%   Δp: -5.307%
+ 0.1%   Δp: -14.534%
+Minimum Δp: -99.365%
+RMS Δp    :  2.444 ± 0.121 %
+Same top p: 96.735 ± 0.088 %
+

recipe/logs/R4_kld_q106_repeat.log

@@ -0,0 +1,92 @@
+0.00.035.226 I common_init_result: fitting params to device memory ...
+0.00.035.230 I common_init_result: (for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)
+0.00.411.407 W llama_model_loader: direct I/O is enabled, disabling mmap
+0.20.064.888 W llama_context: n_ctx_seq (2048) < n_ctx_train (262144) -- the full capacity of the model will not be utilized
+0.20.130.441 W common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+0.20.272.500 I 
+0.20.272.642 I system_info: n_threads = 16 (n_threads_batch = 16) / 32 | ROCm : NO_VMM = 1 | PEER_MAX_BATCH_SIZE = 128 | FA_ALL_QUANTS = 1 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+0.20.391.428 I kl_divergence: computing over 40 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+0.27.395.913 I kl_divergence: 7.00 seconds per pass - ETA 4.67 minutes
+
+chunk             PPL               ln(PPL(Q)/PPL(base))          KL Divergence              Δp RMS            Same top p
+   1       5.6619 ±    0.4178       0.01265 ±    0.00809       0.03262 ±    0.00203     5.307 ±  0.369 %    91.593 ±  0.868 %
+   2       6.8229 ±    0.3691       0.03044 ±    0.00695       0.04270 ±    0.00205     5.525 ±  0.291 %    90.469 ±  0.649 %
+   3       7.0007 ±    0.3084       0.02712 ±    0.00560       0.03995 ±    0.00158     5.478 ±  0.239 %    90.616 ±  0.526 %
+   4       7.2110 ±    0.2849       0.04230 ±    0.00522       0.04298 ±    0.00164     5.902 ±  0.280 %    90.689 ±  0.454 %
+   5       7.1696 ±    0.2561       0.04858 ±    0.00492       0.04482 ±    0.00163     5.977 ±  0.239 %    90.635 ±  0.407 %
+   6       6.1208 ±    0.1933       0.04104 ±    0.00457       0.04600 ±    0.00166     6.514 ±  0.252 %    90.877 ±  0.368 %
+   7       5.5425 ±    0.1578       0.02881 ±    0.00441       0.05468 ±    0.00326     6.987 ±  0.263 %    90.951 ±  0.339 %
+   8       5.5348 ±    0.1471       0.02425 ±    0.00416       0.05443 ±    0.00290     6.941 ±  0.242 %    91.007 ±  0.316 %
+   9       5.8142 ±    0.1460       0.01736 ±    0.00406       0.05560 ±    0.00263     6.870 ±  0.222 %    90.638 ±  0.304 %
+  10       5.9193 ±    0.1421       0.01922 ±    0.00380       0.05317 ±    0.00238     6.733 ±  0.207 %    90.645 ±  0.288 %
+  11       6.0045 ±    0.1375       0.02128 ±    0.00356       0.05157 ±    0.00218     6.651 ±  0.196 %    90.731 ±  0.273 %
+  12       6.2508 ±    0.1385       0.02058 ±    0.00335       0.05003 ±    0.00200     6.500 ±  0.185 %    90.803 ±  0.261 %
+  13       6.3463 ±    0.1354       0.02070 ±    0.00317       0.04889 ±    0.00185     6.433 ±  0.177 %    90.917 ±  0.249 %
+  14       6.4064 ±    0.1316       0.01969 ±    0.00302       0.04772 ±    0.00173     6.353 ±  0.169 %    90.888 ±  0.240 %
+  15       6.4579 ±    0.1284       0.01972 ±    0.00289       0.04674 ±    0.00162     6.251 ±  0.161 %    90.948 ±  0.232 %
+  16       6.7292 ±    0.1315       0.01453 ±    0.00292       0.04759 ±    0.00159     6.185 ±  0.153 %    90.848 ±  0.225 %
+  17       6.7855 ±    0.1281       0.01448 ±    0.00280       0.04656 ±    0.00150     6.097 ±  0.146 %    90.903 ±  0.218 %
+  18       6.8961 ±    0.1267       0.01462 ±    0.00269       0.04590 ±    0.00142     6.026 ±  0.140 %    90.958 ±  0.211 %
+  19       6.8469 ±    0.1230       0.01601 ±    0.00260       0.04524 ±    0.00136     5.969 ±  0.137 %    91.033 ±  0.205 %
+  20       6.6773 ±    0.1165       0.01807 ±    0.00256       0.04666 ±    0.00132     6.169 ±  0.138 %    91.031 ±  0.200 %
+  21       6.6797 ±    0.1133       0.01734 ±    0.00248       0.04626 ±    0.00127     6.149 ±  0.134 %    91.035 ±  0.195 %
+  22       6.7091 ±    0.1113       0.01836 ±    0.00240       0.04573 ±    0.00121     6.112 ±  0.131 %    91.016 ±  0.191 %
+  23       6.7538 ±    0.1096       0.01833 ±    0.00233       0.04527 ±    0.00116     6.093 ±  0.126 %    91.075 ±  0.186 %
+  24       6.7698 ±    0.1074       0.01868 ±    0.00227       0.04457 ±    0.00112     6.032 ±  0.123 %    91.109 ±  0.182 %
+  25       6.8184 ±    0.1061       0.01868 ±    0.00221       0.04415 ±    0.00107     5.982 ±  0.119 %    91.112 ±  0.178 %
+  26       6.7958 ±    0.1037       0.01895 ±    0.00216       0.04391 ±    0.00104     5.995 ±  0.117 %    91.165 ±  0.174 %
+  27       6.9355 ±    0.1043       0.01909 ±    0.00210       0.04335 ±    0.00100     5.932 ±  0.114 %    91.181 ±  0.171 %
+  28       7.0181 ±    0.1040       0.01911 ±    0.00205       0.04287 ±    0.00097     5.886 ±  0.112 %    91.206 ±  0.167 %
+  29       7.0216 ±    0.1022       0.01904 ±    0.00200       0.04249 ±    0.00094     5.854 ±  0.109 %    91.229 ±  0.164 %
+  30       6.9873 ±    0.0999       0.01891 ±    0.00196       0.04236 ±    0.00091     5.830 ±  0.106 %    91.225 ±  0.162 %
+  31       6.9137 ±    0.0972       0.01937 ±    0.00193       0.04231 ±    0.00089     5.810 ±  0.104 %    91.272 ±  0.158 %
+  32       6.8167 ±    0.0942       0.02022 ±    0.00191       0.04265 ±    0.00087     5.853 ±  0.101 %    91.282 ±  0.156 %
+  33       6.8131 ±    0.0929       0.02393 ±    0.00194       0.04399 ±    0.00092     5.938 ±  0.104 %    91.256 ±  0.154 %
+  34       6.8092 ±    0.0914       0.02408 ±    0.00190       0.04371 ±    0.00089     5.916 ±  0.102 %    91.251 ±  0.152 %
+  35       6.8282 ±    0.0904       0.02472 ±    0.00188       0.04377 ±    0.00088     5.935 ±  0.103 %    91.250 ±  0.149 %
+  36       6.8368 ±    0.0893       0.02544 ±    0.00185       0.04362 ±    0.00087     5.903 ±  0.100 %    91.295 ±  0.147 %
+  37       6.7206 ±    0.0862       0.02570 ±    0.00182       0.04326 ±    0.00085     5.880 ±  0.098 %    91.284 ±  0.145 %
+  38       6.6518 ±    0.0839       0.02525 ±    0.00178       0.04302 ±    0.00083     5.883 ±  0.097 %    91.316 ±  0.143 %
+  39       6.5787 ±    0.0817       0.02469 ±    0.00178       0.04401 ±    0.00107     5.965 ±  0.098 %    91.265 ±  0.141 %
+  40       6.4877 ±    0.0792       0.02466 ±    0.00175       0.04379 ±    0.00105     5.960 ±  0.096 %    91.293 ±  0.139 %
+
+====== Perplexity statistics ======
+Mean PPL(Q)                   :   6.487656 ±   0.079193
+Mean PPL(base)                :   6.329637 ±   0.076239
+Cor(ln(PPL(Q)), ln(PPL(base))):  98.97%
+Mean ln(PPL(Q)/PPL(base))     :   0.024658 ±   0.001747
+Mean PPL(Q)/PPL(base)         :   1.024965 ±   0.001790
+Mean PPL(Q)-PPL(base)         :   0.158019 ±   0.011529
+
+====== KL divergence statistics ======
+Mean    KLD:   0.043788 ±   0.001045
+Maximum KLD:  27.035896
+99.9%   KLD:   1.728752
+99.0%   KLD:   0.426193
+95.0%   KLD:   0.138685
+90.0%   KLD:   0.084163
+Median  KLD:   0.018434
+10.0%   KLD:   0.000298
+ 5.0%   KLD:   0.000074
+ 1.0%   KLD:   0.000008
+ 0.1%   KLD:   0.000000
+Minimum KLD:  -0.000025
+
+====== Token probability statistics ======
+Mean    Δp: -0.150 ± 0.029 %
+Maximum Δp: 99.991%
+99.9%   Δp: 39.974%
+99.0%   Δp: 15.865%
+95.0%   Δp:  7.331%
+90.0%   Δp:  4.280%
+75.0%   Δp:  0.875%
+Median  Δp: -0.001%
+25.0%   Δp: -0.976%
+10.0%   Δp: -4.560%
+ 5.0%   Δp: -7.978%
+ 1.0%   Δp: -18.131%
+ 0.1%   Δp: -47.519%
+Minimum Δp: -99.992%
+RMS Δp    :  5.960 ± 0.096 %
+Same top p: 91.293 ± 0.139 %
+

recipe/logs/S_sizes.log

@@ -0,0 +1,6 @@
+S2_q106    quant size  = 17214.21 MiB (4.42 BPW)
+S2_q102    quant size  = 18180.88 MiB (4.67 BPW)
+S2_q115    quant size  = 32662.55 MiB (8.39 BPW)
+S2_q111    quant size  = 32205.84 MiB (8.27 BPW)
+S3_q106i   quant size  = 17214.21 MiB (4.42 BPW)
+S3_q102i   quant size  = 18180.88 MiB (4.67 BPW)

recipe/logs/afternoon_B1_ppl_bf16.log

@@ -0,0 +1,15 @@
+0.00.092.256 I common_init_result: fitting params to device memory ...
+0.00.092.259 I common_init_result: (for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)
+0.00.609.874 W llama_model_loader: direct I/O is enabled, disabling mmap
+1.19.756.478 W llama_context: n_ctx_seq (2048) < n_ctx_train (262144) -- the full capacity of the model will not be utilized
+1.19.820.478 W common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+1.20.207.139 I 
+1.20.207.240 I system_info: n_threads = 16 (n_threads_batch = 16) / 32 | ROCm : NO_VMM = 1 | PEER_MAX_BATCH_SIZE = 128 | FA_ALL_QUANTS = 1 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+1.20.207.327 I perplexity: saving all logits to kld/bf16.kld
+1.20.207.330 I perplexity: tokenizing the input ..
+1.20.544.564 I perplexity: tokenization took 337.224 ms
+1.20.544.662 I perplexity: calculating perplexity over 40 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+1.29.548.010 I perplexity: 8.82 seconds per pass - ETA 5.87 minutes
+[1]5.6019,[2]6.6236,[3]6.8194,[4]6.9173,[5]6.8369,[6]5.8754,[7]5.3855,[8]5.4015,[9]5.7130,[10]5.8071,[11]5.8787,[12]6.1229,[13]6.2160,[14]6.2810,[15]6.3313,[16]6.6308,[17]6.6871,[18]6.7961,[19]6.7386,[20]6.5601,[21]6.5666,[22]6.5887,[23]6.6323,[24]6.6455,[25]6.6935,[26]6.6695,[27]6.8055,[28]6.8864,[29]6.8906,[30]6.8577,[31]6.7818,[32]6.6810,[33]6.6536,[34]6.6486,[35]6.6628,[36]6.6664,[37]6.5514,[38]6.4871,[39]6.4183,[40]6.3294,
+7.22.831.014 I Final estimate: PPL = 6.3294 +/- 0.07627
+

recipe/logs/afternoon_B5_kld_q102.log

@@ -0,0 +1,92 @@
+0.00.058.988 I common_init_result: fitting params to device memory ...
+0.00.058.992 I common_init_result: (for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)
+0.00.523.545 W llama_model_loader: direct I/O is enabled, disabling mmap
+0.27.436.824 W llama_context: n_ctx_seq (2048) < n_ctx_train (262144) -- the full capacity of the model will not be utilized
+0.27.739.526 W common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+0.27.914.290 I 
+0.27.914.465 I system_info: n_threads = 16 (n_threads_batch = 16) / 32 | ROCm : NO_VMM = 1 | PEER_MAX_BATCH_SIZE = 128 | FA_ALL_QUANTS = 1 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+0.28.084.292 I kl_divergence: computing over 40 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+0.39.683.234 I kl_divergence: 11.60 seconds per pass - ETA 7.72 minutes
+
+chunk             PPL               ln(PPL(Q)/PPL(base))          KL Divergence              Δp RMS            Same top p
+   1       5.7067 ±    0.4219       0.01853 ±    0.00868       0.02815 ±    0.00217     5.106 ±  0.425 %    91.984 ±  0.849 %
+   2       6.7880 ±    0.3673       0.02449 ±    0.00670       0.03267 ±    0.00146     4.931 ±  0.285 %    91.984 ±  0.600 %
+   3       6.9694 ±    0.3067       0.02174 ±    0.00548       0.03315 ±    0.00155     4.998 ±  0.259 %    92.343 ±  0.480 %
+   4       7.0949 ±    0.2782       0.02534 ±    0.00468       0.03281 ±    0.00135     5.113 ±  0.250 %    92.326 ±  0.416 %
+   5       7.0303 ±    0.2486       0.02789 ±    0.00430       0.03576 ±    0.00133     5.400 ±  0.219 %    91.730 ±  0.385 %
+   6       6.0449 ±    0.1893       0.02842 ±    0.00404       0.03802 ±    0.00154     6.086 ±  0.260 %    91.659 ±  0.353 %
+   7       5.4972 ±    0.1555       0.02051 ±    0.00401       0.04623 ±    0.00323     6.630 ±  0.279 %    91.831 ±  0.324 %
+   8       5.4907 ±    0.1449       0.01638 ±    0.00379       0.04719 ±    0.00288     6.569 ±  0.252 %    91.777 ±  0.304 %
+   9       5.7798 ±    0.1444       0.01161 ±    0.00371       0.04764 ±    0.00260     6.438 ±  0.231 %    91.463 ±  0.291 %
+  10       5.8815 ±    0.1404       0.01305 ±    0.00347       0.04547 ±    0.00234     6.283 ±  0.215 %    91.574 ±  0.275 %
+  11       5.9546 ±    0.1355       0.01311 ±    0.00324       0.04370 ±    0.00214     6.151 ±  0.202 %    91.700 ±  0.260 %
+  12       6.1895 ±    0.1362       0.01108 ±    0.00305       0.04243 ±    0.00197     5.974 ±  0.191 %    91.748 ±  0.248 %
+  13       6.2796 ±    0.1330       0.01042 ±    0.00289       0.04149 ±    0.00183     5.933 ±  0.186 %    91.909 ±  0.236 %
+  14       6.3432 ±    0.1294       0.01009 ±    0.00275       0.04023 ±    0.00171     5.795 ±  0.177 %    91.880 ±  0.228 %
+  15       6.3972 ±    0.1265       0.01060 ±    0.00264       0.03927 ±    0.00160     5.708 ±  0.168 %    91.913 ±  0.220 %
+  16       6.6601 ±    0.1293       0.00470 ±    0.00278       0.04077 ±    0.00159     5.690 ±  0.161 %    91.746 ±  0.215 %
+  17       6.7181 ±    0.1260       0.00490 ±    0.00265       0.03978 ±    0.00150     5.606 ±  0.154 %    91.823 ±  0.208 %
+  18       6.8387 ±    0.1250       0.00666 ±    0.00255       0.03932 ±    0.00142     5.539 ±  0.147 %    91.843 ±  0.202 %
+  19       6.8021 ±    0.1217       0.00978 ±    0.00251       0.03969 ±    0.00137     5.601 ±  0.151 %    91.845 ±  0.196 %
+  20       6.6233 ±    0.1150       0.00997 ±    0.00247       0.04101 ±    0.00132     5.705 ±  0.143 %    91.848 ±  0.191 %
+  21       6.6267 ±    0.1118       0.00947 ±    0.00238       0.04071 ±    0.00127     5.690 ±  0.139 %    91.859 ±  0.187 %
+  22       6.6518 ±    0.1098       0.00994 ±    0.00230       0.03998 ±    0.00121     5.631 ±  0.135 %    91.900 ±  0.182 %
+  23       6.6989 ±    0.1082       0.01039 ±    0.00223       0.03943 ±    0.00116     5.602 ±  0.132 %    91.933 ±  0.178 %
+  24       6.7201 ±    0.1061       0.01153 ±    0.00216       0.03892 ±    0.00112     5.550 ±  0.128 %    91.952 ±  0.174 %
+  25       6.7704 ±    0.1050       0.01179 ±    0.00211       0.03857 ±    0.00107     5.505 ±  0.124 %    91.973 ±  0.170 %
+  26       6.7474 ±    0.1025       0.01194 ±    0.00206       0.03813 ±    0.00104     5.477 ±  0.121 %    92.052 ±  0.166 %
+  27       6.8869 ±    0.1032       0.01222 ±    0.00200       0.03752 ±    0.00100     5.416 ±  0.118 %    92.093 ±  0.162 %
+  28       6.9687 ±    0.1029       0.01219 ±    0.00195       0.03700 ±    0.00097     5.357 ±  0.115 %    92.138 ±  0.159 %
+  29       6.9685 ±    0.1010       0.01155 ±    0.00190       0.03657 ±    0.00093     5.313 ±  0.112 %    92.214 ±  0.156 %
+  30       6.9334 ±    0.0987       0.01127 ±    0.00186       0.03628 ±    0.00090     5.283 ±  0.109 %    92.216 ±  0.153 %
+  31       6.8581 ±    0.0960       0.01148 ±    0.00183       0.03611 ±    0.00088     5.276 ±  0.107 %    92.249 ±  0.150 %
+  32       6.7613 ±    0.0930       0.01222 ±    0.00180       0.03635 ±    0.00086     5.338 ±  0.104 %    92.217 ±  0.148 %
+  33       6.7304 ±    0.0912       0.01183 ±    0.00178       0.03627 ±    0.00083     5.329 ±  0.102 %    92.224 ±  0.146 %
+  34       6.7241 ±    0.0897       0.01163 ±    0.00175       0.03596 ±    0.00081     5.299 ±  0.100 %    92.200 ±  0.144 %
+  35       6.7459 ±    0.0887       0.01271 ±    0.00172       0.03593 ±    0.00080     5.316 ±  0.100 %    92.188 ±  0.142 %
+  36       6.7563 ±    0.0877       0.01371 ±    0.00170       0.03588 ±    0.00078     5.299 ±  0.098 %    92.221 ±  0.140 %
+  37       6.6418 ±    0.0846       0.01401 ±    0.00167       0.03553 ±    0.00077     5.278 ±  0.096 %    92.278 ±  0.137 %
+  38       6.5733 ±    0.0824       0.01350 ±    0.00164       0.03531 ±    0.00075     5.263 ±  0.094 %    92.298 ±  0.135 %
+  39       6.5036 ±    0.0802       0.01350 ±    0.00162       0.03565 ±    0.00078     5.308 ±  0.095 %    92.270 ±  0.134 %
+  40       6.4119 ±    0.0777       0.01323 ±    0.00159       0.03544 ±    0.00076     5.297 ±  0.093 %    92.314 ±  0.132 %
+
+====== Perplexity statistics ======
+Mean PPL(Q)                   :   6.411907 ±   0.077742
+Mean PPL(base)                :   6.327630 ±   0.076192
+Cor(ln(PPL(Q)), ln(PPL(base))):  99.14%
+Mean ln(PPL(Q)/PPL(base))     :   0.013231 ±   0.001591
+Mean PPL(Q)/PPL(base)         :   1.013319 ±   0.001612
+Mean PPL(Q)-PPL(base)         :   0.084277 ±   0.010236
+
+====== KL divergence statistics ======
+Mean    KLD:   0.035442 ±   0.000759
+Maximum KLD:  16.939222
+99.9%   KLD:   1.459745
+99.0%   KLD:   0.367319
+95.0%   KLD:   0.117396
+90.0%   KLD:   0.068599
+Median  KLD:   0.013563
+10.0%   KLD:   0.000242
+ 5.0%   KLD:   0.000061
+ 1.0%   KLD:   0.000006
+ 0.1%   KLD:   0.000000
+Minimum KLD:  -0.000039
+
+====== Token probability statistics ======
+Mean    Δp: -0.096 ± 0.026 %
+Maximum Δp: 92.911%
+99.9%   Δp: 33.288%
+99.0%   Δp: 14.066%
+95.0%   Δp:  6.242%
+90.0%   Δp:  3.613%
+75.0%   Δp:  0.797%
+Median  Δp:  0.000%
+25.0%   Δp: -0.727%
+10.0%   Δp: -3.611%
+ 5.0%   Δp: -6.663%
+ 1.0%   Δp: -17.432%
+ 0.1%   Δp: -45.413%
+Minimum Δp: -99.745%
+RMS Δp    :  5.297 ± 0.093 %
+Same top p: 92.314 ± 0.132 %
+

recipe/logs/afternoon_B5_kld_q102i.log

@@ -0,0 +1,92 @@
+0.00.035.205 I common_init_result: fitting params to device memory ...
+0.00.035.208 I common_init_result: (for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)
+0.00.407.141 W llama_model_loader: direct I/O is enabled, disabling mmap
+0.20.843.312 W llama_context: n_ctx_seq (2048) < n_ctx_train (262144) -- the full capacity of the model will not be utilized
+0.20.903.231 W common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+0.21.044.087 I 
+0.21.044.185 I system_info: n_threads = 16 (n_threads_batch = 16) / 32 | ROCm : NO_VMM = 1 | PEER_MAX_BATCH_SIZE = 128 | FA_ALL_QUANTS = 1 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+0.21.163.273 I kl_divergence: computing over 40 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+0.29.185.228 I kl_divergence: 8.02 seconds per pass - ETA 5.33 minutes
+
+chunk             PPL               ln(PPL(Q)/PPL(base))          KL Divergence              Δp RMS            Same top p
+   1       5.6624 ±    0.4130       0.01072 ±    0.00883       0.02711 ±    0.00287     5.306 ±  0.812 %    93.157 ±  0.790 %
+   2       6.7150 ±    0.3593       0.01369 ±    0.00661       0.02707 ±    0.00159     4.812 ±  0.461 %    92.229 ±  0.592 %
+   3       6.9054 ±    0.3010       0.01251 ±    0.00532       0.02723 ±    0.00151     4.986 ±  0.388 %    92.473 ±  0.476 %
+   4       7.0811 ±    0.2763       0.02339 ±    0.00538       0.03177 ±    0.00236     5.385 ±  0.404 %    92.522 ±  0.411 %
+   5       6.9868 ±    0.2455       0.02169 ±    0.00462       0.03245 ±    0.00201     5.425 ±  0.338 %    92.375 ±  0.371 %
+   6       6.0055 ±    0.1868       0.02188 ±    0.00437       0.03425 ±    0.00188     5.931 ±  0.315 %    92.278 ±  0.341 %
+   7       5.4627 ±    0.1534       0.01421 ±    0.00419       0.04334 ±    0.00399     6.453 ±  0.314 %    92.487 ±  0.312 %
+   8       5.4645 ±    0.1432       0.01158 ±    0.00391       0.04363 ±    0.00352     6.423 ±  0.283 %    92.326 ±  0.294 %
+   9       5.7630 ±    0.1432       0.00870 ±    0.00365       0.04310 ±    0.00315     6.260 ±  0.261 %    92.017 ±  0.282 %
+  10       5.8584 ±    0.1390       0.00912 ±    0.00337       0.04073 ±    0.00284     6.099 ±  0.244 %    92.170 ±  0.266 %
+  11       5.9284 ±    0.1341       0.00871 ±    0.00313       0.03883 ±    0.00258     5.946 ±  0.230 %    92.349 ±  0.251 %
+  12       6.1692 ±    0.1351       0.00780 ±    0.00295       0.03749 ±    0.00237     5.823 ±  0.218 %    92.416 ±  0.239 %
+  13       6.2601 ±    0.1320       0.00732 ±    0.00280       0.03678 ±    0.00220     5.730 ±  0.206 %    92.473 ±  0.229 %
+  14       6.3277 ±    0.1285       0.00765 ±    0.00265       0.03547 ±    0.00204     5.597 ±  0.196 %    92.487 ±  0.220 %
+  15       6.3813 ±    0.1255       0.00813 ±    0.00253       0.03437 ±    0.00191     5.483 ±  0.187 %    92.506 ±  0.213 %
+  16       6.6362 ±    0.1281       0.00112 ±    0.00262       0.03556 ±    0.00182     5.428 ±  0.178 %    92.326 ±  0.208 %
+  17       6.6944 ±    0.1248       0.00136 ±    0.00250       0.03461 ±    0.00172     5.336 ±  0.171 %    92.416 ±  0.201 %
+  18       6.8136 ±    0.1238       0.00299 ±    0.00240       0.03397 ±    0.00162     5.250 ±  0.164 %    92.392 ±  0.195 %
+  19       6.7649 ±    0.1202       0.00430 ±    0.00233       0.03357 ±    0.00155     5.255 ±  0.163 %    92.447 ±  0.190 %
+  20       6.5880 ±    0.1136       0.00464 ±    0.00227       0.03436 ±    0.00149     5.330 ±  0.154 %    92.498 ±  0.184 %
+  21       6.5904 ±    0.1105       0.00398 ±    0.00220       0.03400 ±    0.00142     5.307 ±  0.149 %    92.422 ±  0.181 %
+  22       6.6171 ±    0.1085       0.00471 ±    0.00212       0.03349 ±    0.00136     5.290 ±  0.145 %    92.380 ±  0.177 %
+  23       6.6614 ±    0.1069       0.00477 ±    0.00206       0.03296 ±    0.00130     5.238 ±  0.141 %    92.460 ±  0.172 %
+  24       6.6793 ±    0.1048       0.00544 ±    0.00200       0.03255 ±    0.00125     5.196 ±  0.136 %    92.518 ±  0.168 %
+  25       6.7290 ±    0.1036       0.00565 ±    0.00195       0.03232 ±    0.00120     5.153 ±  0.133 %    92.598 ±  0.164 %
+  26       6.7078 ±    0.1013       0.00606 ±    0.00191       0.03197 ±    0.00116     5.123 ±  0.129 %    92.631 ±  0.160 %
+  27       6.8471 ±    0.1019       0.00642 ±    0.00186       0.03144 ±    0.00112     5.073 ±  0.126 %    92.676 ±  0.157 %
+  28       6.9293 ±    0.1017       0.00653 ±    0.00181       0.03094 ±    0.00108     5.017 ±  0.123 %    92.752 ±  0.153 %
+  29       6.9314 ±    0.0999       0.00621 ±    0.00176       0.03053 ±    0.00104     4.977 ±  0.120 %    92.776 ±  0.150 %
+  30       6.8970 ±    0.0976       0.00601 ±    0.00173       0.03036 ±    0.00101     4.953 ±  0.117 %    92.783 ±  0.148 %
+  31       6.8246 ±    0.0949       0.00658 ±    0.00170       0.03025 ±    0.00098     4.938 ±  0.114 %    92.836 ±  0.145 %
+  32       6.7270 ±    0.0920       0.00713 ±    0.00167       0.03052 ±    0.00095     4.997 ±  0.111 %    92.818 ±  0.143 %
+  33       6.7007 ±    0.0903       0.00741 ±    0.00164       0.03041 ±    0.00093     4.981 ±  0.109 %    92.840 ±  0.140 %
+  34       6.6952 ±    0.0888       0.00731 ±    0.00162       0.03017 ±    0.00091     4.950 ±  0.106 %    92.853 ±  0.138 %
+  35       6.7172 ±    0.0879       0.00845 ±    0.00159       0.03011 ±    0.00089     4.958 ±  0.106 %    92.850 ±  0.136 %
+  36       6.7247 ±    0.0868       0.00902 ±    0.00157       0.02991 ±    0.00087     4.933 ±  0.104 %    92.897 ±  0.134 %
+  37       6.6136 ±    0.0838       0.00975 ±    0.00154       0.02972 ±    0.00085     4.924 ±  0.102 %    92.922 ±  0.132 %
+  38       6.5471 ±    0.0816       0.00950 ±    0.00151       0.02950 ±    0.00082     4.916 ±  0.100 %    92.949 ±  0.130 %
+  39       6.4781 ±    0.0795       0.00956 ±    0.00149       0.02956 ±    0.00082     4.925 ±  0.098 %    92.959 ±  0.128 %
+  40       6.3888 ±    0.0771       0.00961 ±    0.00146       0.02942 ±    0.00080     4.919 ±  0.096 %    92.972 ±  0.126 %
+
+====== Perplexity statistics ======
+Mean PPL(Q)                   :   6.388763 ±   0.077065
+Mean PPL(base)                :   6.327630 ±   0.076192
+Cor(ln(PPL(Q)), ln(PPL(base))):  99.26%
+Mean ln(PPL(Q)/PPL(base))     :   0.009615 ±   0.001463
+Mean PPL(Q)/PPL(base)         :   1.009661 ±   0.001477
+Mean PPL(Q)-PPL(base)         :   0.061133 ±   0.009341
+
+====== KL divergence statistics ======
+Mean    KLD:   0.029416 ±   0.000800
+Maximum KLD:  22.440645
+99.9%   KLD:   1.526750
+99.0%   KLD:   0.303920
+95.0%   KLD:   0.094522
+90.0%   KLD:   0.054514
+Median  KLD:   0.010466
+10.0%   KLD:   0.000195
+ 5.0%   KLD:   0.000048
+ 1.0%   KLD:   0.000005
+ 0.1%   KLD:  -0.000000
+Minimum KLD:  -0.000016
+
+====== Token probability statistics ======
+Mean    Δp: -0.193 ± 0.024 %
+Maximum Δp: 94.884%
+99.9%   Δp: 34.781%
+99.0%   Δp: 12.957%
+95.0%   Δp:  5.345%
+90.0%   Δp:  3.023%
+75.0%   Δp:  0.574%
+Median  Δp: -0.001%
+25.0%   Δp: -0.770%
+10.0%   Δp: -3.480%
+ 5.0%   Δp: -6.066%
+ 1.0%   Δp: -15.712%
+ 0.1%   Δp: -41.449%
+Minimum Δp: -98.386%
+RMS Δp    :  4.919 ± 0.096 %
+Same top p: 92.972 ± 0.126 %
+

recipe/logs/afternoon_B5_kld_q106.log

@@ -0,0 +1,92 @@
+0.00.068.802 I common_init_result: fitting params to device memory ...
+0.00.068.807 I common_init_result: (for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)
+0.00.460.644 W llama_model_loader: direct I/O is enabled, disabling mmap
+0.40.681.506 W llama_context: n_ctx_seq (2048) < n_ctx_train (262144) -- the full capacity of the model will not be utilized
+0.40.751.408 W common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+0.40.912.601 I 
+0.40.912.721 I system_info: n_threads = 16 (n_threads_batch = 16) / 32 | ROCm : NO_VMM = 1 | PEER_MAX_BATCH_SIZE = 128 | FA_ALL_QUANTS = 1 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+0.41.096.613 I kl_divergence: computing over 40 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+0.50.761.361 I kl_divergence: 9.66 seconds per pass - ETA 6.43 minutes
+
+chunk             PPL               ln(PPL(Q)/PPL(base))          KL Divergence              Δp RMS            Same top p
+   1       5.6588 ±    0.4178       0.01009 ±    0.00791       0.02954 ±    0.00195     4.945 ±  0.370 %    92.082 ±  0.845 %
+   2       6.8058 ±    0.3681       0.02711 ±    0.00674       0.03973 ±    0.00203     5.210 ±  0.318 %    91.105 ±  0.630 %
+   3       6.9897 ±    0.3080       0.02465 ±    0.00540       0.03684 ±    0.00156     5.139 ±  0.254 %    90.974 ±  0.517 %
+   4       7.1929 ±    0.2841       0.03906 ±    0.00507       0.04007 ±    0.00162     5.587 ±  0.287 %    91.031 ±  0.447 %
+   5       7.1476 ±    0.2551       0.04443 ±    0.00479       0.04199 ±    0.00162     5.659 ±  0.245 %    91.065 ±  0.399 %
+   6       6.1062 ±    0.1927       0.03852 ±    0.00445       0.04307 ±    0.00160     6.178 ±  0.247 %    91.268 ±  0.360 %
+   7       5.5325 ±    0.1575       0.02692 ±    0.00433       0.05190 ±    0.00368     6.703 ±  0.271 %    91.370 ±  0.332 %
+   8       5.5219 ±    0.1466       0.02203 ±    0.00408       0.05166 ±    0.00325     6.658 ±  0.249 %    91.312 ±  0.311 %
+   9       5.7984 ±    0.1455       0.01482 ±    0.00398       0.05271 ±    0.00294     6.601 ±  0.228 %    90.953 ±  0.299 %
+  10       5.9036 ±    0.1416       0.01680 ±    0.00372       0.05035 ±    0.00265     6.469 ±  0.214 %    91.095 ±  0.282 %
+  11       5.9859 ±    0.1369       0.01836 ±    0.00349       0.04879 ±    0.00242     6.392 ±  0.201 %    91.193 ±  0.267 %
+  12       6.2320 ±    0.1380       0.01792 ±    0.00327       0.04716 ±    0.00223     6.233 ±  0.190 %    91.316 ±  0.254 %
+  13       6.3269 ±    0.1349       0.01792 ±    0.00309       0.04599 ±    0.00206     6.159 ±  0.182 %    91.488 ±  0.242 %
+  14       6.3869 ±    0.1311       0.01695 ±    0.00294       0.04481 ±    0.00192     6.076 ±  0.174 %    91.461 ±  0.234 %
+  15       6.4396 ±    0.1280       0.01721 ±    0.00282       0.04382 ±    0.00180     5.968 ±  0.166 %    91.548 ±  0.225 %
+  16       6.7096 ±    0.1310       0.01211 ±    0.00285       0.04471 ±    0.00175     5.907 ±  0.157 %    91.422 ±  0.219 %
+  17       6.7673 ±    0.1276       0.01219 ±    0.00272       0.04370 ±    0.00165     5.818 ±  0.151 %    91.444 ±  0.212 %
+  18       6.8777 ±    0.1263       0.01236 ±    0.00262       0.04303 ±    0.00156     5.744 ±  0.145 %    91.468 ±  0.206 %
+  19       6.8299 ±    0.1226       0.01386 ±    0.00253       0.04239 ±    0.00149     5.690 ±  0.141 %    91.506 ±  0.200 %
+  20       6.6595 ±    0.1161       0.01543 ±    0.00250       0.04380 ±    0.00145     5.884 ±  0.142 %    91.520 ±  0.195 %
+  21       6.6615 ±    0.1129       0.01471 ±    0.00242       0.04338 ±    0.00138     5.858 ±  0.138 %    91.528 ±  0.190 %
+  22       6.6882 ±    0.1109       0.01540 ±    0.00234       0.04285 ±    0.00132     5.816 ±  0.135 %    91.571 ±  0.185 %
+  23       6.7337 ±    0.1092       0.01556 ±    0.00227       0.04238 ±    0.00127     5.793 ±  0.130 %    91.619 ±  0.181 %
+  24       6.7493 ±    0.1070       0.01586 ±    0.00220       0.04168 ±    0.00122     5.729 ±  0.126 %    91.691 ±  0.176 %
+  25       6.7976 ±    0.1058       0.01579 ±    0.00214       0.04128 ±    0.00117     5.684 ±  0.123 %    91.715 ±  0.172 %
+  26       6.7754 ±    0.1033       0.01608 ±    0.00210       0.04106 ±    0.00113     5.701 ±  0.121 %    91.781 ±  0.168 %
+  27       6.9146 ±    0.1039       0.01622 ±    0.00204       0.04051 ±    0.00109     5.639 ±  0.118 %    91.829 ±  0.165 %
+  28       6.9967 ±    0.1036       0.01621 ±    0.00199       0.04004 ±    0.00105     5.591 ±  0.115 %    91.866 ±  0.162 %
+  29       7.0009 ±    0.1018       0.01618 ±    0.00194       0.03963 ±    0.00102     5.550 ±  0.112 %    91.900 ±  0.158 %
+  30       6.9660 ±    0.0996       0.01596 ±    0.00191       0.03951 ±    0.00099     5.523 ±  0.110 %    91.877 ±  0.156 %
+  31       6.8931 ±    0.0968       0.01656 ±    0.00188       0.03950 ±    0.00096     5.505 ±  0.107 %    91.918 ±  0.153 %
+  32       6.7966 ±    0.0939       0.01742 ±    0.00186       0.03990 ±    0.00094     5.557 ±  0.105 %    91.932 ±  0.151 %
+  33       6.7940 ±    0.0926       0.02123 ±    0.00189       0.04120 ±    0.00098     5.642 ±  0.108 %    91.904 ±  0.148 %
+  34       6.7897 ±    0.0911       0.02132 ±    0.00185       0.04091 ±    0.00095     5.619 ±  0.106 %    91.904 ±  0.146 %
+  35       6.8091 ±    0.0901       0.02204 ±    0.00183       0.04100 ±    0.00094     5.636 ±  0.106 %    91.895 ±  0.144 %
+  36       6.8180 ±    0.0891       0.02280 ±    0.00180       0.04083 ±    0.00092     5.608 ±  0.104 %    91.933 ±  0.142 %
+  37       6.7025 ±    0.0859       0.02311 ±    0.00177       0.04048 ±    0.00090     5.582 ±  0.102 %    91.942 ±  0.140 %
+  38       6.6343 ±    0.0837       0.02274 ±    0.00174       0.04026 ±    0.00088     5.587 ±  0.100 %    91.974 ±  0.138 %
+  39       6.5615 ±    0.0814       0.02235 ±    0.00172       0.04083 ±    0.00094     5.668 ±  0.101 %    91.937 ±  0.136 %
+  40       6.4705 ±    0.0789       0.02233 ±    0.00169       0.04064 ±    0.00092     5.666 ±  0.099 %    91.960 ±  0.134 %
+
+====== Perplexity statistics ======
+Mean PPL(Q)                   :   6.470544 ±   0.078949
+Mean PPL(base)                :   6.327630 ±   0.076192
+Cor(ln(PPL(Q)), ln(PPL(base))):  99.04%
+Mean ln(PPL(Q)/PPL(base))     :   0.022334 ±   0.001688
+Mean PPL(Q)/PPL(base)         :   1.022586 ±   0.001726
+Mean PPL(Q)-PPL(base)         :   0.142914 ±   0.011102
+
+====== KL divergence statistics ======
+Mean    KLD:   0.040638 ±   0.000923
+Maximum KLD:  21.348299
+99.9%   KLD:   1.687412
+99.0%   KLD:   0.418962
+95.0%   KLD:   0.132057
+90.0%   KLD:   0.079330
+Median  KLD:   0.015596
+10.0%   KLD:   0.000272
+ 5.0%   KLD:   0.000068
+ 1.0%   KLD:   0.000007
+ 0.1%   KLD:   0.000000
+Minimum KLD:  -0.000064
+
+====== Token probability statistics ======
+Mean    Δp: -0.107 ± 0.028 %
+Maximum Δp: 95.028%
+99.9%   Δp: 38.430%
+99.0%   Δp: 14.846%
+95.0%   Δp:  6.591%
+90.0%   Δp:  3.874%
+75.0%   Δp:  0.809%
+Median  Δp: -0.001%
+25.0%   Δp: -0.816%
+10.0%   Δp: -3.971%
+ 5.0%   Δp: -7.147%
+ 1.0%   Δp: -17.518%
+ 0.1%   Δp: -46.790%
+Minimum Δp: -99.990%
+RMS Δp    :  5.666 ± 0.099 %
+Same top p: 91.960 ± 0.134 %
+

recipe/logs/afternoon_B5_kld_q106i.log

@@ -0,0 +1,92 @@
+0.00.035.853 I common_init_result: fitting params to device memory ...
+0.00.035.856 I common_init_result: (for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)
+0.00.411.069 W llama_model_loader: direct I/O is enabled, disabling mmap
+0.19.744.305 W llama_context: n_ctx_seq (2048) < n_ctx_train (262144) -- the full capacity of the model will not be utilized
+0.19.803.991 W common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+0.19.941.558 I 
+0.19.941.694 I system_info: n_threads = 16 (n_threads_batch = 16) / 32 | ROCm : NO_VMM = 1 | PEER_MAX_BATCH_SIZE = 128 | FA_ALL_QUANTS = 1 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+0.20.055.269 I kl_divergence: computing over 40 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+0.27.119.826 I kl_divergence: 7.06 seconds per pass - ETA 4.70 minutes
+
+chunk             PPL               ln(PPL(Q)/PPL(base))          KL Divergence              Δp RMS            Same top p
+   1       5.6231 ±    0.4111       0.00377 ±    0.00830       0.02658 ±    0.00227     4.715 ±  0.418 %    92.375 ±  0.830 %
+   2       6.7509 ±    0.3616       0.01901 ±    0.00656       0.03276 ±    0.00181     4.871 ±  0.315 %    91.838 ±  0.605 %
+   3       6.9468 ±    0.3033       0.01848 ±    0.00533       0.03242 ±    0.00172     5.333 ±  0.336 %    92.115 ±  0.487 %
+   4       7.1727 ±    0.2827       0.03625 ±    0.00528       0.03590 ±    0.00185     5.720 ±  0.348 %    92.180 ±  0.420 %
+   5       7.0702 ±    0.2506       0.03355 ±    0.00463       0.03664 ±    0.00164     5.657 ±  0.297 %    91.867 ±  0.382 %
+   6       6.0711 ±    0.1904       0.03275 ±    0.00425       0.03651 ±    0.00143     5.833 ±  0.256 %    92.033 ±  0.346 %
+   7       5.5141 ±    0.1562       0.02358 ±    0.00414       0.04672 ±    0.00445     6.523 ±  0.289 %    92.236 ±  0.316 %
+   8       5.5033 ±    0.1454       0.01866 ±    0.00394       0.04668 ±    0.00393     6.450 ±  0.263 %    92.107 ±  0.298 %
+   9       5.8014 ±    0.1451       0.01534 ±    0.00376       0.04651 ±    0.00351     6.383 ±  0.242 %    91.735 ±  0.287 %
+  10       5.8970 ±    0.1409       0.01569 ±    0.00347       0.04385 ±    0.00316     6.164 ±  0.226 %    91.926 ±  0.269 %
+  11       5.9673 ±    0.1359       0.01524 ±    0.00322       0.04200 ±    0.00287     6.033 ±  0.211 %    92.091 ±  0.254 %
+  12       6.2128 ±    0.1369       0.01485 ±    0.00303       0.04047 ±    0.00264     5.915 ±  0.200 %    92.074 ±  0.244 %
+  13       6.2968 ±    0.1334       0.01315 ±    0.00291       0.03979 ±    0.00244     5.841 ±  0.188 %    92.067 ±  0.234 %
+  14       6.3603 ±    0.1298       0.01278 ±    0.00275       0.03850 ±    0.00227     5.722 ±  0.179 %    91.977 ±  0.227 %
+  15       6.4122 ±    0.1266       0.01295 ±    0.00263       0.03740 ±    0.00212     5.609 ±  0.171 %    92.004 ±  0.219 %
+  16       6.6471 ±    0.1285       0.00275 ±    0.00288       0.04017 ±    0.00207     5.647 ±  0.163 %    91.667 ±  0.216 %
+  17       6.7070 ±    0.1253       0.00325 ±    0.00275       0.03909 ±    0.00195     5.540 ±  0.156 %    91.772 ±  0.208 %
+  18       6.8201 ±    0.1241       0.00394 ±    0.00263       0.03834 ±    0.00185     5.472 ±  0.150 %    91.745 ±  0.203 %
+  19       6.7684 ±    0.1203       0.00482 ±    0.00252       0.03751 ±    0.00175     5.410 ±  0.146 %    91.799 ±  0.197 %
+  20       6.5981 ±    0.1139       0.00617 ±    0.00247       0.03867 ±    0.00169     5.527 ±  0.143 %    91.799 ±  0.192 %
+  21       6.5963 ±    0.1107       0.00488 ±    0.00240       0.03871 ±    0.00164     5.524 ±  0.139 %    91.794 ±  0.187 %
+  22       6.6228 ±    0.1087       0.00557 ±    0.00232       0.03806 ±    0.00157     5.494 ±  0.136 %    91.829 ±  0.183 %
+  23       6.6725 ±    0.1071       0.00643 ±    0.00224       0.03755 ±    0.00150     5.461 ±  0.132 %    91.899 ±  0.178 %
+  24       6.6888 ±    0.1050       0.00686 ±    0.00217       0.03696 ±    0.00144     5.405 ±  0.128 %    91.956 ±  0.174 %
+  25       6.7371 ±    0.1038       0.00686 ±    0.00211       0.03658 ±    0.00139     5.354 ±  0.124 %    91.996 ±  0.170 %
+  26       6.7173 ±    0.1014       0.00748 ±    0.00207       0.03631 ±    0.00134     5.350 ±  0.122 %    92.037 ±  0.166 %
+  27       6.8569 ±    0.1021       0.00784 ±    0.00201       0.03572 ±    0.00129     5.287 ±  0.119 %    92.104 ±  0.162 %
+  28       6.9388 ±    0.1018       0.00789 ±    0.00196       0.03524 ±    0.00125     5.248 ±  0.117 %    92.166 ±  0.159 %
+  29       6.9431 ±    0.1001       0.00790 ±    0.00190       0.03477 ±    0.00120     5.205 ±  0.114 %    92.207 ±  0.156 %
+  30       6.9087 ±    0.0978       0.00770 ±    0.00186       0.03457 ±    0.00117     5.176 ±  0.111 %    92.235 ±  0.153 %
+  31       6.8352 ±    0.0951       0.00813 ±    0.00183       0.03447 ±    0.00113     5.157 ±  0.108 %    92.256 ±  0.150 %
+  32       6.7335 ±    0.0921       0.00810 ±    0.00180       0.03484 ±    0.00110     5.192 ±  0.106 %    92.265 ±  0.148 %
+  33       6.7353 ±    0.0910       0.01256 ±    0.00186       0.03698 ±    0.00117     5.369 ±  0.117 %    92.162 ±  0.146 %
+  34       6.7312 ±    0.0895       0.01267 ±    0.00182       0.03665 ±    0.00113     5.339 ±  0.115 %    92.180 ±  0.144 %
+  35       6.7510 ±    0.0886       0.01347 ±    0.00179       0.03659 ±    0.00111     5.334 ±  0.114 %    92.166 ±  0.142 %
+  36       6.7586 ±    0.0875       0.01404 ±    0.00176       0.03634 ±    0.00108     5.313 ±  0.112 %    92.218 ±  0.140 %
+  37       6.6444 ±    0.0844       0.01440 ±    0.00172       0.03604 ±    0.00106     5.296 ±  0.110 %    92.256 ±  0.137 %
+  38       6.5786 ±    0.0822       0.01431 ±    0.00169       0.03576 ±    0.00103     5.290 ±  0.108 %    92.290 ±  0.135 %
+  39       6.5045 ±    0.0800       0.01363 ±    0.00167       0.03602 ±    0.00102     5.336 ±  0.107 %    92.265 ±  0.134 %
+  40       6.4131 ±    0.0775       0.01342 ±    0.00164       0.03579 ±    0.00100     5.328 ±  0.105 %    92.309 ±  0.132 %
+
+====== Perplexity statistics ======
+Mean PPL(Q)                   :   6.413088 ±   0.077536
+Mean PPL(base)                :   6.327630 ±   0.076192
+Cor(ln(PPL(Q)), ln(PPL(base))):  99.08%
+Mean ln(PPL(Q)/PPL(base))     :   0.013415 ±   0.001642
+Mean PPL(Q)/PPL(base)         :   1.013506 ±   0.001664
+Mean PPL(Q)-PPL(base)         :   0.085458 ±   0.010539
+
+====== KL divergence statistics ======
+Mean    KLD:   0.035791 ±   0.001001
+Maximum KLD:  27.125303
+99.9%   KLD:   1.792250
+99.0%   KLD:   0.381905
+95.0%   KLD:   0.113879
+90.0%   KLD:   0.064828
+Median  KLD:   0.012275
+10.0%   KLD:   0.000234
+ 5.0%   KLD:   0.000053
+ 1.0%   KLD:   0.000006
+ 0.1%   KLD:   0.000000
+Minimum KLD:  -0.000019
+
+====== Token probability statistics ======
+Mean    Δp: -0.200 ± 0.026 %
+Maximum Δp: 94.884%
+99.9%   Δp: 39.516%
+99.0%   Δp: 13.954%
+95.0%   Δp:  5.807%
+90.0%   Δp:  3.177%
+75.0%   Δp:  0.596%
+Median  Δp: -0.001%
+25.0%   Δp: -0.845%
+10.0%   Δp: -3.738%
+ 5.0%   Δp: -6.497%
+ 1.0%   Δp: -16.639%
+ 0.1%   Δp: -43.751%
+Minimum Δp: -98.872%
+RMS Δp    :  5.328 ± 0.105 %
+Same top p: 92.309 ± 0.132 %
+

recipe/logs/afternoon_B5_kld_q111.log

@@ -0,0 +1,92 @@
+0.00.045.114 I common_init_result: fitting params to device memory ...
+0.00.045.117 I common_init_result: (for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)
+0.00.478.581 W llama_model_loader: direct I/O is enabled, disabling mmap
+0.37.579.838 W llama_context: n_ctx_seq (2048) < n_ctx_train (262144) -- the full capacity of the model will not be utilized
+0.37.651.344 W common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+0.37.859.083 I 
+0.37.859.213 I system_info: n_threads = 16 (n_threads_batch = 16) / 32 | ROCm : NO_VMM = 1 | PEER_MAX_BATCH_SIZE = 128 | FA_ALL_QUANTS = 1 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+0.37.976.758 I kl_divergence: computing over 40 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+0.46.116.757 I kl_divergence: 8.14 seconds per pass - ETA 5.42 minutes
+
+chunk             PPL               ln(PPL(Q)/PPL(base))          KL Divergence              Δp RMS            Same top p
+   1       5.6059 ±    0.4093       0.00069 ±    0.00337       0.00478 ±    0.00036     1.948 ±  0.123 %    97.458 ±  0.492 %
+   2       6.6305 ±    0.3529       0.00102 ±    0.00266       0.00604 ±    0.00039     2.228 ±  0.174 %    96.237 ±  0.421 %
+   3       6.8224 ±    0.2960       0.00043 ±    0.00210       0.00596 ±    0.00033     2.358 ±  0.175 %    96.383 ±  0.337 %
+   4       6.8978 ±    0.2659      -0.00284 ±    0.00191       0.00624 ±    0.00037     2.408 ±  0.172 %    96.676 ±  0.280 %
+   5       6.8264 ±    0.2374      -0.00155 ±    0.00174       0.00657 ±    0.00037     2.448 ±  0.164 %    96.461 ±  0.258 %
+   6       5.8711 ±    0.1809      -0.00075 ±    0.00160       0.00680 ±    0.00033     2.558 ±  0.141 %    96.351 ±  0.239 %
+   7       5.3771 ±    0.1501      -0.00159 ±    0.00200       0.01596 ±    0.00375     3.643 ±  0.351 %    96.271 ±  0.224 %
+   8       5.3744 ±    0.1400      -0.00504 ±    0.00203       0.01666 ±    0.00329     3.739 ±  0.314 %    95.907 ±  0.219 %
+   9       5.6841 ±    0.1409      -0.00509 ±    0.00194       0.01604 ±    0.00293     3.659 ±  0.289 %    95.688 ±  0.212 %
+  10       5.7797 ±    0.1368      -0.00441 ±    0.00181       0.01510 ±    0.00264     3.537 ±  0.269 %    95.797 ±  0.198 %
+  11       5.8547 ±    0.1322      -0.00381 ±    0.00168       0.01425 ±    0.00240     3.447 ±  0.251 %    95.868 ±  0.188 %
+  12       6.1082 ±    0.1337      -0.00214 ±    0.00160       0.01394 ±    0.00221     3.354 ±  0.237 %    95.837 ±  0.180 %
+  13       6.1990 ±    0.1306      -0.00250 ±    0.00153       0.01344 ±    0.00204     3.309 ±  0.223 %    95.879 ±  0.172 %
+  14       6.2654 ±    0.1271      -0.00225 ±    0.00146       0.01297 ±    0.00189     3.252 ±  0.211 %    95.825 ±  0.167 %
+  15       6.3162 ±    0.1240      -0.00214 ±    0.00142       0.01258 ±    0.00177     3.221 ±  0.202 %    95.855 ±  0.161 %
+  16       6.6139 ±    0.1282      -0.00225 ±    0.00153       0.01319 ±    0.00170     3.247 ±  0.196 %    95.778 ±  0.157 %
+  17       6.6699 ±    0.1249      -0.00231 ±    0.00147       0.01282 ±    0.00160     3.192 ±  0.188 %    95.825 ±  0.152 %
+  18       6.7785 ±    0.1236      -0.00218 ±    0.00141       0.01246 ±    0.00152     3.146 ±  0.180 %    95.802 ±  0.148 %
+  19       6.7237 ±    0.1198      -0.00180 ±    0.00135       0.01206 ±    0.00144     3.092 ±  0.174 %    95.802 ±  0.144 %
+  20       6.5488 ±    0.1132      -0.00133 ±    0.00133       0.01227 ±    0.00138     3.215 ±  0.174 %    95.777 ±  0.141 %
+  21       6.5550 ±    0.1102      -0.00140 ±    0.00128       0.01224 ±    0.00133     3.216 ±  0.168 %    95.811 ±  0.137 %
+  22       6.5767 ±    0.1081      -0.00141 ±    0.00124       0.01195 ±    0.00127     3.176 ±  0.163 %    95.819 ±  0.133 %
+  23       6.6160 ±    0.1063      -0.00208 ±    0.00121       0.01186 ±    0.00122     3.184 ±  0.158 %    95.852 ±  0.130 %
+  24       6.6269 ±    0.1041      -0.00243 ±    0.00117       0.01174 ±    0.00117     3.157 ±  0.153 %    95.866 ±  0.127 %
+  25       6.6721 ±    0.1028      -0.00285 ±    0.00114       0.01158 ±    0.00112     3.132 ±  0.149 %    95.883 ±  0.124 %
+  26       6.6480 ±    0.1004      -0.00290 ±    0.00110       0.01134 ±    0.00108     3.107 ±  0.144 %    95.917 ±  0.121 %
+  27       6.7853 ±    0.1010      -0.00264 ±    0.00107       0.01109 ±    0.00104     3.066 ±  0.141 %    95.952 ±  0.119 %
+  28       6.8669 ±    0.1008      -0.00252 ±    0.00106       0.01125 ±    0.00102     3.030 ±  0.138 %    95.968 ±  0.116 %
+  29       6.8723 ±    0.0991      -0.00234 ±    0.00103       0.01105 ±    0.00099     3.000 ±  0.134 %    95.989 ±  0.114 %
+  30       6.8393 ±    0.0969      -0.00240 ±    0.00101       0.01090 ±    0.00096     2.975 ±  0.131 %    96.008 ±  0.112 %
+  31       6.7637 ±    0.0941      -0.00239 ±    0.00098       0.01077 ±    0.00093     2.954 ±  0.128 %    96.036 ±  0.110 %
+  32       6.6655 ±    0.0912      -0.00205 ±    0.00096       0.01069 ±    0.00090     2.967 ±  0.123 %    96.041 ±  0.108 %
+  33       6.6325 ±    0.0893      -0.00282 ±    0.00097       0.01074 ±    0.00087     2.954 ±  0.120 %    96.022 ±  0.106 %
+  34       6.6281 ±    0.0879      -0.00276 ±    0.00094       0.01060 ±    0.00084     2.933 ±  0.118 %    96.001 ±  0.105 %
+  35       6.6458 ±    0.0869      -0.00224 ±    0.00095       0.01069 ±    0.00084     2.967 ±  0.122 %    95.978 ±  0.104 %
+  36       6.6507 ±    0.0857      -0.00205 ±    0.00093       0.01058 ±    0.00082     2.951 ±  0.119 %    96.003 ±  0.102 %
+  37       6.5368 ±    0.0827      -0.00192 ±    0.00091       0.01044 ±    0.00080     2.933 ±  0.117 %    96.011 ±  0.101 %
+  38       6.4710 ±    0.0805      -0.00218 ±    0.00089       0.01031 ±    0.00078     2.914 ±  0.115 %    96.028 ±  0.099 %
+  39       6.4058 ±    0.0785      -0.00166 ±    0.00089       0.01054 ±    0.00077     2.985 ±  0.115 %    96.030 ±  0.098 %
+  40       6.3176 ±    0.0761      -0.00158 ±    0.00087       0.01048 ±    0.00075     2.987 ±  0.113 %    96.026 ±  0.097 %
+
+====== Perplexity statistics ======
+Mean PPL(Q)                   :   6.317616 ±   0.076081
+Mean PPL(base)                :   6.327630 ±   0.076192
+Cor(ln(PPL(Q)), ln(PPL(base))):  99.74%
+Mean ln(PPL(Q)/PPL(base))     :  -0.001584 ±   0.000875
+Mean PPL(Q)/PPL(base)         :   0.998417 ±   0.000873
+Mean PPL(Q)-PPL(base)         :  -0.010014 ±   0.005531
+
+====== KL divergence statistics ======
+Mean    KLD:   0.010477 ±   0.000749
+Maximum KLD:  22.860224
+99.9%   KLD:   0.681516
+99.0%   KLD:   0.085791
+95.0%   KLD:   0.025180
+90.0%   KLD:   0.015262
+Median  KLD:   0.003005
+10.0%   KLD:   0.000068
+ 5.0%   KLD:   0.000017
+ 1.0%   KLD:   0.000002
+ 0.1%   KLD:  -0.000001
+Minimum KLD:  -0.000043
+
+====== Token probability statistics ======
+Mean    Δp: -0.013 ± 0.015 %
+Maximum Δp: 94.851%
+99.9%   Δp: 19.339%
+99.0%   Δp:  7.373%
+95.0%   Δp:  3.232%
+90.0%   Δp:  1.833%
+75.0%   Δp:  0.372%
+Median  Δp: -0.000%
+25.0%   Δp: -0.393%
+10.0%   Δp: -1.826%
+ 5.0%   Δp: -3.214%
+ 1.0%   Δp: -7.349%
+ 0.1%   Δp: -20.986%
+Minimum Δp: -97.772%
+RMS Δp    :  2.987 ± 0.113 %
+Same top p: 96.026 ± 0.097 %
+

recipe/logs/afternoon_B5_kld_q115.log

@@ -0,0 +1,92 @@
+0.00.065.550 I common_init_result: fitting params to device memory ...
+0.00.065.557 I common_init_result: (for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)
+0.00.633.205 W llama_model_loader: direct I/O is enabled, disabling mmap
+0.37.969.769 W llama_context: n_ctx_seq (2048) < n_ctx_train (262144) -- the full capacity of the model will not be utilized
+0.38.030.027 W common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+0.38.234.320 I 
+0.38.234.434 I system_info: n_threads = 16 (n_threads_batch = 16) / 32 | ROCm : NO_VMM = 1 | PEER_MAX_BATCH_SIZE = 128 | FA_ALL_QUANTS = 1 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+0.38.357.351 I kl_divergence: computing over 40 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+0.46.087.366 I kl_divergence: 7.73 seconds per pass - ETA 5.15 minutes
+
+chunk             PPL               ln(PPL(Q)/PPL(base))          KL Divergence              Δp RMS            Same top p
+   1       5.5566 ±    0.4036      -0.00814 ±    0.00272       0.00315 ±    0.00028     1.550 ±  0.110 %    97.458 ±  0.492 %
+   2       6.6107 ±    0.3509      -0.00197 ±    0.00206       0.00353 ±    0.00030     1.796 ±  0.228 %    96.774 ±  0.391 %
+   3       6.8101 ±    0.2950      -0.00138 ±    0.00159       0.00313 ±    0.00020     1.651 ±  0.168 %    96.839 ±  0.316 %
+   4       6.8838 ±    0.2650      -0.00487 ±    0.00217       0.00685 ±    0.00217     2.692 ±  0.513 %    96.970 ±  0.268 %
+   5       6.8057 ±    0.2361      -0.00458 ±    0.00192       0.00658 ±    0.00175     2.563 ±  0.432 %    96.911 ±  0.242 %
+   6       5.8506 ±    0.1799      -0.00425 ±    0.00170       0.00637 ±    0.00146     2.524 ±  0.368 %    96.937 ±  0.220 %
+   7       5.3534 ±    0.1491      -0.00600 ±    0.00186       0.01421 ±    0.00369     3.438 ±  0.382 %    96.900 ±  0.205 %
+   8       5.3640 ±    0.1397      -0.00698 ±    0.00182       0.01412 ±    0.00324     3.459 ±  0.342 %    96.640 ±  0.199 %
+   9       5.6737 ±    0.1405      -0.00693 ±    0.00174       0.01339 ±    0.00288     3.393 ±  0.316 %    96.448 ±  0.193 %
+  10       5.7712 ±    0.1365      -0.00589 ±    0.00162       0.01246 ±    0.00260     3.254 ±  0.296 %    96.559 ±  0.180 %
+  11       5.8449 ±    0.1319      -0.00548 ±    0.00150       0.01157 ±    0.00236     3.134 ±  0.280 %    96.623 ±  0.170 %
+  12       6.0885 ±    0.1330      -0.00537 ±    0.00139       0.01098 ±    0.00217     3.031 ±  0.265 %    96.611 ±  0.163 %
+  13       6.1823 ±    0.1300      -0.00520 ±    0.00132       0.01042 ±    0.00200     2.962 ±  0.251 %    96.699 ±  0.155 %
+  14       6.2493 ±    0.1266      -0.00482 ±    0.00125       0.00997 ±    0.00186     2.891 ±  0.239 %    96.649 ±  0.150 %
+  15       6.3012 ±    0.1236      -0.00451 ±    0.00121       0.00954 ±    0.00173     2.820 ±  0.229 %    96.631 ±  0.146 %
+  16       6.5894 ±    0.1274      -0.00597 ±    0.00134       0.00989 ±    0.00166     2.783 ±  0.218 %    96.585 ±  0.142 %
+  17       6.6478 ±    0.1242      -0.00563 ±    0.00128       0.00955 ±    0.00156     2.727 ±  0.209 %    96.613 ±  0.137 %
+  18       6.7575 ±    0.1230      -0.00527 ±    0.00123       0.00922 ±    0.00148     2.676 ±  0.201 %    96.600 ±  0.134 %
+  19       6.6994 ±    0.1190      -0.00542 ±    0.00118       0.00904 ±    0.00140     2.638 ±  0.194 %    96.584 ±  0.130 %
+  20       6.5206 ±    0.1124      -0.00566 ±    0.00116       0.00915 ±    0.00134     2.736 ±  0.189 %    96.564 ±  0.127 %
+  21       6.5276 ±    0.1095      -0.00558 ±    0.00111       0.00912 ±    0.00129     2.746 ±  0.184 %    96.597 ±  0.124 %
+  22       6.5501 ±    0.1074      -0.00547 ±    0.00107       0.00888 ±    0.00123     2.706 ±  0.178 %    96.601 ±  0.121 %
+  23       6.5926 ±    0.1057      -0.00561 ±    0.00104       0.00875 ±    0.00118     2.705 ±  0.174 %    96.617 ±  0.118 %
+  24       6.6052 ±    0.1035      -0.00572 ±    0.00100       0.00860 ±    0.00113     2.684 ±  0.168 %    96.636 ±  0.115 %
+  25       6.6526 ±    0.1023      -0.00578 ±    0.00097       0.00846 ±    0.00108     2.662 ±  0.163 %    96.657 ±  0.112 %
+  26       6.6305 ±    0.1000      -0.00553 ±    0.00094       0.00823 ±    0.00104     2.624 ±  0.159 %    96.684 ±  0.110 %
+  27       6.7675 ±    0.1006      -0.00528 ±    0.00091       0.00801 ±    0.00100     2.584 ±  0.156 %    96.716 ±  0.107 %
+  28       6.8491 ±    0.1004      -0.00511 ±    0.00091       0.00823 ±    0.00100     2.557 ±  0.152 %    96.718 ±  0.105 %
+  29       6.8537 ±    0.0986      -0.00505 ±    0.00088       0.00806 ±    0.00096     2.529 ±  0.148 %    96.707 ±  0.104 %
+  30       6.8219 ±    0.0964      -0.00495 ±    0.00086       0.00789 ±    0.00093     2.498 ±  0.145 %    96.722 ±  0.102 %
+  31       6.7475 ±    0.0937      -0.00479 ±    0.00084       0.00775 ±    0.00090     2.472 ±  0.142 %    96.755 ±  0.099 %
+  32       6.6475 ±    0.0908      -0.00475 ±    0.00082       0.00764 ±    0.00087     2.471 ±  0.139 %    96.762 ±  0.098 %
+  33       6.6137 ±    0.0889      -0.00567 ±    0.00083       0.00772 ±    0.00085     2.467 ±  0.135 %    96.750 ±  0.097 %
+  34       6.6095 ±    0.0875      -0.00558 ±    0.00081       0.00759 ±    0.00082     2.445 ±  0.132 %    96.754 ±  0.095 %
+  35       6.6264 ±    0.0865      -0.00516 ±    0.00081       0.00766 ±    0.00082     2.480 ±  0.138 %    96.741 ±  0.094 %
+  36       6.6315 ±    0.0853      -0.00494 ±    0.00079       0.00755 ±    0.00079     2.459 ±  0.135 %    96.752 ±  0.092 %
+  37       6.5180 ±    0.0823      -0.00481 ±    0.00077       0.00742 ±    0.00077     2.439 ±  0.132 %    96.748 ±  0.091 %
+  38       6.4529 ±    0.0802      -0.00499 ±    0.00076       0.00735 ±    0.00075     2.427 ±  0.130 %    96.743 ±  0.090 %
+  39       6.3883 ±    0.0781      -0.00439 ±    0.00080       0.00769 ±    0.00084     2.416 ±  0.127 %    96.744 ±  0.089 %
+  40       6.3014 ±    0.0758      -0.00415 ±    0.00078       0.00767 ±    0.00082     2.412 ±  0.124 %    96.733 ±  0.088 %
+
+====== Perplexity statistics ======
+Mean PPL(Q)                   :   6.301422 ±   0.075776
+Mean PPL(base)                :   6.327630 ±   0.076192
+Cor(ln(PPL(Q)), ln(PPL(base))):  99.79%
+Mean ln(PPL(Q)/PPL(base))     :  -0.004150 ±   0.000783
+Mean PPL(Q)/PPL(base)         :   0.995858 ±   0.000779
+Mean PPL(Q)-PPL(base)         :  -0.026208 ±   0.004959
+
+====== KL divergence statistics ======
+Mean    KLD:   0.007674 ±   0.000819
+Maximum KLD:  21.973774
+99.9%   KLD:   0.502867
+99.0%   KLD:   0.055398
+95.0%   KLD:   0.015163
+90.0%   KLD:   0.008620
+Median  KLD:   0.001653
+10.0%   KLD:   0.000034
+ 5.0%   KLD:   0.000008
+ 1.0%   KLD:   0.000001
+ 0.1%   KLD:  -0.000002
+Minimum KLD:  -0.000042
+
+====== Token probability statistics ======
+Mean    Δp:  0.043 ± 0.012 %
+Maximum Δp: 94.882%
+99.9%   Δp: 17.864%
+99.0%   Δp:  5.560%
+95.0%   Δp:  2.372%
+90.0%   Δp:  1.393%
+75.0%   Δp:  0.304%
+Median  Δp:  0.000%
+25.0%   Δp: -0.255%
+10.0%   Δp: -1.293%
+ 5.0%   Δp: -2.245%
+ 1.0%   Δp: -5.244%
+ 0.1%   Δp: -14.511%
+Minimum Δp: -99.366%
+RMS Δp    :  2.412 ± 0.124 %
+Same top p: 96.733 ± 0.088 %
+

recipe/patches/mtp-prompt-cache-fix.patch

@@ -0,0 +1,253 @@
+diff --git a/common/common.cpp b/common/common.cpp
+index 5b92940..1421040 100644
+--- a/common/common.cpp
++++ b/common/common.cpp
+@@ -2068,6 +2068,7 @@ common_prompt_checkpoint::common_prompt_checkpoint(const common_prompt_checkpoin
+     pos_max(other.pos_max),
+     data_tgt(other.data_tgt),
+     data_dft(other.data_dft),
++    data_spec(other.data_spec),
+     storage_tgt(llama_state_seq_storage_clone(other.storage_tgt)),
+     storage_dft(llama_state_seq_storage_clone(other.storage_dft)) {
+ }
+@@ -2083,6 +2084,7 @@ common_prompt_checkpoint & common_prompt_checkpoint::operator=(const common_prom
+ 
+     data_tgt = other.data_tgt;
+     data_dft = other.data_dft;
++    data_spec = other.data_spec;
+ 
+     llama_state_seq_storage_free(storage_tgt);
+     llama_state_seq_storage_free(storage_dft);
+@@ -2098,6 +2100,7 @@ common_prompt_checkpoint::common_prompt_checkpoint(common_prompt_checkpoint && o
+     pos_max(other.pos_max),
+     data_tgt(std::move(other.data_tgt)),
+     data_dft(std::move(other.data_dft)),
++    data_spec(std::move(other.data_spec)),
+     storage_tgt(other.storage_tgt),
+     storage_dft(other.storage_dft) {
+     other.storage_tgt = nullptr;
+@@ -2118,6 +2121,7 @@ common_prompt_checkpoint & common_prompt_checkpoint::operator=(common_prompt_che
+ 
+     data_tgt = std::move(other.data_tgt);
+     data_dft = std::move(other.data_dft);
++    data_spec = std::move(other.data_spec);
+ 
+     storage_tgt = other.storage_tgt;
+     storage_dft = other.storage_dft;
+@@ -2131,6 +2135,7 @@ common_prompt_checkpoint & common_prompt_checkpoint::operator=(common_prompt_che
+ size_t common_prompt_checkpoint::size() const {
+     return data_tgt.size() +
+            data_dft.size() +
++           data_spec.size() +
+            llama_state_seq_storage_size(storage_tgt) +
+            llama_state_seq_storage_size(storage_dft);
+ }
+@@ -2147,6 +2152,7 @@ void common_prompt_checkpoint::clear() {
+ 
+     data_tgt.clear();
+     data_dft.clear();
++    data_spec.clear();
+ 
+     llama_state_seq_storage_free(storage_tgt);
+     llama_state_seq_storage_free(storage_dft);
+diff --git a/common/common.h b/common/common.h
+index 4084470..b461ea7 100644
+--- a/common/common.h
++++ b/common/common.h
+@@ -1075,6 +1075,11 @@ struct common_prompt_checkpoint {
+     std::vector<uint8_t> data_tgt;
+     std::vector<uint8_t> data_dft;
+ 
++    // speculative-impl boundary state (e.g. the MTP pending hidden rows)
++    // captured at the same position as the KV data above, so a restore
++    // reinstates an exact (KV, boundary) pair
++    std::vector<uint8_t> data_spec;
++
+     llama_state_seq_storage * storage_tgt = nullptr;
+     llama_state_seq_storage * storage_dft = nullptr;
+ 
+diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
+index 80ee81e..02bbb97 100644
+--- a/tools/server/server-context.cpp
++++ b/tools/server/server-context.cpp
+@@ -2029,6 +2029,15 @@ private:
+         cur.update_tgt(ctx_tgt,       slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);
+         cur.update_dft(ctx_dft.get(), slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);
+ 
++        // capture the speculative-impl boundary state (MTP pending hidden rows)
++        // at the same position as the KV payload, so both are restored as an
++        // exact pair. get_state() fails (returns false with empty data) when
++        // no valid boundary exists yet - then there is nothing to pair.
++        if (common_speculative_state_required(spec.get())) {
++            cur.data_spec.clear();
++            common_speculative_get_state(spec.get(), slot.id, cur.data_spec);
++        }
++
+         SLT_INF(slot,
+                 "created context checkpoint %d of %d (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB)\n",
+                 (int) slot.prompt.checkpoints.size(), params_base.n_ctx_checkpoints, cur.pos_min,
+@@ -2788,18 +2797,47 @@ private:
+                                 n_past = 0;
+                             }
+ 
+-                            // MTP carries only the endpoint and immediately preceding target hidden
+-                            // boundaries. Arbitrary partial-prefix rollback cannot be reconstructed
+-                            // from target/draft KV state, so reprocess cold instead of pairing a token
+-                            // with the wrong hidden row. Full-prefix extension and the exact-hit
+-                            // one-token replay below remain supported.
++                            // Stateful speculative decoding (e.g. the MTP pending hidden rows) pairs
++                            // each token with the boundary at its exact position. An arbitrary partial
++                            // prefix rollback cannot reconstruct that boundary, so the state must be
++                            // restored, not rebuilt: find the newest checkpoint that ends at or before
++                            // the last common position and restore its exact (KV, boundary) pair.
+                             if (common_speculative_state_required(spec.get()) &&
+                                 n_past > 0 && n_past < slot.prompt.n_tokens()) {
+-                                SLT_INF(slot,
+-                                        "prompt cache cold fallback: reason=spec-boundary-mismatch lcp=%d cached_tokens=%d request_tokens=%d\n",
+-                                        n_past, slot.prompt.n_tokens(), slot.task->n_tokens());
+-                                n_past = 0;
+-                                common_speculative_set_state(spec.get(), slot.id, {});
++                                const llama_pos pos_rollback = slot.prompt.tokens.pos_next(n_past) - 1;
++
++                                const auto it_spec = std::find_if(
++                                    slot.prompt.checkpoints.rbegin(), slot.prompt.checkpoints.rend(),
++                                    [pos_rollback](const common_prompt_checkpoint & cur) {
++                                        return cur.pos_max <= pos_rollback && !cur.data_spec.empty();
++                                    });
++
++                                if (it_spec != slot.prompt.checkpoints.rend()) {
++                                    // restore the checkpoint's exact state (target KV, draft KV and
++                                    // the speculative boundary) instead of re-seeding it
++                                    const bool restored_tgt = it_spec->load_tgt(ctx_tgt,       slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);
++                                    const bool restored_dft = it_spec->load_dft(ctx_dft.get(), slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);
++                                    const bool restored_spec = common_speculative_set_state(spec.get(), slot.id, it_spec->data_spec);
++
++                                    if (restored_tgt && restored_dft && restored_spec) {
++                                        n_past = std::min(slot.prompt.tokens.size_up_to_pos(it_spec->pos_max + 1), (size_t) it_spec->n_tokens);
++                                        SLT_WRN(slot,
++                                                "restored spec-stateful context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", n_past = %d, size = %.3f MiB)\n",
++                                                it_spec->pos_min, it_spec->pos_max, it_spec->n_tokens, n_past, (float) it_spec->size() / 1024 / 1024);
++                                    } else {
++                                        SLT_WRN(slot,
++                                                "failed to restore spec-stateful context checkpoint (target=%d, draft=%d, spec=%d, pos_min = %d, pos_max = %d); reprocessing cold\n",
++                                                (int) restored_tgt, (int) restored_dft, (int) restored_spec, it_spec->pos_min, it_spec->pos_max);
++                                        n_past = 0;
++                                        common_speculative_set_state(spec.get(), slot.id, {});
++                                    }
++                                } else {
++                                    SLT_INF(slot,
++                                            "prompt cache cold fallback: reason=spec-checkpoint-missing lcp=%d cached_tokens=%d request_tokens=%d\n",
++                                            n_past, slot.prompt.n_tokens(), slot.task->n_tokens());
++                                    n_past = 0;
++                                    common_speculative_set_state(spec.get(), slot.id, {});
++                                }
+                             }
+ 
+                             llama_pos pos_next = slot.prompt.tokens.pos_next(n_past);
+@@ -2875,6 +2913,12 @@ private:
+                                             if (cur.pos_max > pos_next) {
+                                                 return false;
+                                             }
++                                            // a spec-stateful checkpoint must also be able to restore
++                                            // its boundary pair, otherwise the reprocessed tokens
++                                            // would pair with a stale MTP boundary
++                                            if (common_speculative_state_required(spec.get()) && cur.data_spec.empty()) {
++                                                return false;
++                                            }
+                                             return cur.pos_min < pos_min_thold || cur.pos_min == 0;
+                                         }
+                                     );
+@@ -2886,11 +2930,13 @@ private:
+ 
+                                         const bool restored_tgt = it->load_tgt(ctx_tgt,       slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);
+                                         const bool restored_dft = it->load_dft(ctx_dft.get(), slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);
++                                        const bool restored_spec = !common_speculative_state_required(spec.get()) ||
++                                            common_speculative_set_state(spec.get(), slot.id, it->data_spec);
+ 
+-                                        if (!restored_tgt || !restored_dft) {
++                                        if (!restored_tgt || !restored_dft || !restored_spec) {
+                                             SLT_WRN(slot,
+-                                                    "failed to restore context checkpoint (target=%d, draft=%d, pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB); forcing full prompt re-processing\n",
+-                                                    (int) restored_tgt, (int) restored_dft,
++                                                    "failed to restore context checkpoint (target=%d, draft=%d, spec=%d, pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB); forcing full prompt re-processing\n",
++                                                    (int) restored_tgt, (int) restored_dft, (int) restored_spec,
+                                                     it->pos_min, it->pos_max, it->n_tokens,
+                                                     (float) it->size() / 1024 / 1024);
+                                             do_reset = true;
+diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp
+index 6e0bb85..b018f64 100644
+--- a/tools/server/server-task.cpp
++++ b/tools/server/server-task.cpp
+@@ -3004,8 +3004,14 @@ bool server_prompt_cache::load(
+ 
+     const int lcp_best = prompt.tokens.get_common_prefix(tokens_new);
+ 
++    // With a stateful speculative implementation the slot's own cached prompt
++    // may be longer than the common prefix. That entry keeps its full KV but
++    // its exact-boundary spec state applies only at its endpoint; a partial
++    // prefix is still a valid cache hit because the prompt-processing path can
++    // restore a spec-stateful checkpoint instead of rebuilding the boundary.
+     const bool base_boundary_valid = !spec_state_required ||
+-        lcp_best == (int) prompt.tokens.size();
++        lcp_best == (int) prompt.tokens.size() ||
++        !prompt.checkpoints.empty();
+     float f_keep_best = base_boundary_valid && prompt.tokens.size() > 0 ? float(lcp_best) / prompt.tokens.size() : -1.0f; // empty slot: any cache entry wins
+     float sim_best    = base_boundary_valid ? float(lcp_best) / std::max<size_t>(1, tokens_new.size()) : -1.0f;
+ 
+@@ -3019,7 +3025,11 @@ bool server_prompt_cache::load(
+     auto it_best_ram  = states.end();
+     auto it_best_disk = disk_states.end();
+     size_t lcp_selected = 0;
+-    size_t spec_boundary_best = base_boundary_valid ? prompt.tokens.size() : 0;
++    // the base entry's usable boundary is its endpoint on an exact hit, or the
++    // common prefix (reachable via checkpoint restore) on a partial hit
++    size_t spec_boundary_best = base_boundary_valid
++        ? (lcp_best == (int) prompt.tokens.size() ? prompt.tokens.size() : (size_t) std::max(0, lcp_best))
++        : 0;
+     bool ram_loaded = false;
+ 
+     // Find the most similar RAM prompt first. On an equal match, the hot RAM
+@@ -3027,8 +3037,13 @@ bool server_prompt_cache::load(
+     for (auto it = states.begin(); it != states.end(); ++it) {
+         const int lcp_cur = it->tokens.get_common_prefix(tokens_new);
+ 
+-        if (spec_state_required &&
+-            lcp_cur != (int) it->tokens.size()) {
++        // exact boundary -> the entry's own spec state is usable as-is;
++        // partial prefix -> still usable, but only via checkpoint restore
++        // (see server-context.cpp), so require checkpoints to exist
++        const bool boundary_ok = !spec_state_required ||
++            lcp_cur == (int) it->tokens.size() ||
++            !it->checkpoints.empty();
++        if (!boundary_ok) {
+             SRV_INF("prompt cache skip: reason=spec-boundary-mismatch source=ram lcp=%d cached_tokens=%zu request_tokens=%zu spec_bytes=%zu\n",
+                     lcp_cur, it->tokens.size(), tokens_new.size(), it->data.spec.size());
+             continue;
+@@ -3048,12 +3063,14 @@ bool server_prompt_cache::load(
+         }
+ 
+         const bool is_better = spec_state_required
+-            ? it->tokens.size() > spec_boundary_best
++            ? (lcp_cur == (int) it->tokens.size()
++                ? it->tokens.size() > spec_boundary_best
++                : lcp_cur > (int) spec_boundary_best)
+             : f_keep_best < f_keep_cur && sim_best < sim_cur;
+         if (is_better) {
+             f_keep_best = f_keep_cur;
+             sim_best    = sim_cur;
+-            spec_boundary_best = it->tokens.size();
++            spec_boundary_best = std::max(spec_boundary_best, (size_t) std::max(0, lcp_cur));
+ 
+             it_best_ram  = it;
+             it_best_disk = disk_states.end();
+@@ -3158,6 +3175,10 @@ bool server_prompt_cache::load(
+         ram_loaded = true;
+     }
+ 
++    // note: a partial-prefix base entry with checkpoints was made valid above;
++    // the prompt-processing path in server-context.cpp then either restores a
++    // spec-stateful checkpoint at/below the common prefix or falls back to a
++    // cold reprocess, so correctness never depends on the relaxed acceptance.
+     return base_boundary_valid || ram_loaded;
+ }
+ 

recipe/pipeline/aggregate_v2.py

@@ -0,0 +1,230 @@
+#!/usr/bin/env python3
+"""Collect every measured number for the in-file-MTP Agnes cards into results/summary_v2.json.
+Nothing is typed by hand: each value is parsed from a log, a results file, or the GGUF itself; missing => None.
+Quality (KLD/PPL) comes from regrade.sh: the SHIPPED files graded against BF16 logits recomputed in the same session
+(R2/R3), and only if the repeat run (R4) reproduced every per-chunk row. The afternoon grades (pre-reboot) did not
+reproduce for the 4-bit tiers and are kept only as history."""
+import hashlib, json, os, re, sys
+sys.path.insert(0, "/opt/llama-rocm/rocmfpx-724/gguf-py")
+import gguf  # noqa: E402
+
+W = "/mnt/models/agnes-3.0-flash"; N = "Agnes-3.0-Flash-Preview"
+OB = "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin"
+FAST = os.environ.get("AGG_FAST") == "1"          # dry run: never hash multi-GiB files that were not staged
+OUT = os.environ.get("AGG_OUT", "/mnt/models/agnes-3.0-flash/results/summary_v2.json")
+PB = "/opt/llama-rocm/rocmfpx-724-mtpcache/build-hipvk/bin"
+
+
+def rd(p):
+    try: return open(p, errors="replace").read()
+    except FileNotFoundError: return ""
+
+
+def f(rx, s, g=1, cast=float):
+    m = re.search(rx, s)
+    return cast(m.group(g)) if m else None
+
+
+def jl(p):
+    return [json.loads(l) for l in rd(p).splitlines() if l.strip()]
+
+
+def pubname(fn):
+    """Published filename: the quant token must be hyphen-delimited or the Hub cannot parse the variant
+    (`Q4_0_ROCMFP4_STRIX_LEAN` reads as the garbage label `Q4_0_ROCMFP`)."""
+    for a, b in (("Q4_0_ROCMFP4_", "Q4_0-ROCmFP4-"), ("Q8_0_ROCMFPX_AGENT", "Q8_0-ROCmFPX-AGENT"), ("Q8_0_ROCMFPX", "Q8_0-ROCmFPX")):
+        if a in fn:
+            return fn.replace(a, b)
+    return fn
+
+
+STAGED = {}  # sha256 already computed at staging time (hf_publish_v2.py stage) - avoids re-reading 145 GiB
+for _k in ("std", "imat"):
+    for _l in rd(f"{W}/hf-upload-v2/{_k}/SHA256SUMS").splitlines():
+        _h, _n = _l.split(maxsplit=1); STAGED[_n.strip()] = _h
+
+
+def sha_file(p):
+    if not os.path.exists(p): return None
+    if pubname(os.path.basename(p)) in STAGED: return STAGED[pubname(os.path.basename(p))]
+    if FAST and os.path.getsize(p) > 1 << 30: return None
+    h = hashlib.sha256()
+    with open(p, "rb") as fh:
+        for c in iter(lambda: fh.read(64 << 20), b""): h.update(c)
+    return h.hexdigest()
+
+
+def tensors(p):
+    out = {}
+    for t in gguf.GGUFReader(p).tensors:
+        try: tn = t.tensor_type.name
+        except Exception: tn = f"TYPE_{int(t.tensor_type)}"
+        out[t.name] = (tn, int(t.n_bytes))
+    return out
+
+
+S = {"layout": "in-file MTP (one GGUF per tier; the MTP block shares the trunk's embedding and output matrices)",
+     "model": N, "model_repo_std": f"kingjones777/{N}-MTP-ROCmFP4-GGUF",
+     "model_repo_imat": f"kingjones777/{N}-MTP-ROCmFP4-imatrix-GGUF",
+     "source": {"repo": "Agnes-AI/Agnes-3.0-Flash", "revision": "891ce4f9ffb89b22888aa7fcc2bb2f3618867684"}}
+
+# ---------- BF16 reference (re-run in the grading session) ----------
+REG = {d.get("step"): d for d in jl(f"{W}/results/regrade.jsonl")}
+MEASURED = REG.get("R4", {}).get("result") == "MATCH"
+b1 = rd(f"{W}/logs/R2_ppl_bf16.log"); k106 = rd(f"{W}/logs/R3_kld_q106.log")
+b1_old = rd(f"{W}/logs/B1_ppl_bf16.log"); k106_old = rd(f"{W}/logs/B5_kld_q106.log")
+chunks = f(r"calculating perplexity over (\d+) chunks", b1, cast=int); nctx = f(r"n_ctx=(\d+)", b1, cast=int)
+S["bf16"] = {"ppl": f(r"Final estimate: PPL = ([\d.]+)", b1), "ppl_err": f(r"Final estimate: PPL = [\d.]+ \+/- ([\d.]+)", b1),
+             "ppl_paired": f(r"Mean PPL\(base\)\s*:\s*([\d.]+)", k106),
+             "ppl_paired_err": f(r"Mean PPL\(base\)\s*:\s*[\d.]+\s*±\s*([\d.]+)", k106),
+             "afternoon_ppl": f(r"Final estimate: PPL = ([\d.]+)", b1_old),
+             "afternoon_ppl_paired": f(r"Mean PPL\(base\)\s*:\s*([\d.]+)", k106_old),
+             "chunks": chunks, "n_ctx": nctx, "scored_tokens": chunks * (nctx // 2 - 1) if chunks and nctx else None,
+             "trunk_size_bytes": os.path.getsize(f"{W}/gguf/{N}-BF16.gguf") if os.path.exists(f"{W}/gguf/{N}-BF16.gguf") else None,
+             "with_mtp_size_bytes": os.path.getsize(f"{W}/gguf/{N}-MTP-BF16.gguf") if os.path.exists(f"{W}/gguf/{N}-MTP-BF16.gguf") else None}
+
+# ---------- binaries ----------
+pl = rd(f"{W}/logs/mtpcache_build.log")
+S["binary"] = {
+    "repo": "https://github.com/charlie12345/ROCmFPX", "commit": "d3ca53726109b46fd5cd6fa42f688c707079b44e",
+    "commit_in_official_main": "yes (ROCmFPX/ROCmFPX main is ahead of it, 0 behind; checked via the GitHub compare API 2026-09-16)",
+    "original": {"dir": OB, "sha256": {x: sha_file(f"{OB}/{x}") for x in
+                                       ("llama-server", "llama-quantize", "llama-imatrix", "llama-perplexity")}},
+    "patched": {"dir": PB, "sha256": {"llama-server": sha_file(f"{PB}/llama-server")},
+                "patch_file": "recipe/patches/mtp-prompt-cache-fix.patch",
+                "patch_sha256": (rd(f"{W}/logs/P1_patch.sha256").split() or [None])[0],
+                "patch_source": "kingjones777/Qwen3.8-27B-ROCmFP4-STRIX-MTP-GGUF (patches/mtp-prompt-cache-fix.patch)",
+                "applied": "cleanly (git apply --check)" if "applied CLEANLY" in pl else None,
+                "diffstat": [l.strip() for l in pl.splitlines() if "|" in l and ("+" in l or "-" in l)][:8],
+                "built_target": "llama-server only (same CMake cache as the original build)"}}
+
+vp = f"{W}/calib/vision_probe.png"
+if os.path.exists(vp):
+    raw = open(vp, "rb").read()
+    S["vision_probe"] = {"width": int.from_bytes(raw[16:20], "big"), "height": int.from_bytes(raw[20:24], "big"),
+                         "bytes": len(raw), "sha256": hashlib.sha256(raw).hexdigest()}
+imat = rd(f"{W}/logs/B2_imatrix.log")
+S["imatrix"] = {"chunks": f(r"computing over (\d+) chunks", imat, cast=int), "n_ctx": f(r"n_ctx=(\d+)", imat, cast=int),
+                "calibration": "bartowski calibration_datav3.txt", "file": f"{N}.imatrix",
+                "sha256": (rd(f"{W}/imat/{N}.imatrix.sha256").split() or [None])[0],
+                "size_bytes": os.path.getsize(f"{W}/imat/{N}.imatrix") if os.path.exists(f"{W}/imat/{N}.imatrix") else None}
+
+# ---------- tiers ----------
+TIERS = {  # tag: (new dir, new file, quant log, graded dir, graded file, ftype)
+    "q106": ("out-mtp", f"{N}-MTP-Q4_0_ROCMFP4_STRIX_LEAN.gguf", "S2_q106", "out", f"{N}-Q4_0_ROCMFP4_STRIX_LEAN.gguf", 106),
+    "q102": ("out-mtp", f"{N}-MTP-Q4_0_ROCMFP4_COHERENT.gguf", "S2_q102", "out", f"{N}-Q4_0_ROCMFP4_COHERENT.gguf", 102),
+    "q115": ("out-mtp", f"{N}-MTP-Q8_0_ROCMFPX_AGENT.gguf", "S2_q115", "out", f"{N}-Q8_0_ROCMFPX_AGENT.gguf", 115),
+    "q111": ("out-mtp", f"{N}-MTP-Q8_0_ROCMFPX.gguf", "S2_q111", "out", f"{N}-Q8_0_ROCMFPX.gguf", 111),
+    "q106i": ("out-mtp-imat", f"{N}-MTP-imatrix-Q4_0_ROCMFP4_STRIX_LEAN.gguf", "S3_q106i", "out-imat",
+              f"{N}-imatrix-Q4_0_ROCMFP4_STRIX_LEAN.gguf", 106),
+    "q102i": ("out-mtp-imat", f"{N}-MTP-imatrix-Q4_0_ROCMFP4_COHERENT.gguf", "S3_q102i", "out-imat",
+              f"{N}-imatrix-Q4_0_ROCMFP4_COHERENT.gguf", 102)}
+rb = rd(f"{W}/logs/S_readback.log"); ident = rd(f"{W}/logs/S_identity.log")
+receipt = {r["tag"]: r for r in jl(f"{W}/results/trunk_receipt.jsonl")}
+
+
+def kld_stats(k):
+    return {"ppl": f(r"Mean PPL\(Q\)\s*:\s*([\d.]+)", k), "ppl_err": f(r"Mean PPL\(Q\)\s*:\s*[\d.]+\s*±\s*([\d.]+)", k),
+            "ppl_ratio": f(r"Mean PPL\(Q\)/PPL\(base\)\s*:\s*([\d.]+)", k),
+            "kld_mean": f(r"Mean\s+KLD:\s*([\d.]+)", k), "kld_err": f(r"Mean\s+KLD:\s*[\d.]+\s*±\s*([\d.]+)", k),
+            "kld_p99": f(r"99\.0%\s+KLD:\s*([\d.]+)", k), "kld_median": f(r"Median\s+KLD:\s*([\d.]+)", k),
+            "same_top_p": f(r"Same top p:\s*([\d.]+)", k), "rms_dp": f(r"RMS Δp\s*:\s*([\d.]+)", k)}
+S["tiers"] = {}
+TEN = {}
+for tag, (d, fn, qlog, gd, gfn, ft) in TIERS.items():
+    p, gp = f"{W}/{d}/{fn}", f"{W}/{gd}/{gfn}"
+    q = rd(f"{W}/logs/{qlog}.log"); k = rd(f"{W}/logs/R3_kld_{tag}.log")
+    line = next((l for l in rb.splitlines() if f" {fn} " in l), "")
+    iline = next((l for l in ident.splitlines() if l.startswith(fn + " ")), "")
+    idj = json.loads(iline[len(fn) + 1:]) if iline else {}
+    ten = tensors(p) if os.path.exists(p) else {}
+    TEN[tag] = ten
+    mtp = {n: v for n, v in ten.items() if n.startswith("blk.72.")}
+    measured = bool(MEASURED and f(r"Mean\s+KLD:\s*([\d.]+)", k) is not None)
+    t = {"file": pubname(fn), "local_file": fn, "dir": d, "ftype": ft, "size_bytes": os.path.getsize(p) if os.path.exists(p) else None,
+         "graded_file": gfn, "graded_size_bytes": os.path.getsize(gp) if os.path.exists(gp) else None,
+         "quant_mib": f(r"quant size\s*=\s*([\d.]+) MiB", q), "bpw": f(r"quant size\s*=\s*[\d.]+ MiB \(([\d.]+) BPW\)", q),
+         "quant_seconds": (f(r"quantize time\s*=\s*([\d.]+) ms", q) or 0) / 1000 or None,
+         "imatrix_entries": f(r"loaded (\d+) importance matrix entries", q, cast=int),
+         "readback": line.split()[0] if line else None,
+         "tensors": f(r"tensors=(\d+)", line, cast=int), "nextn_tensors": f(r"nextn=(\d+)", line, cast=int),
+         "output_weight": f(r"output\.weight=(\S+)", line, cast=str), "token_embd": f(r"token_embd\.weight=(\S+)", line, cast=str),
+         "trunk_identity": {k2: idj.get(k2) for k2 in ("result", "shared", "n_type_changed", "n_bytes_changed", "only_in_old")},
+         "receipt_vs_afternoon": receipt.get(tag, {}).get("result"),
+         "quality_measured": measured,
+         "afternoon": kld_stats(rd(f"{W}/logs/B5_kld_{tag}.log")),
+         "mtp_block": {"tensors": len(mtp), "bytes": sum(v[1] for v in mtp.values()),
+                       "types": {n[7:]: v[0] for n, v in sorted(mtp.items()) if not v[0].startswith("F32")}}}
+    t.update(kld_stats(k) if measured else {x: None for x in kld_stats("")})
+    S["tiers"][tag] = t
+for a_, b_ in (("q106", "q106i"), ("q102", "q102i")):
+    pa = f"{W}/{TIERS[a_][0]}/{TIERS[a_][1]}"; pb = f"{W}/{TIERS[b_][0]}/{TIERS[b_][1]}"
+    if TEN.get(a_) and TEN.get(b_):   # same names, types and per-tensor byte sizes => same per-token work
+        S["tiers"][b_]["same_tensor_types_as_standard"] = TEN[a_] == TEN[b_]
+        S["tiers"][b_]["file_size_delta_bytes"] = S["tiers"][b_]["size_bytes"] - S["tiers"][a_]["size_bytes"]
+    ha, hb = (sha_file(pa), sha_file(pb)) if os.path.exists(pa) and os.path.exists(pb) else (None, None)
+    S["tiers"][b_]["differs_from_standard"] = (ha != hb) if ha and hb else None
+
+# the separate head this layout replaces (not shipped any more)
+S["separate_head_replaced"] = {}
+for fn in (f"mtp-{N}-Q4_0.gguf", f"mtp-{N}-Q8_0.gguf"):
+    p = f"{W}/out/{fn}"
+    if os.path.exists(p):
+        ten = tensors(p)
+        S["separate_head_replaced"][fn] = {
+            "size_bytes": os.path.getsize(p), "tensors": len(ten), "tensor_bytes": sum(v[1] for v in ten.values()),
+            "vocab_matrices_bytes": sum(v[1] for n, v in ten.items() if n in ("output.weight", "token_embd.weight")),
+            "vocab_matrix_types": {n: v[0] for n, v in ten.items() if n in ("output.weight", "token_embd.weight")}}
+S["aux"] = {f"mmproj-{N}-BF16.gguf": os.path.getsize(f"{W}/out/mmproj-{N}-BF16.gguf")
+            if os.path.exists(f"{W}/out/mmproj-{N}-BF16.gguf") else None}
+
+# ---------- measurements ----------
+S["bench"], S["gates"] = [], []
+for d in jl(f"{W}/results/mtpfix.jsonl"):
+    if "tg_median" in d: S["bench"].append({k: v for k, v in d.items() if k != "runs"})
+    else:
+        S["gates"].append({k: v for k, v in d.items() if k != "text"} if d.get("label", "").startswith("id-") else d)
+PE = re.compile(r"prompt eval time =\s*([\d.]+) ms /\s*(\d+) tokens")
+for g_ in S["gates"]:
+    if not g_.get("label", "").startswith(("c2-", "c3-")) or not g_.get("rows"): continue
+    ev = [(float(a), int(b)) for a, b in PE.findall(rd(f"{W}/logs/v_{g_['label']}.log"))]
+    per = 3  # warm-up A, B warm, B cold - one server log line each, in order
+    for i, r_ in enumerate(g_["rows"]):
+        blk = ev[per * i: per * i + per]
+        if len(blk) == per and blk[1][1] == r_["warm_prompt_n"] and blk[2][1] == r_["cold_prompt_n"]:
+            r_["warm_prompt_ms"], r_["cold_prompt_ms"] = blk[1][0], blk[2][0]
+        else:
+            r_["warm_prompt_ms"] = r_["cold_prompt_ms"] = None
+sw = [b for b in S["bench"] if b["label"] in ("p-infile-rocm-n3", "p-infile-rocm-n4", "p-infile-rocm-n5")]
+S["knee"] = max(sw, key=lambda b: b["tg_median"])["nmax"] if len(sw) == 3 else None
+S["tier_bench_nmax"] = 4  # vplan_b.py runs every tier at --spec-draft-n-max 4
+S["history_separate_head"] = {
+    "note": "Measured 2026-09-16 with the separate 18-tensor head file (not shipped any more); same trunk bytes.",
+    "bench": [{k: v for k, v in d.items() if k != "runs"} for d in jl(f"{W}/results/phase_b.jsonl") if "tg_median" in d],
+    "gates": [d for d in jl(f"{W}/results/phase_b.jsonl") if "tg_median" not in d],
+    "knee": (json.loads(rd(f"{W}/results/knee.json")) if rd(f"{W}/results/knee.json") else {}).get("knee"),
+    "seat_sizing": jl(f"{W}/results/seat_sizing.jsonl")}
+S["seat_sizing2"] = jl(f"{W}/results/seat_sizing2.jsonl")
+S["trunk_receipt"] = list(receipt.values())
+S["regrade"] = {"R1_cpu_burn": REG.get("R1"), "R2_base_logits": REG.get("R2"), "R4_repeat": REG.get("R4"),
+                "kld_control": jl(f"{W}/results/kld_control.jsonl"),
+                "R1_pass_seconds": {k: f(r"([\d.]+) seconds per pass", rd(f"{W}/logs/{v}"))
+                                    for k, v in (("cpu_burn", "R1_kld4_old_q106_cpuburn.log"),
+                                                 ("quiet_today", "K_c1-old-q106.log"),
+                                                 ("afternoon", "B5_kld_q106.log"))},
+                "R1_burn_started": "quantizing" in rd(f"{W}/logs/R1_burn.log"),
+                "note": ("R1: 4-bit rows under a concurrent 16-thread CPU burn vs today's quiet run and the afternoon run. "
+                         "The burn_seen field in regrade.jsonl is a grep bug (searched 'quantize', the log says "
+                         "'quantizing'); R1_burn_started and R1_pass_seconds are the evidence that the load was real.")}
+S["fold"] = json.loads(rd(f"{W}/qwen35/FOLD_PROVENANCE.json") or "{}")
+vf = rd(f"{W}/logs/A3_verify_fold.log")
+S["fold_verify"] = {"result": f(r"RESULT: (\w+)", vf, cast=str),
+                    "g3": re.findall(r"G3 (layer\s+\d+|mtp)\s*: max rel err ([\d.e+-]+)", vf)}
+json.dump(S, open(OUT, "w"), indent=2)
+
+print("knee(in-file)=", S["knee"], "| bench rows", len(S["bench"]), "| gate rows", len(S["gates"]),
+      "| sizing2 rows", len(S["seat_sizing2"]), "| receipts", len(S["trunk_receipt"]))
+for tg_, v in S["tiers"].items():
+    print(tg_, {k: v[k] for k in ("size_bytes", "bpw", "readback", "tensors", "nextn_tensors", "receipt_vs_afternoon",
+                                  "quality_measured", "kld_mean")}, "mtp_block", v["mtp_block"]["bytes"])
+print("binary", json.dumps(S["binary"], indent=1))

recipe/pipeline/agnes_harness.py

@@ -0,0 +1,321 @@
+#!/usr/bin/env python3
+"""Phase B harness for Agnes-3.0-Flash Preview on MAX-1 (stdlib only).
+House protocol (Qwen3.8-27B card): ctx 65536, batch 1, greedy (temp 0, top_k 1), ignore_eos -> exactly
+256 tokens, unique nonce + cache_prompt:false (cached tokens asserted 0), median of reps after 1 warm-up.
+Subcommands:  bench | cachegate | tools | vision   (see argparse)"""
+import argparse, base64, json, os, signal, statistics, subprocess, sys, time, urllib.request, uuid
+
+W = "/mnt/models/agnes-3.0-flash"
+BIN = os.environ.get("AGNES_BIN", "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin")
+ENV = dict(os.environ, LD_LIBRARY_PATH=f"{BIN}:/opt/rocm-7.2.4/lib", HSA_OVERRIDE_GFX_VERSION="11.5.1",
+           GGML_HIP_ENABLE_UNIFIED_MEMORY="1")
+
+def post(port, path, body, timeout=1800):
+    req = urllib.request.Request(f"http://127.0.0.1:{port}{path}", data=json.dumps(body).encode(),
+                                 headers={"Content-Type": "application/json"})
+    return json.load(urllib.request.urlopen(req, timeout=timeout))
+
+class Server:
+    def __init__(self, a, port, mmproj=None, fa="on"):
+        self.port = port
+        cmd = [f"{BIN}/llama-server", "-m", a.model, "-dev", a.dev, "-ngl", "999", "-fa", fa, "-dio",
+               "--jinja", "-fit", "off", "--parallel", "1", "-c", str(a.ctx), "-b", "2048", "-ub", "1024",
+               "--host", "127.0.0.1", "--port", str(port), "--no-webui"]
+        if a.draft or a.mtp_infile:
+            cmd += ["--spec-type", "draft-mtp"]
+            if a.draft:
+                cmd += ["--model-draft", a.draft, "--spec-draft-ngl", "99", "--spec-draft-device", a.dev]
+            cmd += ["--spec-draft-n-max", str(a.nmax), "--spec-draft-n-min", "0", "--spec-draft-p-min", str(a.pmin)]
+            if a.strict:
+                cmd += ["--spec-mtp-strict-qwen"]
+        if mmproj:
+            cmd += ["--mmproj", mmproj]
+        self.cmd = cmd
+        self.logf = open(a.serverlog, "w")
+        self.t0 = time.time()
+        self.p = subprocess.Popen(cmd, env=ENV, stdout=self.logf, stderr=subprocess.STDOUT, start_new_session=True)
+        while True:
+            if self.p.poll() is not None:
+                raise SystemExit(f"SERVER DIED rc={self.p.returncode} see {a.serverlog}")
+            try:
+                if json.load(urllib.request.urlopen(f"http://127.0.0.1:{port}/health", timeout=3)).get("status") == "ok":
+                    break
+            except Exception:
+                pass
+            if time.time() - self.t0 > 900:
+                self.stop(); raise SystemExit("SERVER LOAD TIMEOUT")
+            time.sleep(2)
+        self.load_s = time.time() - self.t0
+    def stop(self):
+        try: os.killpg(self.p.pid, signal.SIGTERM); self.p.wait(60)
+        except Exception:
+            try: os.killpg(self.p.pid, signal.SIGKILL)
+            except Exception: pass
+        self.logf.close()
+
+def prompt_8k(kind):
+    if kind == "code":
+        src = open("/opt/llama-rocm/rocmfpx-724/convert_hf_to_gguf.py").read()[:30000]
+        return src, "Above is part of a model converter. Write a new, complete Python function that validates a GGUF tensor-name map against a list of HF tensor names and reports unmapped names. Code only."
+    txt = open(f"{W}/calib/wikitext-2-raw/wiki.train.raw").read()[:34000]
+    return txt, "Above are encyclopedia excerpts. Write a long, detailed new encyclopedia article in the same style about the history of lighthouses."
+
+def one_request(port, ctx_text, instr, n=256, cache=False, nonce=True):
+    tag = f"[req {uuid.uuid4()}]\n" if nonce else ""
+    body = {"messages": [{"role": "user", "content": tag + ctx_text + "\n\n" + instr}],
+            "max_tokens": n, "temperature": 0, "top_k": 1, "ignore_eos": True, "cache_prompt": cache,
+            "chat_template_kwargs": {"enable_thinking": False}}
+    t = time.time(); r = post(port, "/v1/chat/completions", body); el = time.time() - t
+    tm = r.get("timings", {})
+    return {"pred_n": tm.get("predicted_n"), "tg": tm.get("predicted_per_second"),
+            "pp": tm.get("prompt_per_second"), "prompt_n": tm.get("prompt_n"), "cache_n": tm.get("cache_n"),
+            "draft_n": tm.get("draft_n"), "draft_acc": tm.get("draft_n_accepted"), "wall": el,
+            "usage": r.get("usage", {})}
+
+def cmd_bench(a):
+    s = Server(a, a.port)
+    out = {"label": a.label, "model": os.path.basename(a.model),
+           "draft": os.path.basename(a.draft) if a.draft else ("in-file" if a.mtp_infile else None),
+           "nmax": a.nmax if (a.draft or a.mtp_infile) else None, "strict": bool(a.strict), "bin": BIN, "dev": a.dev, "ctx": a.ctx, "workload": a.workload,
+           "load_s": round(s.load_s, 1), "cmd": " ".join(s.cmd), "runs": []}
+    try:
+        ctx_text, instr = prompt_8k(a.workload)
+        one_request(a.port, ctx_text, instr)                      # warm-up, discarded
+        for _ in range(a.reps):
+            r = one_request(a.port, ctx_text, instr)
+            cached = (r["usage"].get("prompt_tokens_details") or {}).get("cached_tokens", 0) or 0
+            if r["pred_n"] != 256: raise SystemExit(f"GATE FAIL: predicted_n={r['pred_n']} != 256")
+            if (r["cache_n"] or 0) != 0 or cached != 0: raise SystemExit(f"GATE FAIL: cache hit cache_n={r['cache_n']} cached={cached}")
+            out["runs"].append(r)
+    finally:
+        s.stop()
+    tg = [r["tg"] for r in out["runs"]]; pp = [r["pp"] for r in out["runs"]]
+    dn = sum(r["draft_n"] or 0 for r in out["runs"]); da = sum(r["draft_acc"] or 0 for r in out["runs"])
+    out.update(tg_median=round(statistics.median(tg), 2), tg_min=round(min(tg), 2), tg_max=round(max(tg), 2),
+               pp_median=round(statistics.median(pp), 1), prompt_n=out["runs"][0]["prompt_n"],
+               accept=(round(da / dn, 3) if dn else None))
+    print(json.dumps({k: v for k, v in out.items() if k != "runs"}))
+    with open(a.jsonl, "a") as f: f.write(json.dumps(out) + "\n")
+
+def cmd_identity(a):
+    """Fixed prompt, no nonce, no cache, greedy: return the exact generated text for cross-config diffing."""
+    s = Server(a, a.port)
+    texts = []
+    try:
+        ctx_text, instr = prompt_8k(a.workload)
+        for _ in range(a.reps):
+            body = {"messages": [{"role": "user", "content": ctx_text + "\n\n" + instr}], "max_tokens": 256,
+                    "temperature": 0, "top_k": 1, "ignore_eos": True, "cache_prompt": False,
+                    "chat_template_kwargs": {"enable_thinking": False}}
+            r = post(a.port, "/v1/chat/completions", body)
+            texts.append(r["choices"][0]["message"].get("content") or "")
+    finally:
+        s.stop()
+    import hashlib
+    res = {"label": a.label, "reps": a.reps, "sha256": [hashlib.sha256(t.encode()).hexdigest()[:16] for t in texts],
+           "self_consistent": len(set(texts)) == 1, "text": texts[0]}
+    print(json.dumps({k: v for k, v in res.items() if k != "text"}))
+    with open(a.jsonl, "a") as f: f.write(json.dumps(res) + "\n")
+
+def cmd_cachegate2(a):
+    """Partial-prefix reuse WITH output identity: warm on A, run B warm (partial reuse), run B cold, diff outputs."""
+    import hashlib
+    s = Server(a, a.port)
+    rows = []
+    try:
+        ctx_text, _ = prompt_8k("code")
+        variants = ["Write a function that lists every tensor name in this file.",
+                    "Write a function that counts the model classes registered in this file.",
+                    "Write a function that finds the longest method in this file."]
+        for i, vb in enumerate(variants[:a.reps]):
+            def req(instr, n, cache):
+                body = {"messages": [{"role": "user", "content": ctx_text + "\n\n" + instr}], "max_tokens": n,
+                        "temperature": 0, "top_k": 1, "ignore_eos": True, "cache_prompt": cache,
+                        "chat_template_kwargs": {"enable_thinking": False}}
+                r = post(a.port, "/v1/chat/completions", body)
+                tm = r.get("timings", {})
+                return (r["choices"][0]["message"].get("content") or ""), tm.get("prompt_n"), tm.get("cache_n")
+            req(f"Summarise this file. (warm-up {i})", 16, True)
+            tw, pw, cw = req(vb, 192, True)
+            tc, pc, cc = req(vb, 192, False)
+            rows.append({"variant": i, "warm_prompt_n": pw, "warm_cache_n": cw, "cold_prompt_n": pc, "cold_cache_n": cc,
+                         "identical": tw == tc, "warm_sha": hashlib.sha256(tw.encode()).hexdigest()[:12],
+                         "cold_sha": hashlib.sha256(tc.encode()).hexdigest()[:12]})
+            print("  ", json.dumps(rows[-1]), flush=True)
+    finally:
+        s.stop()
+    reused = all((r["warm_cache_n"] or 0) > 0 for r in rows)
+    ident = all(r["identical"] for r in rows)
+    res = {"label": a.label, "rows": rows, "all_reused": reused, "all_identical": ident,
+           "result": "PASS" if (reused and ident) else "FAIL"}
+    print(json.dumps({k: v for k, v in res.items() if k != "rows"}))
+    with open(a.jsonl, "a") as f: f.write(json.dumps(res) + "\n")
+    sys.exit(0 if res["result"] == "PASS" else 1)
+
+def cmd_cachegate(a):
+    """Repeat-prompt test WITH the draft head loaded: turn 2 must reuse turn 1's prefix."""
+    s = Server(a, a.port)
+    try:
+        ctx_text, instr = prompt_8k("code")
+        r1 = one_request(a.port, ctx_text, instr, n=32, cache=True, nonce=False)
+        r2 = one_request(a.port, ctx_text, instr + " Also add type hints.", n=32, cache=True, nonce=False)
+    finally:
+        s.stop()
+    res = {"label": a.label, "turn1_prompt_n": r1["prompt_n"], "turn2_prompt_n": r2["prompt_n"],
+           "turn2_cache_n": r2["cache_n"]}
+    # hybrid recurrent models resume only from context checkpoints (~1024-token spacing), so the test is
+    # "any prefix reuse" -- report the fraction rather than demand near-total reuse.
+    ok = isinstance(r2["cache_n"], int) and r2["cache_n"] > 0
+    res["reuse_fraction"] = round(r2["cache_n"] / r1["prompt_n"], 3) if ok and r1["prompt_n"] else 0.0
+    res["result"] = "PASS" if ok else "FAIL"
+    print(json.dumps(res))
+    with open(a.jsonl, "a") as f: f.write(json.dumps(res) + "\n")
+    sys.exit(0 if ok else 1)
+
+TOOLS = [
+ {"type": "function", "function": {"name": "get_weather", "description": "Current weather for a city",
+  "parameters": {"type": "object", "properties": {"city": {"type": "string"},
+   "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}}, "required": ["city", "unit"]}}},
+ {"type": "function", "function": {"name": "create_event", "description": "Create a calendar event",
+  "parameters": {"type": "object", "properties": {"title": {"type": "string"},
+   "when": {"type": "object", "properties": {"date": {"type": "string"}, "time": {"type": "string"}},
+            "required": ["date", "time"]},
+   "attendees": {"type": "array", "items": {"type": "string"}}}, "required": ["title", "when", "attendees"]}}},
+]
+LEAK = ("<tool_call>", "<function=", "<parameter=", "<think>", "</think>")
+
+def chat(port, msgs, think, stream=False, tools=TOOLS):
+    body = {"messages": msgs, "tools": tools, "tool_choice": "auto", "temperature": 1.0, "top_p": 0.95,
+            "top_k": 20, "max_tokens": 4096,
+            "chat_template_kwargs": ({"enable_thinking": True, "reasoning_effort": "low"} if think else {"enable_thinking": False})}
+    if not stream:
+        return post(port, "/v1/chat/completions", body)["choices"][0]["message"]
+    body["stream"] = True
+    req = urllib.request.Request(f"http://127.0.0.1:{port}/v1/chat/completions", data=json.dumps(body).encode(),
+                                 headers={"Content-Type": "application/json"})
+    calls, content = {}, ""
+    for line in urllib.request.urlopen(req, timeout=1800):
+        line = line.decode().strip()
+        if not line.startswith("data:") or line.endswith("[DONE]"): continue
+        d = json.loads(line[5:])["choices"][0]["delta"]
+        content += d.get("content") or ""
+        for tc in d.get("tool_calls") or []:
+            c = calls.setdefault(tc["index"], {"name": "", "arguments": ""})
+            c["name"] += (tc.get("function") or {}).get("name") or ""
+            c["arguments"] += (tc.get("function") or {}).get("arguments") or ""
+    return {"content": content, "tool_calls": [{"function": v} for _, v in sorted(calls.items())]}
+
+def args_of(m, i=0):
+    return json.loads(m["tool_calls"][i]["function"]["arguments"])
+
+def cmd_tools(a):
+    s = Server(a, a.port)
+    results = {}
+    try:
+        for think in (True, False):
+            def check(name, fn):
+                try: ok, why = fn()
+                except Exception as e: ok, why = False, f"exception {e!r}"[:160]
+                results[f"{name}|think={think}"] = (ok, why)
+                print(f"  {'PASS' if ok else 'FAIL'} think={think!s:5} {name}: {why}", flush=True)
+            def clean(m): return not any(x in (m.get("content") or "") for x in LEAK)
+            def t1():
+                m = chat(a.port, [{"role": "user", "content": "What's the weather in Paris in celsius?"}], think)
+                ag = args_of(m); return (m["tool_calls"][0]["function"]["name"] == "get_weather" and ag.get("city", "").lower().startswith("paris")
+                        and ag.get("unit") == "celsius" and clean(m)), f"args={ag}"
+            def t2():
+                m = chat(a.port, [{"role": "user", "content": "Book 'Design review' on 2026-10-02 at 14:00 with ana@x.io and bo@x.io."}], think)
+                ag = args_of(m); return (isinstance(ag.get("when"), dict) and ag["when"].get("date") == "2026-10-02"
+                        and sorted(ag.get("attendees", [])) == ["ana@x.io", "bo@x.io"] and clean(m)), f"args={ag}"
+            def t3():
+                m = chat(a.port, [{"role": "user", "content": "Weather in Denver, and give it to me in fahrenheit."}], think)
+                return args_of(m).get("unit") == "fahrenheit" and clean(m), f"unit={args_of(m).get('unit')}"
+            def t4():
+                m = chat(a.port, [{"role": "user", "content": "What is 17 times 23? Answer directly."}], think)
+                c = m.get("content") or ""
+                return (not m.get("tool_calls")) and "391" in c and clean(m), f"content={c[:60]!r}"
+            def t5():
+                msgs = [{"role": "user", "content": "What's the weather in Tokyo in celsius?"}]
+                m = chat(a.port, msgs, think)
+                tc = m["tool_calls"][0]
+                msgs += [{"role": "assistant", "content": m.get("content") or "", "tool_calls": [
+                          {"id": "call_1", "type": "function", "function": tc["function"]}]},
+                         {"role": "tool", "tool_call_id": "call_1", "content": json.dumps({"temp_c": 21, "sky": "clear"})}]
+                m2 = chat(a.port, msgs, think)
+                c = m2.get("content") or ""
+                return ("21" in c and not m2.get("tool_calls") and clean(m2)), f"final={c[:70]!r}"
+            def t6():
+                m = chat(a.port, [{"role": "user", "content": "What's the weather in Rome in celsius?"}], think, stream=True)
+                ag = args_of(m); return (m["tool_calls"][0]["function"]["name"] == "get_weather"
+                        and ag.get("city", "").lower().startswith("rome") and clean(m)), f"stream args={ag}"
+            def t7():
+                m = chat(a.port, [{"role": "user", "content": "Get the weather in Oslo AND in Lima, both in celsius. Call the tool for each city."}], think)
+                cities = sorted(args_of(m, i).get("city", "").lower() for i in range(len(m.get("tool_calls") or [])))
+                return (len(cities) == 2 and cities[0].startswith("lima") and cities[1].startswith("oslo") and clean(m)), f"calls={cities}"
+            for nm, fn in (("multi-arg", t1), ("nested-object", t2), ("enum", t3), ("correct-decline", t4),
+                           ("multi-turn", t5), ("streaming", t6), ("parallel", t7)):
+                check(nm, fn)
+    finally:
+        s.stop()
+    n_ok = sum(v[0] for v in results.values())
+    summary = {"label": a.label, "passed": n_ok, "total": len(results),
+               "detail": {k: v[0] for k, v in results.items()}}
+    print(json.dumps(summary))
+    with open(a.jsonl, "a") as f: f.write(json.dumps(summary) + "\n")
+
+def cmd_vision(a):
+    """Image gate. A server that fails to load or dies on the image is a RESULT (FAIL row), not a harness crash."""
+    fa = a.fa or "off"
+    res = {"label": a.label, "fa": fa, "mtp": bool(a.draft or a.mtp_infile), "expected": a.expect,
+           "answer": "", "hits": [], "error": None, "server_died": False, "server_log_errors": []}
+    c = ""
+    try:
+        s = Server(a, a.port, mmproj=a.mmproj, fa=fa)
+    except SystemExit as e:
+        res.update(error=f"server did not start: {e}", server_died=True); s = None
+    if s is not None:
+        try:
+            img = base64.b64encode(open(a.image, "rb").read()).decode()
+            body = {"messages": [{"role": "user", "content": [
+                        {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img}"}},
+                        {"type": "text", "text": a.question}]}],
+                    "temperature": 0, "top_k": 1, "max_tokens": 300, "chat_template_kwargs": {"enable_thinking": False}}
+            try:
+                r = post(a.port, "/v1/chat/completions", body, timeout=900)
+                c = r["choices"][0]["message"].get("content") or ""
+            except Exception as e:
+                res["error"] = f"{type(e).__name__}: {e}"[:300]
+            time.sleep(1)
+            res["server_died"] = s.p.poll() is not None
+        finally:
+            s.stop()
+    try:
+        res["server_log_errors"] = [l.strip()[-200:] for l in open(a.serverlog, errors="replace")
+                                    if any(k in l for k in ("GGML_ABORT", "abort", "failed to process", " E "))][-5:]
+    except OSError:
+        pass
+    res["answer"] = c[:300]
+    res["hits"] = [w for w in a.expect.split(",") if w.lower() in c.lower()]
+    ok = res["error"] is None and not res["server_died"] and len(res["hits"]) == len(a.expect.split(","))
+    res["result"] = "PASS" if ok else "FAIL"
+    print(json.dumps(res))
+    with open(a.jsonl, "a") as f: f.write(json.dumps(res) + "\n")
+    sys.exit(0 if ok else 1)
+
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser()
+    ap.add_argument("cmd", choices=["bench", "cachegate", "cachegate2", "tools", "vision", "identity"])
+    ap.add_argument("--model", required=True); ap.add_argument("--draft")
+    ap.add_argument("--nmax", type=int, default=4); ap.add_argument("--pmin", type=float, default=0.0)
+    ap.add_argument("--dev", default="ROCm0"); ap.add_argument("--ctx", type=int, default=65536)
+    ap.add_argument("--port", type=int, default=18600); ap.add_argument("--reps", type=int, default=3)
+    ap.add_argument("--workload", default="code", choices=["code", "prose"])
+    ap.add_argument("--label", default=""); ap.add_argument("--jsonl", default=f"{W}/results/phase_b.jsonl")
+    ap.add_argument("--serverlog", default=f"{W}/logs/server_last.log")
+    ap.add_argument("--mmproj"); ap.add_argument("--image"); ap.add_argument("--question"); ap.add_argument("--expect")
+    ap.add_argument("--mtp-infile", action="store_true"); ap.add_argument("--strict", action="store_true")
+    ap.add_argument("--fa", choices=["on", "off", "auto"], help="vision only; default off")
+    a = ap.parse_args()
+    os.makedirs(os.path.dirname(a.jsonl), exist_ok=True)
+    {"bench": cmd_bench, "cachegate": cmd_cachegate, "tools": cmd_tools, "vision": cmd_vision,
+     "identity": cmd_identity, "cachegate2": cmd_cachegate2}[a.cmd](a)

recipe/pipeline/bench_plan.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+"""Phase B, GPU queue 3: speed matrix. ROCm n-max sweep first, knee chosen FROM THE DATA, then every
+other arm at that knee. Each run is an isolated agnes_harness.py process (house protocol)."""
+import json, os, subprocess, sys, time
+W = "/mnt/models/agnes-3.0-flash"; N = "Agnes-3.0-Flash-Preview"
+J = f"{W}/results/phase_b.jsonl"
+H = ["python3", f"{W}/agnes_harness.py"]
+T = {"q106": f"{W}/out/{N}-Q4_0_ROCMFP4_STRIX_LEAN.gguf", "q102": f"{W}/out/{N}-Q4_0_ROCMFP4_COHERENT.gguf",
+     "q115": f"{W}/out/{N}-Q8_0_ROCMFPX_AGENT.gguf", "q111": f"{W}/out/{N}-Q8_0_ROCMFPX.gguf",
+     "q106i": f"{W}/out-imat/{N}-imatrix-Q4_0_ROCMFP4_STRIX_LEAN.gguf"}
+D4, D8 = f"{W}/out/mtp-{N}-Q4_0.gguf", f"{W}/out/mtp-{N}-Q8_0.gguf"
+def log(m): print(f"[{time.strftime('%FT%TZ', time.gmtime())}] {m}", flush=True)
+def bench(label, model, dev="ROCm0", draft=None, nmax=4, workload="code", reps=3):
+    a = H + ["bench", "--model", model, "--dev", dev, "--label", label, "--workload", workload,
+             "--reps", str(reps), "--serverlog", f"{W}/logs/srv_{label}.log"]
+    if draft: a += ["--draft", draft, "--nmax", str(nmax)]
+    log(f"bench {label}")
+    r = subprocess.run(a, capture_output=True, text=True)
+    print(r.stdout.strip()[-600:], flush=True)
+    if r.returncode != 0:
+        print("  ERR", r.stderr.strip()[-400:], flush=True)
+    return r.returncode == 0
+def rows(prefix):
+    out = []
+    for line in open(J):
+        d = json.loads(line)
+        if d.get("label", "").startswith(prefix) and "tg_median" in d: out.append(d)
+    return out
+# 1. ROCm sweep on the recommended tier
+bench("sweep-rocm-off", T["q106"])
+for n in (1, 2, 3, 4, 5, 6):
+    bench(f"sweep-rocm-n{n}", T["q106"], draft=D4, nmax=n)
+sw = [d for d in rows("sweep-rocm-n")]
+knee = max(sw, key=lambda d: d["tg_median"])["nmax"]
+log(f"KNEE (ROCm, Q4_0 head) = n-max {knee}")
+# 2. Vulkan sweep around the knee (+ off)
+bench("sweep-vk-off", T["q106"], dev="Vulkan0")
+for n in sorted({max(1, knee - 1), knee, knee + 1}):
+    bench(f"sweep-vk-n{n}", T["q106"], dev="Vulkan0", draft=D4, nmax=n)
+# 3. draft head precision at the knee
+bench(f"head-q8-rocm-n{knee}", T["q106"], draft=D8, nmax=knee)
+# 4. every other tier, both backends, in the DEPLOYED config (MTP at the knee)
+for tag in ("q102", "q115", "q111"):
+    for dev, short in (("ROCm0", "rocm"), ("Vulkan0", "vk")):
+        bench(f"tier-{tag}-{short}-n{knee}", T[tag], dev=dev, draft=D4, nmax=knee)
+# 5. imatrix tier: speed should be identical by construction -- confirm, don't assume
+bench(f"tier-q106i-rocm-n{knee}", T["q106i"], draft=D4, nmax=knee)
+# 6. workload range for the headline config
+bench(f"prose-q106-rocm-n{knee}", T["q106"], draft=D4, nmax=knee, workload="prose")
+bench(f"prose-q106-vk-n{knee}", T["q106"], dev="Vulkan0", draft=D4, nmax=knee, workload="prose")
+json.dump({"knee": knee}, open(f"{W}/results/knee.json", "w"))
+log("BENCH_PLAN_DONE")

recipe/pipeline/cachegate3.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+"""Chunk-ALIGNED warm/cold identity gate for the MTP prompt-cache patch.
+
+Why a third gate: llama-server splits every prompt so that context checkpoints land (4 + n_ubatch) and 4 tokens
+before its end (tools/server/server-context.cpp `checkpoint_offsets`, upstream PR #20288). A warm request restores a
+checkpoint whose position was fixed by the length of the request that CREATED it. When those lengths differ, the warm
+tail is processed in different chunks than a cold run of the same prompt, so float rounding differs and a greedy
+token can flip - with or without a draft head. cachegate2 mixed prompt lengths and hit exactly that.
+
+Here every prompt in the run is padded to ONE token length L, so warm and cold see identical chunking. What is left
+under test is the patch's own responsibility: is the restored checkpoint (target + MTP state) exact?
+Per variant: warm-up A (cache on) -> B warm (cache on, must restore L-(4+ub)) -> B cold (cache off). PASS = every
+warm B reused the cache at the aligned position AND is byte-identical to its cold twin, over all variants."""
+import argparse, hashlib, json, os, sys
+sys.path.insert(0, "/mnt/models/agnes-3.0-flash")
+from agnes_harness import Server, post, prompt_8k  # noqa: E402  (reads AGNES_BIN at import)
+
+UB = 1024  # agnes_harness.Server passes -ub 1024
+WARM = "Summarise this file."
+INSTR = ["Write a function that lists every tensor name in this file.",
+         "Write a function that counts the model classes registered in this file.",
+         "Write a function that finds the longest method in this file.",
+         "Write a function that returns every regular expression used in this file.",
+         "Write a function that maps each class in this file to its base classes.",
+         "Write a function that extracts all string constants from this file.",
+         "Write a function that reports which imports in this file are unused.",
+         "Write a function that lists every method that raises an exception in this file.",
+         "Write a function that counts the lines of code per class in this file.",
+         "Write a function that finds duplicate method names across classes in this file."]
+KW = {"enable_thinking": False}
+
+
+def plen(port, content):
+    p = post(port, "/apply-template", {"messages": [{"role": "user", "content": content}],
+                                       "chat_template_kwargs": KW})["prompt"]
+    return len(post(port, "/tokenize", {"content": p, "add_special": True, "parse_special": True})["tokens"])
+
+
+def pad(port, ctx, instr, L):
+    s = instr
+    n = plen(port, ctx + "\n\n" + s)
+    for filler in (" ok", ".", " x"):
+        while n < L:
+            t = s + filler
+            m = plen(port, ctx + "\n\n" + t)
+            if m > L:
+                break
+            s, n = t, m
+        if n == L:
+            return s
+    raise SystemExit(f"could not pad {instr!r} to {L} (stuck at {n})")
+
+
+def req(port, ctx, instr, n, cache):
+    body = {"messages": [{"role": "user", "content": ctx + "\n\n" + instr}], "max_tokens": n,
+            "temperature": 0, "top_k": 1, "ignore_eos": True, "cache_prompt": cache, "chat_template_kwargs": KW}
+    r = post(port, "/v1/chat/completions", body)
+    tm = r.get("timings", {})
+    return (r["choices"][0]["message"].get("content") or ""), tm.get("prompt_n"), tm.get("cache_n"), \
+        tm.get("draft_n"), tm.get("draft_n_accepted")
+
+
+def first_diff(x, y):
+    if x == y:
+        return None
+    return next((i for i, (p, q) in enumerate(zip(x, y)) if p != q), min(len(x), len(y)))
+
+
+def main(a):
+    s = Server(a, a.port)
+    rows = []
+    try:
+        ctx, _ = prompt_8k("code")
+        instr = INSTR[:a.reps]
+        warm = [f"{WARM} (warm-up {i})" for i in range(len(instr))]
+        # L over the FULL prompt set, whatever --reps is: runs with different --reps then share byte-identical
+        # prompts, so their outputs can be compared across configs (e.g. MTP vs no draft head).
+        allp = INSTR + [f"{WARM} (warm-up {i})" for i in range(len(INSTR))]
+        L = max(plen(a.port, ctx + "\n\n" + t) for t in allp) + 1
+        instr = [pad(a.port, ctx, t, L) for t in instr]
+        warm = [pad(a.port, ctx, t, L) for t in warm]
+        expect_cache = L - (4 + UB)
+        for i, (wa, vb) in enumerate(zip(warm, instr)):
+            req(a.port, ctx, wa, 16, True)
+            tw, pw, cw, dw, aw = req(a.port, ctx, vb, 192, True)
+            tc, pc, cc, dc, ac = req(a.port, ctx, vb, 192, False)
+            rows.append({"variant": i, "L": L, "warm_prompt_n": pw, "warm_cache_n": cw, "cold_prompt_n": pc,
+                         "cold_cache_n": cc, "aligned": (cw == expect_cache and pc == L),
+                         "warm_draft": [aw, dw], "cold_draft": [ac, dc], "identical": tw == tc,
+                         "first_diff_char": first_diff(tw, tc),
+                         "warm_sha": hashlib.sha256(tw.encode()).hexdigest()[:12],
+                         "cold_sha": hashlib.sha256(tc.encode()).hexdigest()[:12], "instr": vb, "warm_instr": wa,
+                         "warm_text": tw, "cold_text": tc})
+            print("  ", json.dumps({k: v for k, v in rows[-1].items() if not k.endswith("_text")}), flush=True)
+    finally:
+        s.stop()
+    reused = all((r["warm_cache_n"] or 0) > 0 for r in rows)
+    aligned = all(r["aligned"] for r in rows)
+    ident = all(r["identical"] for r in rows)
+    res = {"label": a.label, "gate": "cachegate3-aligned", "n": len(rows), "L": rows[0]["L"] if rows else None,
+           "expect_cache_n": expect_cache if rows else None, "all_reused": reused, "all_aligned": aligned,
+           "identical": sum(r["identical"] for r in rows), "rows": rows,
+           "result": "PASS" if (rows and reused and aligned and ident) else "FAIL"}
+    print(json.dumps({k: v for k, v in res.items() if k != "rows"}), flush=True)
+    with open(a.jsonl, "a") as f:
+        f.write(json.dumps(res) + "\n")
+    sys.exit(0 if res["result"] == "PASS" else 1)
+
+
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--model", required=True); ap.add_argument("--draft")
+    ap.add_argument("--nmax", type=int, default=4); ap.add_argument("--pmin", type=float, default=0.0)
+    ap.add_argument("--dev", default="ROCm0"); ap.add_argument("--ctx", type=int, default=65536)
+    ap.add_argument("--port", type=int, default=18600); ap.add_argument("--reps", type=int, default=10)
+    ap.add_argument("--label", required=True); ap.add_argument("--jsonl", required=True)
+    ap.add_argument("--serverlog", required=True)
+    ap.add_argument("--mtp-infile", action="store_true"); ap.add_argument("--strict", action="store_true")
+    main(ap.parse_args())

recipe/pipeline/gpu_q1.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+# Phase B, GPU queue 1: BF16 perplexity + KLD base logits, then imatrix, then imatrix quants (CPU).
+set -euo pipefail
+W=/mnt/models/agnes-3.0-flash; B=/opt/llama-rocm/rocmfpx-724/build-hipvk/bin; N=Agnes-3.0-Flash-Preview
+export LD_LIBRARY_PATH=$B:/opt/rocm-7.2.4/lib HSA_OVERRIDE_GFX_VERSION=11.5.1 GGML_HIP_ENABLE_UNIFIED_MEMORY=1
+cd $W; mkdir -p kld imat out-imat logs results
+log(){ echo "[$(date -u +%FT%TZ)] $*"; }
+BF=gguf/$N-BF16.gguf; WT=calib/wikitext-2-raw/wiki.test.raw
+
+log "B1 BF16 perplexity (c=2048, 40 chunks) + save KLD base logits"
+$B/llama-perplexity -m $BF -f $WT -c 2048 --chunks 40 -b 2048 -ngl 999 -fa on -dio -dev ROCm0 \
+    --kl-divergence-base kld/bf16.kld > logs/B1_ppl_bf16.log 2>&1
+grep -E "Final estimate" logs/B1_ppl_bf16.log
+
+log "B2 imatrix on BF16 (bartowski calibration_datav3, c=512)"
+$B/llama-imatrix -m $BF -f calib/calibration_datav3.txt -o imat/$N.imatrix \
+    -ngl 999 -c 512 -b 512 -fa on -dio -dev ROCm0 > logs/B2_imatrix.log 2>&1
+ls -la imat/
+sha256sum imat/$N.imatrix | tee imat/$N.imatrix.sha256
+
+cap(){ systemd-run --scope --quiet -p MemoryMax=20G -p MemorySwapMax=0 "$@"; }
+log "B3 imatrix quant STRIX_LEAN (106)"
+cap $B/llama-quantize --imatrix imat/$N.imatrix --output-tensor-type q6_K $BF \
+    out-imat/$N-imatrix-Q4_0_ROCMFP4_STRIX_LEAN.gguf Q4_0_ROCMFP4_STRIX_LEAN 16 > logs/B3_imat_q106.log 2>&1
+python3 readback.py Q6_K Q5_K out-imat/$N-imatrix-Q4_0_ROCMFP4_STRIX_LEAN.gguf | tee -a logs/B_readback.log
+log "B4 imatrix quant COHERENT (102)"
+cap $B/llama-quantize --imatrix imat/$N.imatrix --output-tensor-type q6_K --token-embedding-type q6_K $BF \
+    out-imat/$N-imatrix-Q4_0_ROCMFP4_COHERENT.gguf Q4_0_ROCMFP4_COHERENT 16 > logs/B4_imat_q102.log 2>&1
+python3 readback.py Q6_K Q6_K out-imat/$N-imatrix-Q4_0_ROCMFP4_COHERENT.gguf | tee -a logs/B_readback.log
+log "GPU_Q1_DONE"

recipe/pipeline/gpu_q2.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+# Phase B, GPU queue 2: PPL + KL-divergence vs BF16 logits for every tier (waits for each read-back PASS).
+set -euo pipefail
+W=/mnt/models/agnes-3.0-flash; B=/opt/llama-rocm/rocmfpx-724/build-hipvk/bin; N=Agnes-3.0-Flash-Preview
+export LD_LIBRARY_PATH=$B:/opt/rocm-7.2.4/lib HSA_OVERRIDE_GFX_VERSION=11.5.1 GGML_HIP_ENABLE_UNIFIED_MEMORY=1
+cd $W
+log(){ echo "[$(date -u +%FT%TZ)] $*"; }
+log "waiting for GPU (imatrix finished)"
+until grep -q "B3 imatrix quant" logs/gpu_q1.log 2>/dev/null; do
+  pgrep -f "bash gpu_q1.sh" >/dev/null || { log "gpu_q1 died before imatrix finished"; exit 1; }; sleep 20; done
+waitpass(){  # $1 = file basename, $2 = readback log
+  until grep -q "^PASS $1 " "$2" 2>/dev/null; do
+    grep -q "^FAIL $1 " "$2" 2>/dev/null && { log "readback FAIL for $1"; exit 1; }; sleep 20; done; }
+kld(){  # $1 tag  $2 path  $3 readback log
+  waitpass "$(basename $2)" "$3"
+  log "KLD $1"
+  $B/llama-perplexity -m "$2" --kl-divergence-base kld/bf16.kld --kl-divergence -c 2048 -b 2048 \
+      -ngl 999 -fa on -dio -dev ROCm0 > logs/B5_kld_$1.log 2>&1
+  grep -E "Mean PPL\(Q\)|Mean KLD|Same top p" logs/B5_kld_$1.log | tr -s " " | tr "\n" " "; echo
+}
+kld q106  out/$N-Q4_0_ROCMFP4_STRIX_LEAN.gguf          logs/A7_readback.log
+kld q102  out/$N-Q4_0_ROCMFP4_COHERENT.gguf            logs/A7_readback.log
+kld q115  out/$N-Q8_0_ROCMFPX_AGENT.gguf               logs/A7_readback.log
+kld q111  out/$N-Q8_0_ROCMFPX.gguf                     logs/A7_readback.log
+kld q106i out-imat/$N-imatrix-Q4_0_ROCMFP4_STRIX_LEAN.gguf logs/B_readback.log
+kld q102i out-imat/$N-imatrix-Q4_0_ROCMFP4_COHERENT.gguf logs/B_readback.log
+log "GPU_Q2_DONE"

recipe/pipeline/gpu_q4.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# Phase B, queue 4: correctness gates on the recommended tier in its DEPLOYED config.
+set -uo pipefail
+W=/mnt/models/agnes-3.0-flash; N=Agnes-3.0-Flash-Preview; cd $W
+H="python3 agnes_harness.py"; M=out/$N-Q4_0_ROCMFP4_STRIX_LEAN.gguf; D=out/mtp-$N-Q4_0.gguf
+KNEE=$(python3 -c "import json;print(json.load(open('results/knee.json'))['knee'])")
+log(){ echo "[$(date -u +%FT%TZ)] $*"; }
+log "G1 prompt-cache gate WITH MTP draft (n-max $KNEE)"
+$H cachegate --model $M --draft $D --nmax $KNEE --label cache-mtp --serverlog logs/srv_cache_mtp.log; echo "  exit=$?"
+log "G2 prompt-cache control, no draft"
+$H cachegate --model $M --label cache-nodraft --serverlog logs/srv_cache_nodraft.log; echo "  exit=$?"
+log "G3 tool-call suite, deployed config (MTP on), thinking on + off"
+$H tools --model $M --draft $D --nmax $KNEE --label tools-q106-mtp --serverlog logs/srv_tools.log; echo "  exit=$?"
+log "G4 vision (mmproj, -fa off)"
+$H vision --model $M --mmproj out/mmproj-$N-BF16.gguf --image calib/vision_probe.png \
+  --question "Describe the two shapes in this image and their colors." \
+  --expect "red,blue,circle,square" --label vision-q106 --serverlog logs/srv_vision.log; echo "  exit=$?"
+log "GPU_Q4_DONE"

recipe/pipeline/hf_publish_v2.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+"""Stage (hardlinks), upload, and byte-verify the two Agnes repos - in-file-MTP layout.
+usage: hf_publish_v2.py stage|upload|verify|readme <std|imat> [README path]
+Run upload under a memory cap, e.g.  systemd-run --scope -p MemoryMax=8G -p MemorySwapMax=0 python3 hf_publish_v2.py upload std
+Env: default HF_HOME (token); HF_XET_CACHE / TMPDIR pinned to /mnt/models by the caller."""
+import hashlib, json, os, shutil, sys, urllib.request
+
+W = "/mnt/models/agnes-3.0-flash"; N = "Agnes-3.0-Flash-Preview"; U = "kingjones777"
+REPOS = {"std": f"{U}/{N}-MTP-ROCmFP4-GGUF", "imat": f"{U}/{N}-MTP-ROCmFP4-imatrix-GGUF"}
+
+
+def pubname(fn):
+    """Published filename: the quant token must be hyphen-delimited or the Hub cannot parse the variant
+    (`Q4_0_ROCMFP4_STRIX_LEAN` reads as the garbage label `Q4_0_ROCMFP`)."""
+    for a, b in (("Q4_0_ROCMFP4_", "Q4_0-ROCmFP4-"), ("Q8_0_ROCMFPX_AGENT", "Q8_0-ROCmFPX-AGENT"), ("Q8_0_ROCMFPX", "Q8_0-ROCmFPX")):
+        if a in fn:
+            return fn.replace(a, b)
+    return fn
+
+
+# published name -> (source dir, local filename)
+MMPROJ = {f"mmproj-{N}-BF16.gguf": (f"{W}/out", f"mmproj-{N}-BF16.gguf")}
+FILES = {"std": {**{pubname(f"{N}-MTP-{t}.gguf"): (f"{W}/out-mtp", f"{N}-MTP-{t}.gguf") for t in (
+                    "Q4_0_ROCMFP4_STRIX_LEAN", "Q4_0_ROCMFP4_COHERENT", "Q8_0_ROCMFPX_AGENT", "Q8_0_ROCMFPX")}, **MMPROJ},
+         "imat": {**{pubname(f"{N}-MTP-imatrix-{t}.gguf"): (f"{W}/out-mtp-imat", f"{N}-MTP-imatrix-{t}.gguf") for t in (
+                    "Q4_0_ROCMFP4_STRIX_LEAN", "Q4_0_ROCMFP4_COHERENT")},
+                  f"{N}.imatrix": (f"{W}/imat", f"{N}.imatrix"), **MMPROJ}}
+P = "recipe/pipeline/"
+RECIPE = {
+    "recipe/fold_agnes.py": "fold_agnes.py", "recipe/verify_fold.py": "verify_fold.py",
+    "recipe/FOLD_PROVENANCE.json": "qwen35/FOLD_PROVENANCE.json", "recipe/verify_fold.log": "logs/A3_verify_fold.log",
+    "recipe/results_summary.json": "results/summary_v2.json",
+    "recipe/patches/mtp-prompt-cache-fix.patch": "mtpfix/mtp-prompt-cache-fix.patch",
+    **{P + x: x for x in ("phase_a.sh", "gpu_q1.sh", "gpu_q2.sh", "gpu_q4.sh", "bench_plan.py", "agnes_harness.py",
+                          "readback.py", "verify_download.py", "seat_sizing.sh", "mtp_bake.sh", "mtpcache_build.sh",
+                          "tensor_identity.py", "trunk_receipt.sh", "vplan_a.py", "vplan_b.py", "cachegate3.py",
+                          "seat_sizing2.sh", "aggregate_v2.py")},
+    "recipe/raw/mtpfix.jsonl": "results/mtpfix.jsonl",
+    "recipe/raw/trunk_receipt.jsonl": "results/trunk_receipt.jsonl",
+    "recipe/raw/seat_sizing2.jsonl": "results/seat_sizing2.jsonl",
+    "recipe/raw/separate_head_phase_b.jsonl": "results/phase_b.jsonl",
+    "recipe/raw/separate_head_seat_sizing.jsonl": "results/seat_sizing.jsonl",
+    "recipe/logs/S_readback.log": "logs/S_readback.log", "recipe/logs/S_identity.log": "logs/S_identity.log",
+    "recipe/logs/S_sizes.log": "logs/S_sizes.log", "recipe/logs/P1_apply_check.log": "logs/P1_apply_check.log",
+    "recipe/logs/mtpcache_build.log": "logs/mtpcache_build.log",
+    "recipe/raw/regrade.jsonl": "results/regrade.jsonl", "recipe/raw/kld_control.jsonl": "results/kld_control.jsonl",
+    "recipe/logs/R2_ppl_bf16.log": "logs/R2_ppl_bf16.log", "recipe/logs/R4_kld_q106_repeat.log": "logs/R4_kld_q106_repeat.log",
+    "recipe/logs/R1_kld4_old_q106_cpuburn.log": "logs/R1_kld4_old_q106_cpuburn.log",
+    **{f"recipe/logs/R3_kld_{t}.log": f"logs/R3_kld_{t}.log" for t in ("q106", "q102", "q106i", "q102i", "q115", "q111")},
+    **{f"recipe/logs/afternoon_B5_kld_{t}.log": f"logs/B5_kld_{t}.log" for t in ("q106", "q102", "q106i", "q102i", "q115", "q111")},
+    "recipe/logs/afternoon_B1_ppl_bf16.log": "logs/B1_ppl_bf16.log",
+    **{P + x: x for x in ("regrade.sh", "kld_control.sh", "judge_v2.py", "render_cards_v2.py", "hf_publish_v2.py")}}
+if True:  # the imatrix card's methodology cites the imatrix run
+    RECIPE["recipe/logs/B2_imatrix.log"] = "logs/B2_imatrix.log"
+
+
+def separation_errors(kind, names):
+    """The imatrix build is its OWN repo (King): no imatrix weights in the standard repo, no standard weights in the
+    imatrix repo. The vision projector is the only model file both carry."""
+    errs = []
+    for fn in names:
+        if not fn.endswith(".gguf") or fn.startswith("mmproj-"):
+            continue
+        is_imat = "-imatrix-" in fn
+        if kind == "std" and is_imat: errs.append(f"imatrix model file in the standard repo: {fn}")
+        if kind == "imat" and not is_imat: errs.append(f"standard model file in the imatrix repo: {fn}")
+    if kind == "imat" and f"{N}.imatrix" not in names: errs.append("imatrix repo is missing the .imatrix file")
+    return errs
+
+
+assert REPOS["std"] != REPOS["imat"]
+assert set(FILES["std"]) & set(FILES["imat"]) == set(MMPROJ), "only the vision projector may be shared"
+for _k in FILES:
+    assert not separation_errors(_k, list(FILES[_k])), separation_errors(_k, list(FILES[_k]))
+
+
+def sha(p):
+    h = hashlib.sha256()
+    with open(p, "rb") as fh:
+        for b in iter(lambda: fh.read(64 << 20), b""): h.update(b)
+    return h.hexdigest()
+
+
+kind = sys.argv[2]; stage = f"{W}/hf-upload-v2/{kind}"; repo = REPOS[kind]
+if sys.argv[1] == "stage":
+    missing = [f"{d}/{src}" for fn, (d, src) in FILES[kind].items() if not os.path.exists(f"{d}/{src}")] + \
+              [src for src in RECIPE.values() if not os.path.exists(f"{W}/{src}")]
+    if missing:
+        raise SystemExit(f"STAGE ABORT - missing: {missing}")
+    shutil.rmtree(stage, ignore_errors=True)
+    for fn, (d, src) in FILES[kind].items():
+        os.makedirs(stage, exist_ok=True); os.link(f"{d}/{src}", f"{stage}/{fn}")
+    for dst, src in RECIPE.items():
+        os.makedirs(os.path.dirname(f"{stage}/{dst}"), exist_ok=True); shutil.copy2(f"{W}/{src}", f"{stage}/{dst}")
+    sums = {fn: sha(f"{stage}/{fn}") for fn in sorted(FILES[kind])}
+    with open(f"{stage}/SHA256SUMS", "w") as fh:
+        for fn, h in sums.items(): fh.write(f"{h}  {fn}\n")
+    sizes = {x: os.path.getsize(f"{stage}/{x}") for x in sums}
+    json.dump(sizes, open(f"{stage}/../staged_sizes_{kind}.json", "w"))
+    print(json.dumps({"stage": stage, "files": len(sums), "bytes": sum(sizes.values()), "recipe_files": len(RECIPE)}))
+elif sys.argv[1] == "upload":
+    from huggingface_hub import HfApi
+    api = HfApi()
+    print("whoami:", api.whoami()["name"], flush=True)
+    api.create_repo(repo, repo_type="model", private=False, exist_ok=True)
+    # model files + receipts first; README goes up only after verify passes
+    api.upload_large_folder(repo_id=repo, repo_type="model", folder_path=stage,
+                            ignore_patterns=["README.md", ".cache/**"], num_workers=2)
+    print("UPLOAD_DONE", repo, flush=True)
+elif sys.argv[1] == "verify":
+    tree = json.load(urllib.request.urlopen(f"https://huggingface.co/api/models/{repo}/tree/main?recursive=true", timeout=60))
+    remote = {t["path"]: t for t in tree if t.get("type") == "file"}
+    local = dict(l.split()[::-1] for l in open(f"{stage}/SHA256SUMS").read().splitlines())
+    bad = []
+    for fn, h in local.items():
+        r = remote.get(fn)
+        if not r: bad.append((fn, "MISSING remote")); continue
+        if r["size"] != os.path.getsize(f"{stage}/{fn}"): bad.append((fn, f"size {r['size']}")); continue
+        oid = (r.get("lfs") or {}).get("oid")
+        if oid != h: bad.append((fn, f"sha {oid} != {h[:12]}"))
+    for fn in list(RECIPE) + ["SHA256SUMS"]:
+        if fn not in remote: bad.append((fn, "MISSING remote"))
+    stale = sorted(x for x in remote if x not in local and x not in RECIPE and x not in ("SHA256SUMS", "README.md", ".gitattributes"))
+    bad += [(x, "SEPARATION") for x in separation_errors(kind, list(remote))]
+    print(json.dumps({"repo": repo, "checked": len(local), "bad": bad, "unexpected_remote_files": stale,
+                      "result": "PASS" if not bad and not stale else "FAIL"}))
+    sys.exit(0 if not bad and not stale else 1)
+elif sys.argv[1] == "readme":
+    from huggingface_hub import HfApi
+    HfApi().upload_file(path_or_fileobj=sys.argv[3], path_in_repo="README.md", repo_id=repo, repo_type="model",
+                        commit_message="Model card: measured results, reproduction, known issues")
+    print("README_DONE", repo)

recipe/pipeline/kld_control.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+# The 4-bit in-file-MTP files did not reproduce the graded per-chunk rows (Q8 AGENT did).
+# C1: determinism control - re-run the grading command on the ORIGINAL graded 4-bit files; do they reproduce themselves?
+# C2: full 40-chunk grade of the shipped in-file STRIX_LEAN file, to measure the real size of any difference.
+set -uo pipefail
+W=/mnt/models/agnes-3.0-flash; B=/opt/llama-rocm/rocmfpx-724/build-hipvk/bin; N=Agnes-3.0-Flash-Preview; cd $W
+export LD_LIBRARY_PATH=$B:/opt/rocm-7.2.4/lib HSA_OVERRIDE_GFX_VERSION=11.5.1 GGML_HIP_ENABLE_UNIFIED_MEMORY=1
+log(){ echo "[$(date -u +%FT%TZ)] $*"; }
+rows(){
+  python3 - "$1" <<'EOF'
+import sys
+out = []
+for l in open(sys.argv[1], errors="replace"):
+    t = l.split()
+    if len(t) > 2 and t[0].isdigit() and l.rstrip().endswith("%") and l.count("%") == 2:
+        out.append(" ".join(t))
+    if len(out) == 4:
+        break
+print("\n".join(out))
+EOF
+}
+until grep -q '"tag":"q102i"' results/trunk_receipt.jsonl 2>/dev/null; do sleep 15; done
+log "receipts done; starting controls"
+ctl(){  # $1 label  $2 file  $3 reference log
+  local L=logs/K_$1.log
+  $B/llama-perplexity -m "$2" --kl-divergence-base kld/bf16.kld --kl-divergence -c 2048 -b 2048 \
+      -ngl 999 -fa on -dio -dev ROCm0 > $L 2>&1 &
+  local pid=$!
+  for i in $(seq 1 900); do [ "$(rows $L | grep -c .)" -ge 4 ] && break; kill -0 $pid 2>/dev/null || break; sleep 1; done
+  kill $pid 2>/dev/null; wait $pid 2>/dev/null
+  local a b v
+  a=$(rows $L); b=$(rows $3)
+  if [ "$(printf '%s\n' "$a" | grep -c .)" -eq 4 ] && [ "$a" = "$b" ]; then v=MATCH; else v=MISMATCH; fi
+  echo "{\"label\":\"$1\",\"file\":\"$(basename "$2")\",\"reference\":\"$(basename "$3")\",\"result\":\"$v\"}" | tee -a results/kld_control.jsonl
+}
+ctl c1-old-q106 out/$N-Q4_0_ROCMFP4_STRIX_LEAN.gguf logs/B5_kld_q106.log
+ctl c1-old-q102 out/$N-Q4_0_ROCMFP4_COHERENT.gguf  logs/B5_kld_q102.log
+ctl c1-new-q106-again out-mtp/$N-MTP-Q4_0_ROCMFP4_STRIX_LEAN.gguf logs/R_kld4_q106.log
+log "C2 full grade of the shipped in-file STRIX_LEAN"
+$B/llama-perplexity -m out-mtp/$N-MTP-Q4_0_ROCMFP4_STRIX_LEAN.gguf --kl-divergence-base kld/bf16.kld --kl-divergence \
+    -c 2048 -b 2048 -ngl 999 -fa on -dio -dev ROCm0 > logs/K_full_new_q106.log 2>&1
+log "C2 exit=$?"
+grep -E "Mean PPL\(Q\)|Mean +KLD|Same top p|99.0%" logs/K_full_new_q106.log
+echo "--- graded (old file) ---"; grep -E "Mean PPL\(Q\)|Mean +KLD|Same top p|99.0%" logs/B5_kld_q106.log
+log "KLD_CONTROL_DONE"

recipe/pipeline/mtp_bake.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+# Shrink: bake the MTP layer INTO each tier (shares the trunk's embeddings/output) instead of a separate 1.9 GiB head.
+set -euo pipefail
+W=/mnt/models/agnes-3.0-flash; T=/opt/llama-rocm/rocmfpx-724; B=$T/build-hipvk/bin; N=Agnes-3.0-Flash-Preview
+export LD_LIBRARY_PATH=$B:/opt/rocm-7.2.4/lib TMPDIR=/mnt/models/.tmp PYTHONUNBUFFERED=1
+cd $W; mkdir -p out-mtp out-mtp-imat logs
+log(){ echo "[$(date -u +%FT%TZ)] $*"; }
+cap(){ systemd-run --scope --quiet -p MemoryMax=24G -p MemorySwapMax=0 "$@"; }
+log "S1 convert trunk WITH MTP layer (converter default)"
+cap python3 $T/convert_hf_to_gguf.py qwen35 --outtype bf16 --model-name "$N" --outfile gguf/$N-MTP-BF16.gguf > logs/S1_conv_mtp_trunk.log 2>&1
+python3 readback.py - - gguf/$N-MTP-BF16.gguf | tee logs/S_readback.log
+BF=gguf/$N-MTP-BF16.gguf; Q="cap $B/llama-quantize"
+log "S2 standard tiers"
+$Q --output-tensor-type q6_K $BF out-mtp/$N-MTP-Q4_0_ROCMFP4_STRIX_LEAN.gguf Q4_0_ROCMFP4_STRIX_LEAN 16 > logs/S2_q106.log 2>&1
+$Q --output-tensor-type q6_K --token-embedding-type q6_K $BF out-mtp/$N-MTP-Q4_0_ROCMFP4_COHERENT.gguf Q4_0_ROCMFP4_COHERENT 16 > logs/S2_q102.log 2>&1
+$Q --output-tensor-type q8_0 --token-embedding-type q8_0 $BF out-mtp/$N-MTP-Q8_0_ROCMFPX_AGENT.gguf Q8_0_ROCMFPX_AGENT 16 > logs/S2_q115.log 2>&1
+$Q --output-tensor-type q8_0 --token-embedding-type q8_0 $BF out-mtp/$N-MTP-Q8_0_ROCMFPX.gguf Q8_0_ROCMFPX 16 > logs/S2_q111.log 2>&1
+log "S3 imatrix tiers (MTP layer has no imatrix data: it never runs in the forward pass -> unweighted path for blk.72 only)"
+$Q --imatrix imat/$N.imatrix --output-tensor-type q6_K $BF out-mtp-imat/$N-MTP-imatrix-Q4_0_ROCMFP4_STRIX_LEAN.gguf Q4_0_ROCMFP4_STRIX_LEAN 16 > logs/S3_q106i.log 2>&1
+$Q --imatrix imat/$N.imatrix --output-tensor-type q6_K --token-embedding-type q6_K $BF out-mtp-imat/$N-MTP-imatrix-Q4_0_ROCMFP4_COHERENT.gguf Q4_0_ROCMFP4_COHERENT 16 > logs/S3_q102i.log 2>&1
+log "S4 read-back (head protection) + trunk identity vs the graded builds"
+for pair in "Q6_K Q5_K out-mtp/$N-MTP-Q4_0_ROCMFP4_STRIX_LEAN.gguf out/$N-Q4_0_ROCMFP4_STRIX_LEAN.gguf" \
+            "Q6_K Q6_K out-mtp/$N-MTP-Q4_0_ROCMFP4_COHERENT.gguf out/$N-Q4_0_ROCMFP4_COHERENT.gguf" \
+            "Q8_0 Q8_0 out-mtp/$N-MTP-Q8_0_ROCMFPX_AGENT.gguf out/$N-Q8_0_ROCMFPX_AGENT.gguf" \
+            "Q8_0 Q8_0 out-mtp/$N-MTP-Q8_0_ROCMFPX.gguf out/$N-Q8_0_ROCMFPX.gguf" \
+            "Q6_K Q5_K out-mtp-imat/$N-MTP-imatrix-Q4_0_ROCMFP4_STRIX_LEAN.gguf out-imat/$N-imatrix-Q4_0_ROCMFP4_STRIX_LEAN.gguf" \
+            "Q6_K Q6_K out-mtp-imat/$N-MTP-imatrix-Q4_0_ROCMFP4_COHERENT.gguf out-imat/$N-imatrix-Q4_0_ROCMFP4_COHERENT.gguf"; do
+  set -- $pair
+  python3 readback.py $1 $2 $3 | tee -a logs/S_readback.log
+  echo "$(basename $3) $(python3 tensor_identity.py $4 $3 || true)" | tee -a logs/S_identity.log
+done
+for l in S2_q106 S2_q102 S2_q115 S2_q111 S3_q106i S3_q102i; do printf "%-10s " $l; grep -oE "quant size\s*=\s*[0-9.]+ MiB \([0-9.]+ BPW\)" logs/$l.log; done | tee -a logs/S_sizes.log
+log "SHRINK_DONE"

recipe/pipeline/mtpcache_build.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+# Cache fix: build a PATCHED llama-server from d3ca537 in a SEPARATE clone. The live tree is never touched.
+set -uo pipefail
+SRC=/opt/llama-rocm/rocmfpx-724
+NEW=/opt/llama-rocm/rocmfpx-724-mtpcache
+BASE=d3ca53726109b46fd5cd6fa42f688c707079b44e
+W=/mnt/models/agnes-3.0-flash; PATCH=$W/mtpfix/mtp-prompt-cache-fix.patch; L=$W/logs
+log(){ echo "[$(date -u +%FT%TZ)] $*"; }
+[ "$(git -C $SRC rev-parse HEAD)" = "$BASE" ] || { log "live tree is not at $BASE"; exit 1; }
+log "P0 separate clone at $BASE (hardlinked objects; live tree untouched)"
+if [ ! -d $NEW/.git ]; then git clone -q --no-checkout $SRC $NEW && git -C $NEW checkout -q $BASE; fi
+[ "$(git -C $NEW rev-parse HEAD)" = "$BASE" ] || { log "clone not at base"; exit 1; }
+[ -z "$(git -C $NEW status --porcelain)" ] || { log "clone not clean before patching"; exit 1; }
+log "P1 patch applicability"
+if git -C $NEW apply --check -v $PATCH > $L/P1_apply_check.log 2>&1; then
+  git -C $NEW apply $PATCH && log "  applied CLEANLY"
+elif patch -d $NEW -p1 --dry-run --fuzz=2 < $PATCH > $L/P1_patch_fuzz.log 2>&1; then
+  patch -d $NEW -p1 --fuzz=2 < $PATCH > $L/P1_patch_apply.log 2>&1 && log "  applied WITH FUZZ (see P1_patch_apply.log)"
+  grep -iE "fuzz|offset" $L/P1_patch_apply.log | head
+else
+  log "  PATCH DOES NOT APPLY -> needs a manual port (P1 logs)"; echo PORT_NEEDED; exit 3
+fi
+git -C $NEW diff --stat | tail -6
+( cd $NEW && git diff ) > $L/P1_applied.diff
+sha256sum $PATCH | tee $L/P1_patch.sha256
+log "P2 configure from the ORIGINAL build's CMakeCache (same flags)"
+IC=$L/P2_initial_cache.cmake; : > $IC
+grep -E '^(GGML_[A-Z0-9_]+|LLAMA_[A-Z0-9_]+|AMDGPU_TARGETS|GPU_TARGETS|CMAKE_BUILD_TYPE|CMAKE_HIP_COMPILER|CMAKE_C_COMPILER|CMAKE_CXX_COMPILER|CMAKE_HIP_ARCHITECTURES):[A-Z]+=' \
+  $SRC/build-hipvk/CMakeCache.txt | grep -v ':INTERNAL=' | while IFS= read -r line; do
+    key=${line%%:*}; rest=${line#*:}; typ=${rest%%=*}; val=${rest#*=}
+    [ "$typ" = "UNINITIALIZED" ] && typ=STRING
+    printf 'set(%s "%s" CACHE %s "")\n' "$key" "$val" "$typ" >> $IC
+  done
+wc -l < $IC | sed 's/^/  cache entries: /'
+cmake -S $NEW -B $NEW/build-hipvk -G "Unix Makefiles" -DCMAKE_MAKE_PROGRAM=/usr/bin/gmake -C $IC > $L/P2_configure.log 2>&1 \
+  || { log "configure FAILED"; tail -20 $L/P2_configure.log; exit 4; }
+log "P3 build llama-server (+ deps)"
+cmake --build $NEW/build-hipvk --target llama-server -j "${JOBS:-28}" > $L/P3_build.log 2>&1 || { log "build FAILED"; tail -25 $L/P3_build.log; exit 5; }
+BIN=$NEW/build-hipvk/bin
+LD_LIBRARY_PATH=$BIN:/opt/rocm-7.2.4/lib $BIN/llama-server --version 2>&1 | tail -3
+LD_LIBRARY_PATH=$BIN:/opt/rocm-7.2.4/lib HSA_OVERRIDE_GFX_VERSION=11.5.1 $BIN/llama-server --list-devices 2>&1 | grep -E "ROCm0|Vulkan0"
+sha256sum $BIN/llama-server | tee $L/P3_server.sha256
+log "MTPCACHE_BUILD_DONE"

recipe/pipeline/phase_a.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+# Agnes-3.0-Flash Preview -- Phase A (no GPU, no seat disruption). Every stage is gated.
+set -euo pipefail
+W=/mnt/models/agnes-3.0-flash
+T=/opt/llama-rocm/rocmfpx-724
+B=$T/build-hipvk/bin
+REV=891ce4f9ffb89b22888aa7fcc2bb2f3618867684
+N=Agnes-3.0-Flash-Preview
+export LD_LIBRARY_PATH=$B:/opt/rocm-7.2.4/lib TMPDIR=/mnt/models/.tmp PYTHONUNBUFFERED=1
+cd "$W"; mkdir -p gguf out logs
+log(){ echo "[$(date -u +%FT%TZ)] $*"; }
+cap(){ systemd-run --scope --quiet -p MemoryMax=20G -p MemorySwapMax=0 "$@"; }
+
+log "A0 wait for download"
+until grep -q "DONE_RC=" dl.log; do sleep 30; done
+grep -q "DONE_RC=0" dl.log || { log "A0 FAIL download"; exit 1; }
+
+log "A1 verify download (size + lfs sha256)"
+python3 verify_download.py Agnes-AI/Agnes-3.0-Flash "$REV" hf > logs/A1_verify_download.log 2>&1
+tail -2 logs/A1_verify_download.log
+
+log "A2 fold -> qwen35"
+python3 fold_agnes.py hf qwen35 > logs/A2_fold.log 2>&1
+tail -1 logs/A2_fold.log | cut -c1-300
+
+log "A3 verify fold (bit-exact + forward equivalence)"
+python3 verify_fold.py hf qwen35 > logs/A3_verify_fold.log 2>&1
+grep -E "^G[0-9]|RESULT" logs/A3_verify_fold.log
+
+log "A4 convert trunk BF16 (no MTP)"
+cap python3 $T/convert_hf_to_gguf.py qwen35 --outtype bf16 --no-mtp --model-name "$N" \
+    --outfile gguf/$N-BF16.gguf > logs/A4_conv_trunk.log 2>&1
+log "A5 convert MTP head BF16"
+cap python3 $T/convert_hf_to_gguf.py qwen35 --outtype bf16 --mtp --model-name "$N" \
+    --outfile gguf/mtp-$N-BF16.gguf > logs/A5_conv_mtp.log 2>&1
+log "A6 convert vision mmproj BF16"
+cap python3 $T/convert_hf_to_gguf.py qwen35 --outtype bf16 --mmproj --model-name "$N" \
+    --outfile gguf/mmproj-$N-BF16.gguf > logs/A6_conv_mmproj.log 2>&1
+ls -la gguf/
+
+log "A7 read-back BF16 artifacts"
+python3 readback.py - - gguf/$N-BF16.gguf        | tee -a logs/A7_readback.log
+python3 readback.py - - gguf/mtp-$N-BF16.gguf    | tee -a logs/A7_readback.log
+
+BF=gguf/$N-BF16.gguf
+q(){ cap $B/llama-quantize "$@"; }
+log "A8 quantize STRIX_LEAN (106, head q6_K, embd stays Q5_K)"
+q --output-tensor-type q6_K $BF out/$N-Q4_0_ROCMFP4_STRIX_LEAN.gguf Q4_0_ROCMFP4_STRIX_LEAN 16 > logs/A8_q106.log 2>&1
+python3 readback.py Q6_K Q5_K out/$N-Q4_0_ROCMFP4_STRIX_LEAN.gguf | tee -a logs/A7_readback.log
+log "A9 quantize COHERENT (102, head+embd q6_K)"
+q --output-tensor-type q6_K --token-embedding-type q6_K $BF out/$N-Q4_0_ROCMFP4_COHERENT.gguf Q4_0_ROCMFP4_COHERENT 16 > logs/A9_q102.log 2>&1
+python3 readback.py Q6_K Q6_K out/$N-Q4_0_ROCMFP4_COHERENT.gguf | tee -a logs/A7_readback.log
+log "A10 quantize Q8 AGENT (115, q8_0 head+embd)"
+q --output-tensor-type q8_0 --token-embedding-type q8_0 $BF out/$N-Q8_0_ROCMFPX_AGENT.gguf Q8_0_ROCMFPX_AGENT 16 > logs/A10_q115.log 2>&1
+python3 readback.py Q8_0 Q8_0 out/$N-Q8_0_ROCMFPX_AGENT.gguf | tee -a logs/A7_readback.log
+log "A11 quantize Q8 plain (111, q8_0 head+embd)"
+q --output-tensor-type q8_0 --token-embedding-type q8_0 $BF out/$N-Q8_0_ROCMFPX.gguf Q8_0_ROCMFPX 16 > logs/A11_q111.log 2>&1
+python3 readback.py Q8_0 Q8_0 out/$N-Q8_0_ROCMFPX.gguf | tee -a logs/A7_readback.log
+log "A12 MTP draft heads (Q4_0 beats Q8_0 on this family -- ship both)"
+q gguf/mtp-$N-BF16.gguf out/mtp-$N-Q4_0.gguf Q4_0 16 > logs/A12_mtp_q4.log 2>&1
+q gguf/mtp-$N-BF16.gguf out/mtp-$N-Q8_0.gguf Q8_0 16 > logs/A12_mtp_q8.log 2>&1
+python3 readback.py - - out/mtp-$N-Q4_0.gguf | tee -a logs/A7_readback.log
+python3 readback.py - - out/mtp-$N-Q8_0.gguf | tee -a logs/A7_readback.log
+cp gguf/mmproj-$N-BF16.gguf out/
+( cd out && sha256sum *.gguf > SHA256SUMS )
+ls -la out/
+log "PHASE_A_DONE"

recipe/pipeline/readback.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+"""Exact-name read-back gate: the built file is the receipt, not the ftype name.
+usage: readback.py <expect_output> <expect_embd> <file.gguf>   (use '-' to skip a check)"""
+import sys
+sys.path.insert(0, "/opt/llama-rocm/rocmfpx-724/gguf-py")
+import gguf
+exp_out, exp_emb, path = sys.argv[1:4]
+r = gguf.GGUFReader(path)
+def tname(t):
+    try: return t.tensor_type.name
+    except Exception: return f"TYPE_{int(t.tensor_type)}"
+ty = {t.name: tname(t) for t in r.tensors}
+f = r.fields.get("general.file_type")
+ftype = int(f.parts[f.data[0]][0]) if f else None
+arch = bytes(r.fields["general.architecture"].parts[-1]).decode()
+nextn = sum(".nextn." in k for k in ty)
+out, emb = ty.get("output.weight"), ty.get("token_embd.weight")
+ok = (exp_out == "-" or out == exp_out) and (exp_emb == "-" or emb == exp_emb)
+print(f"{'PASS' if ok else 'FAIL'} {path.split('/')[-1]} arch={arch} ftype={ftype} tensors={len(ty)} nextn={nextn} "
+      f"output.weight={out} token_embd.weight={emb}")
+sys.exit(0 if ok else 1)

recipe/pipeline/regrade.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+# Re-grade on the SHIPPED files in the current machine state. The afternoon (pre-reboot) 4-bit grades do not reproduce
+# today, while today's runs reproduce each other exactly and the in-file files compute identically to the trunk-only
+# files. R1 tests whether concurrent CPU load (present during the afternoon grading) changes 4-bit results.
+set -uo pipefail
+W=/mnt/models/agnes-3.0-flash; B=/opt/llama-rocm/rocmfpx-724/build-hipvk/bin; N=Agnes-3.0-Flash-Preview; cd $W
+export LD_LIBRARY_PATH=$B:/opt/rocm-7.2.4/lib HSA_OVERRIDE_GFX_VERSION=11.5.1 GGML_HIP_ENABLE_UNIFIED_MEMORY=1
+log(){ echo "[$(date -u +%FT%TZ)] $*"; }
+J=results/regrade.jsonl
+table(){  # all complete per-chunk rows of a KLD log (limit $2)
+  python3 - "$1" "${2:-999}" <<'EOF'
+import sys
+out = []
+for l in open(sys.argv[1], errors="replace"):
+    t = l.split()
+    if len(t) > 2 and t[0].isdigit() and l.rstrip().endswith("%") and l.count("%") == 2:
+        out.append(" ".join(t))
+    if len(out) == int(sys.argv[2]):
+        break
+print("\n".join(out))
+EOF
+}
+until grep -q KLD_CONTROL_DONE logs/kld_control.log 2>/dev/null && grep -q TRUNK_RECEIPT_DONE logs/trunk_receipt.log 2>/dev/null; do sleep 20; done
+log "GPU free of the verification jobs"
+PPL="$B/llama-perplexity -c 2048 -b 2048 -ngl 999 -fa on -dio -dev ROCm0"
+
+log "R1 old q106, first 4 rows, under a 16-thread CPU burn (the afternoon grading ran next to CPU quantization)"
+mkdir -p /mnt/models/.tmp
+( exec $B/llama-quantize --output-tensor-type q6_K gguf/$N-BF16.gguf /mnt/models/.tmp/r1_burn.gguf Q4_0_ROCMFP4_STRIX_LEAN 16 ) > logs/R1_burn.log 2>&1 &
+bp=$!; sleep 30
+L=logs/R1_kld4_old_q106_cpuburn.log
+$PPL -m out/$N-Q4_0_ROCMFP4_STRIX_LEAN.gguf --kl-divergence-base kld/bf16.kld --kl-divergence > $L 2>&1 &
+kp=$!
+for i in $(seq 1 900); do [ "$(table $L 4 | grep -c .)" -ge 4 ] && break; kill -0 $kp 2>/dev/null || break; sleep 1; done
+kill $kp 2>/dev/null; wait $kp 2>/dev/null; kill $bp 2>/dev/null; wait $bp 2>/dev/null; rm -f /mnt/models/.tmp/r1_burn.gguf
+a=$(table $L 4); u=$(table logs/K_c1-old-q106.log 4); g=$(table logs/B5_kld_q106.log 4)
+v=$( [ "$a" = "$u" ] && echo SAME_AS_TODAY || { [ "$a" = "$g" ] && echo SAME_AS_AFTERNOON || echo DIFFERENT_FROM_BOTH; } )
+echo "{\"step\":\"R1\",\"burn_seen\":$(grep -c 'quantize' logs/R1_burn.log),\"result\":\"$v\"}" | tee -a $J
+
+log "R2 BF16 perplexity + fresh base logits"
+$PPL -m gguf/$N-BF16.gguf -f calib/wikitext-2-raw/wiki.test.raw --chunks 40 --kl-divergence-base kld/bf16_r2.kld \
+  > logs/R2_ppl_bf16.log 2>&1; log "R2 exit=$?"
+grep -E "Final estimate" logs/R2_ppl_bf16.log; echo "afternoon:"; grep -E "Final estimate" logs/B1_ppl_bf16.log
+cmp -s kld/bf16.kld kld/bf16_r2.kld && r2=IDENTICAL_BASE_LOGITS || r2=DIFFERENT_BASE_LOGITS
+echo "{\"step\":\"R2\",\"base_logits_vs_afternoon\":\"$r2\"}" | tee -a $J
+
+log "R3 grade every shipped file against the fresh base"
+for pair in "q106 out-mtp/$N-MTP-Q4_0_ROCMFP4_STRIX_LEAN.gguf" "q102 out-mtp/$N-MTP-Q4_0_ROCMFP4_COHERENT.gguf" \
+            "q106i out-mtp-imat/$N-MTP-imatrix-Q4_0_ROCMFP4_STRIX_LEAN.gguf" "q102i out-mtp-imat/$N-MTP-imatrix-Q4_0_ROCMFP4_COHERENT.gguf" \
+            "q115 out-mtp/$N-MTP-Q8_0_ROCMFPX_AGENT.gguf" "q111 out-mtp/$N-MTP-Q8_0_ROCMFPX.gguf"; do
+  set -- $pair
+  $PPL -m $2 --kl-divergence-base kld/bf16_r2.kld --kl-divergence > logs/R3_kld_$1.log 2>&1
+  echo "  $1 exit=$? $(grep -E 'Mean +KLD' logs/R3_kld_$1.log | tr -s ' ')"
+done
+
+log "R4 repeat the STRIX_LEAN grade; every per-chunk row must match R3"
+$PPL -m out-mtp/$N-MTP-Q4_0_ROCMFP4_STRIX_LEAN.gguf --kl-divergence-base kld/bf16_r2.kld --kl-divergence > logs/R4_kld_q106_repeat.log 2>&1
+a=$(table logs/R4_kld_q106_repeat.log); b=$(table logs/R3_kld_q106.log); n=$(printf '%s\n' "$a" | grep -c .)
+echo "{\"step\":\"R4\",\"rows\":$n,\"result\":\"$( [ "$n" -eq 40 ] && [ "$a" = "$b" ] && echo MATCH || echo MISMATCH )\"}" | tee -a $J
+log "REGRADE_DONE"

recipe/pipeline/render_cards_v2.py

@@ -0,0 +1,744 @@
+#!/usr/bin/env python3
+"""Render both HF cards (in-file-MTP layout) from summary_v2.json + judgments_v2.json. Every number comes from the
+summary; absent => '—'. Judgment prose lives in judge_v2.py and only ever references computed values.
+usage: render_cards_v2.py <summary_v2.json> <outdir> <judgments_v2.json> <SHA256SUMS_std> <SHA256SUMS_imat> <staged_sizes.json>"""
+import json, math, os, statistics, sys
+
+S = json.load(open(sys.argv[1])); OUT = sys.argv[2]; os.makedirs(OUT, exist_ok=True)
+JUDGE = json.load(open(sys.argv[3]))
+SUMS = {}
+for sf in sys.argv[4:6]:
+    if os.path.exists(sf):
+        for line in open(sf):
+            h, fn = line.split(maxsplit=1); SUMS[fn.strip()] = h
+SIZES = json.load(open(sys.argv[6])) if len(sys.argv) > 6 and os.path.exists(sys.argv[6]) else {}
+N = S["model"]; STD, IMAT = S["model_repo_std"], S["model_repo_imat"]
+T = S["tiers"]; H = S.get("history_separate_head", {}); B = S["binary"]
+GiB, MiB = 1024 ** 3, 1024 ** 2
+UB = 1024
+
+
+def g(v, fmt="{:.2f}"): return "—" if v is None else fmt.format(v)
+def gib(b): return g(None if b is None else b / GiB, "{:.2f} GiB")
+def pm(v, e, fmt="{:.4f}"): return "—" if v is None else (fmt.format(v) + ("" if e is None else " ± " + fmt.format(e)))
+def bench(label): return next((x for x in S["bench"] if x["label"] == label), None)
+def hbench(label): return next((x for x in H.get("bench", []) if x["label"] == label), None)
+def tg(label):
+    x = bench(label); return None if x is None else x.get("tg_median")
+def gate(label): return next((x for x in S["gates"] if x.get("label") == label), None)
+def J(k): return JUDGE.get(k, f"**[JUDGMENT PENDING: {k}]**")
+def Jopt(k): return JUDGE.get(k, "")
+def speed_label(tag, dev): return f"p-infile-{dev}-n4" if tag == "q106" else f"t-{tag}-{dev}-n4"
+
+
+NAMES = {"q106": "Q4_0_ROCMFP4_STRIX_LEAN", "q102": "Q4_0_ROCMFP4_COHERENT", "q115": "Q8_0_ROCMFPX_AGENT",
+         "q111": "Q8_0_ROCMFPX", "q106i": "Q4_0_ROCMFP4_STRIX_LEAN", "q102i": "Q4_0_ROCMFP4_COHERENT"}
+
+
+def tier_row(tag):
+    t = T[tag]; ratio = t.get("ppl_ratio")
+    r = [f"`{t['file']}`", str(t["ftype"]), gib(t["size_bytes"]), g(t["bpw"]), pm(t["kld_mean"], t["kld_err"]),
+         g(t["same_top_p"], "{:.2f} %"), pm(t["ppl"], t["ppl_err"]) + ("" if ratio is None else f" (×{ratio:.4f})"),
+         g(tg(speed_label(tag, "rocm"))), g(tg(speed_label(tag, "vk")))]
+    return "| " + " | ".join(r) + " |"
+
+
+PROTOCOL = ("Ryzen AI Max+ 395 (MAX-1), ROCm 7.2.4, patched `llama-server` (see [Quick start](#quick-start)), `-c 65536`, "
+            "batch 1, greedy (`temp 0`, `top_k 1`), `ignore_eos` so every arm generates exactly 256 tokens after a ~7.1K-token "
+            "prompt, a unique nonce per request and `cache_prompt: false` (`cache_n = 0` asserted on every timed request), "
+            "1 warm-up then the median of 3. Decode numbers are the server's own `predicted_per_second`. Box iced: no other "
+            "model loaded.")
+QUALITY = ("Quality is graded against the **BF16 GGUF** on a **held-out** corpus (wikitext-2 *test*, `-c 2048`, "
+           f"{S['bf16']['chunks']} chunks × {S['bf16']['n_ctx'] // 2 - 1:,} scored tokens each — the second half of every window, less its first token — = "
+           f"{S['bf16']['scored_tokens']:,}), never on the imatrix calibration text. **KLD** is the per-token KL divergence of each "
+           "quant's next-token distribution from BF16's on the same tokens — far more sensitive than perplexity, whose error bar "
+           "here is wider than most of the differences.")
+TABLE_HEAD = ("| File | ftype | Size | BPW | KLD vs BF16 ↓² | Same top-1 ↑ | PPL (× BF16) | TG MTP ROCm0 | TG MTP Vulkan0 |\n"
+              "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |")
+TG_NOTE = ("TG = decode tokens/s after a ~7.1K-token prompt, in-file MTP head, `--spec-draft-n-max 4`, default "
+           "(non-strict) verification.")
+
+
+def repro(model_file, label):
+    b = bench(label)
+    o, p = B["original"]["sha256"], B["patched"]
+    return f"""```
+server   : {B['repo']} @ {B['commit']}
+           + {p['patch_file']} (sha256 {p['patch_sha256']}), applied {p['applied'] or '—'}
+           built target llama-server with the same CMake options as below; sha256 {p['sha256']['llama-server']}
+tools    : same commit, unpatched (quantize / imatrix / perplexity; the patch touches only common/ and tools/server/)
+           build dir /opt/llama-rocm/rocmfpx-724/build-hipvk, Release, Unix Makefiles, GGML_HIP=ON GGML_VULKAN=ON
+           GGML_HIP_GRAPHS=ON GGML_HIP_NO_VMM=ON GGML_NATIVE=ON AMDGPU_TARGETS=gfx1151 LLAMA_CURL=OFF
+           (this CMake forwards AMDGPU_TARGETS to GPU_TARGETS, the spelling used in Quick start)
+           CMAKE_HIP_COMPILER=/opt/rocm-7.2.4/lib/llvm/bin/clang
+           sha256 llama-quantize   {o['llama-quantize']}
+           sha256 llama-imatrix    {o['llama-imatrix']}
+           sha256 llama-perplexity {o['llama-perplexity']}
+           sha256 llama-server     {o['llama-server']}   (unpatched; used for the unpatched control rows: prompt cache and vision)
+source   : {S['source']['repo']}  revision {S['source']['revision']}
+           folded with recipe/fold_agnes.py (sha256 {S['fold'].get('fold_script_sha256', '—')})
+model    : {model_file}
+argv     : {b['cmd'] if b else '—'}
+env      : LD_LIBRARY_PATH=<patched build>/bin:/opt/rocm-7.2.4/lib
+           HSA_OVERRIDE_GFX_VERSION=11.5.1 GGML_HIP_ENABLE_UNIFIED_MEMORY=1
+box      : aimax — AMD Ryzen AI Max+ 395 / Radeon 8060S (gfx1151), 124 GiB, GTT 131072 MiB,
+           kernel 6.17.6-061706-generic, ROCm 7.2.4
+power    : no platform_profile interface on this box; cpufreq governor=performance, EPP=performance;
+           GPU measured 84-85 W at 95-100% busy under load
+protocol : ~7.1K-token prompt, 256 generated tokens, temp 0 / top_k 1, ignore_eos, cache_prompt false,
+           1 warm-up + median of 3, no co-resident models (box iced)
+measured : 2026-09-16, by the pipeline in recipe/ (every raw number in recipe/results_summary.json and recipe/raw/)
+```"""
+
+
+YAML = """---
+license: apache-2.0
+base_model: Agnes-AI/Agnes-3.0-Flash
+base_model_relation: quantized
+pipeline_tag: image-text-to-text
+library_name: gguf
+tags:
+  - gguf
+  - llama.cpp
+  - rocm
+  - amd
+  - rocmfp4
+  - rocmfpx
+  - strix-halo
+  - amd-strix-halo
+  - gfx1151
+  - ryzen-ai-max
+  - ryzen-ai-max-395
+  - radeon-8060s
+  - mtp
+  - speculative-decoding
+  - reasoning
+  - multimodal
+  - vision
+  - agnes
+  - qwen3.5
+  - quantized{extra}
+---
+"""
+
+
+def spec_table():
+    rows = ["| Config | Decode tok/s (min–max) | Draft acceptance | Prefill tok/s |", "| --- | ---: | ---: | ---: |"]
+    spec = [("no draft head", "p-nospec-rocm", "p-nospec-vk"),
+            ("in-file MTP, `n-max 3`", "p-infile-rocm-n3", None),
+            ("in-file MTP, `n-max 4`", "p-infile-rocm-n4", "p-infile-vk-n4"),
+            ("in-file MTP, `n-max 5`", "p-infile-rocm-n5", None),
+            ("in-file MTP, `n-max 4`, `--spec-mtp-strict-qwen`", "p-infile-strict-rocm-n4", "p-infile-strict-vk-n4")]
+    for name, rl, vl in spec:
+        for dev, lab in (("ROCm0", rl), ("Vulkan0", vl)):
+            if lab is None: continue
+            b = bench(lab)
+            if b is None:
+                rows.append(f"| {dev} · {name} | — | — | — |"); continue
+            bold = "**" if lab == f"p-infile-rocm-n{S.get('knee')}" else ""
+            rows.append(f"| {dev} · {name} | {bold}{b['tg_median']:.2f}{bold} ({b['tg_min']:.2f}–{b['tg_max']:.2f}) | "
+                        f"{g(b.get('accept'), '{:.3f}')} | {g(b.get('pp_median'), '{:.0f}')} |")
+    return "\n".join(rows)
+
+
+def history_line():
+    rs = sorted([x for x in H.get("bench", []) if x["label"].startswith("sweep-rocm-n")], key=lambda x: x["nmax"])
+    if not rs: return ""
+    seq = ", ".join(f"{x['nmax']}: {x['tg_median']:.2f}" for x in rs)
+    return (f"An earlier, wider sweep with the separate head file (not shipped any more, unpatched server) found the same "
+            f"shape on ROCm0 — n-max {seq} tok/s, peak at n-max {H.get('knee')} — so the in-file check above only brackets that "
+            f"peak. Raw rows: `recipe/raw/separate_head_phase_b.jsonl`.")
+
+
+def range_line():
+    c, p = bench("t-q106i-rocm-n4"), bench("p-infile-rocm-n4-prose")
+    if not c or not p: return ""
+    return (f"**Workload range** (imatrix STRIX_LEAN, ROCm0, n-max 4): code {c['tg_median']:.2f} tok/s (acceptance "
+            f"{g(c.get('accept'), '{:.3f}')}) · prose {p['tg_median']:.2f} tok/s (acceptance {g(p.get('accept'), '{:.3f}')}). "
+            "Speculative speed depends on how predictable the text is — quote a range, not a single number.")
+
+
+def tools_block():
+    t = gate("tools-infile-patched")
+    if not t: return "_Not measured._"
+    names = ["multi-arg", "nested-object", "enum", "correct-decline", "multi-turn", "streaming", "parallel"]
+    rows = ["| check | thinking ON (`reasoning_effort: low`) | thinking OFF (`enable_thinking: false`) |",
+            "| --- | :---: | :---: |"]
+    for n in names:
+        a, b = t["detail"].get(f"{n}|think=True"), t["detail"].get(f"{n}|think=False")
+        mk = lambda x: "—" if x is None else ("✅" if x else "❌")
+        rows.append(f"| {n} | {mk(a)} | {mk(b)} |")
+    return (f"**{t['passed']}/{t['total']}** on native `tool_calls` (raw XML left in `content` counts as a failure), run on "
+            f"`{os.path.basename(T['q106i']['file'])}` with the in-file head on and the patched server.\n\n" + "\n".join(rows))
+
+
+def vision_block():
+    order = [("vision-q106-infile", "MTP head, `-fa on` (quick start)"), ("vision-q106-infile-faoff", "MTP head, `-fa off`"),
+             ("vision-q106-nomtp", "no head, `-fa on`"), ("vision-q106-nomtp-faoff", "no head, `-fa off`")]
+    if not any(gate(k) for k, _ in order):
+        return "_Not measured on this layout._"
+    vp = S.get("vision_probe", {})
+    def cell(k):
+        x = gate(k)
+        if not x: return "—"
+        n = f"{len(x.get('hits', []))}/{len(x['expected'].split(','))}"
+        if x.get("result") == "PASS": return f"✅ {n}"
+        return "❌ " + ("server stopped" if x.get("server_died") else ("request failed" if x.get("error") else n))
+    rows = ["| MTP head | `-fa on` | `-fa off` |", "| --- | :---: | :---: |",
+            f"| loaded | {cell('vision-q106-infile')} | {cell('vision-q106-infile-faoff')} |",
+            f"| not loaded | {cell('vision-q106-nomtp')} | {cell('vision-q106-nomtp-faoff')} |"]
+    if gate("vision-q106-infile-unpatched"):
+        rows.append(f"| loaded, **unpatched** server (control) | {cell('vision-q106-infile-unpatched')} | — |")
+    txt = (f"Probe: a synthetic {vp.get('width', '—')}×{vp.get('height', '—')} image with a red circle and a blue square "
+           f"(a model that ignores the image cannot name both), sent to `{T['q106']['file']}` on the patched server with "
+           f"`--mmproj`, temperature 0. Pass = the reply names all four of `red`, `blue`, `circle`, `square`.\n\n"
+           + "\n".join(rows) + "\n\n" + J("vision_note"))
+    ans = next(((k, n) for k, n in order if gate(k) and gate(k).get("result") == "PASS"), None)
+    if ans:
+        a_ = gate(ans[0])["answer"].strip()
+        txt += f"\n\nReply ({ans[1]}):\n\n> {a_}" + (" …" if len(a_) >= 299 else "")
+    return txt
+
+
+def head_table():
+    NM = {"q106": "STRIX_LEAN", "q102": "COHERENT", "q115": "Q8 AGENT", "q111": "Q8"}
+    rows = ["| Tier | Backend | separate head file: tok/s (acceptance) | built-in: tok/s (acceptance) | change |",
+            "| --- | --- | ---: | ---: | ---: |"]
+    have_any = False
+    for t in ("q106", "q102", "q115", "q111"):
+        for dev, dn in (("rocm", "ROCm0"), ("vk", "Vulkan0")):
+            o = hbench(f"sweep-{dev}-n4" if t == "q106" else f"tier-{t}-{dev}-n4")
+            n = bench(speed_label(t, dev))
+            if not (o and n):
+                rows.append(f"| {NM[t]} | {dn} | — | — | — |"); continue
+            have_any = True
+            rows.append(f"| {NM[t]} | {dn} | {o['tg_median']:.2f} ({g(o.get('accept'), '{:.3f}')}) | "
+                        f"{n['tg_median']:.2f} ({g(n.get('accept'), '{:.3f}')}) | "
+                        f"{(n['tg_median'] - o['tg_median']) / o['tg_median'] * 100:+.1f} % |")
+    return "\n".join(rows) if have_any else ""
+
+
+def cache_table():
+    co = gate("c-orig-infile"); d = gate("c3-patched-infile"); n_ = gate("c3-patched-nospec")
+    rows = ["| server | draft head | second-turn prompt tokens reused | processed | warm reply = cold reply |",
+            "| --- | --- | ---: | ---: | :---: |"]
+    if co:
+        rows.append(f"| d3ca537, unpatched | in-file MTP | **{co['turn2_cache_n']}** of {co['turn2_prompt_n']:,} | "
+                    f"{co['turn2_prompt_n']:,} | — |")
+    for lab, x, head in (("d3ca537 + patch", d, "in-file MTP"), ("d3ca537 + patch", gate("c3-patched-infile-strict"),
+                                                                  "in-file MTP, strict"),
+                         ("d3ca537 + patch", n_, "none (control)")):
+        if not x: continue
+        rows.append(f"| {lab} | {head} | **{x['expect_cache_n']:,}** of {x['L']:,} (all {x['n']} exchanges) | "
+                    f"{x['L'] - x['expect_cache_n']:,} | {x['identical']}/{x['n']} |")
+    return "\n".join(rows)
+
+
+def patch_block():
+    p = B["patched"]
+    ds = "\n".join(f"    {x}" for x in p.get("diffstat", []))
+    return f"""`recipe/patches/mtp-prompt-cache-fix.patch` (from [kingjones777/Qwen3.8-27B-ROCmFP4-STRIX-MTP-GGUF](https://huggingface.co/kingjones777/Qwen3.8-27B-ROCmFP4-STRIX-MTP-GGUF),
+sha256 `{p['patch_sha256']}`) applies to `d3ca537` **{'cleanly' if p['applied'] else '— (not verified)'}** (`git apply --check`):
+
+```
+{ds}
+```
+
+The MTP head's state between steps is the target model's hidden row at the cached prompt's exact end position. The
+unpatched server cannot rebuild that row for a shorter common prefix, so it refuses partial reuse and reprocesses the
+whole prompt. The patch stores the row **inside each context checkpoint** (`common_prompt_checkpoint::data_spec`),
+captured at the same moment as the checkpoint's KV state, and on a partial-prefix hit restores the newest checkpoint at
+or before the common prefix — target state, draft state and MTP row together. If no such checkpoint exists it logs
+`reason=spec-checkpoint-missing` and reprocesses cold. The token-generation path is not modified."""
+
+
+def files_table(names):
+    rows = ["| File | Size | sha256 |", "| --- | ---: | --- |"]
+    for fn, size in names:
+        size = SIZES.get(fn, size)
+        sz = "—" if size is None else (gib(size) if size >= GiB // 10 else f"{size / MiB:.1f} MiB")
+        rows.append(f"| `{fn}` | {sz} | `{SUMS.get(fn, '—')}` |")
+    return "\n".join(rows)
+
+
+def receipts_table(tags, imat=False):
+    head = ("| File | `output.weight` | `token_embd.weight` | tensors | trunk vs trunk-only build | MTP block |"
+            + (" imatrix entries | bytes differ from standard |" if imat else ""))
+    sep = "| --- | --- | --- | ---: | :---: | ---: |" + (" ---: | :---: |" if imat else "")
+    rows = [head, sep]
+    for k in tags:
+        t = T[k]; ti = t["trunk_identity"]
+        idt = "—" if not ti.get("result") else (f"{ti['result'].lower()} ({ti['shared']} tensors)")
+        r = (f"| `{t['file']}` | {t['output_weight'] or '—'} | {t['token_embd'] or '—'} | {g(t['tensors'], '{}')} | {idt} | "
+             f"{t['mtp_block']['bytes'] / MiB:.0f} MiB |")
+        if imat:
+            dfs = t.get("differs_from_standard")
+            r += f" {t.get('imatrix_entries') or '—'} | {'yes' if dfs else ('no' if dfs is False else '—')} |"
+        rows.append(r)
+    return "\n".join(rows)
+
+
+def fold_block():
+    fv = S["fold_verify"]; f = S["fold"]
+    g3 = ", ".join((f"layer {k.split()[-1]}: {float(v):.1e}" if k.startswith("layer") else f"MTP: {float(v):.1e}")
+                   for k, v in fv.get("g3", []))
+    return f"""Agnes-3.0-Flash Preview is a Qwen3.5-style dense hybrid (72 layers: 54 gated-delta-rule + 18 global attention,
+interval 4) with one addition: **every layer runs a second, narrower SwiGLU (width {f.get('parallel_ffn', '—')}) in parallel with the
+main one (width {f.get('main_ffn', '—')}), and the two outputs are summed.** The reference implementation is literally
+
+```python
+y = down(act(gate(x)) * up(x)) + parallel_ffn(x)
+```
+
+Because it is a plain sum through the same activation, the branch folds **exactly** into one wider SwiGLU:
+`gate`/`up` concatenated along the output dimension and `down` along the input dimension, giving a stock `qwen35`
+MLP of width **{f.get('folded_ffn', '—')}**. That is the same transformation Agnes AI's own SGLang patch applies. The result loads on any
+llama.cpp/ROCmFPX build with `qwen35` support — **no Agnes-specific patch** (some other GGUFs of this model keep the
+branch as separate `ffn_*_par` tensors and require a patched fork).
+
+Three details that are easy to get wrong:
+
+1. **Rename `delta_attn.*` → `linear_attn.*`.** The converter reorders the 48 value heads into the tiled order ggml
+   expects *only* for tensors named `linear_attn.*`. Skipping the rename produces a model that loads and runs and is wrong.
+2. **The MTP layer has no parallel branch** ({f.get('main_ffn', '—')} wide), so its MLP is zero-padded to {f.get('folded_ffn', '—')}. Zero
+   SwiGLU rows contribute exactly zero.
+3. Agnes's RMSNorm is one-centred, `x / rms(x) · (1 + w)`, so the converter's `+1` weight offset is correct; the gated
+   delta-rule norm is a plain scale and is correctly left alone.
+
+The fold boundary sits at column {f.get('main_ffn', '—')} = 544 × 32 = 68 × 256, so no quantization block mixes main and parallel weights.
+
+**Verification (`recipe/verify_fold.py`, which re-derives every tensor from the source independently): {fv.get('result', '—')}.**
+{f.get('tensors_out', '—')} output tensors, every one **bit-identical** to its source slice ({f.get('stats', {}).get('fold', '—')} folded,
+{f.get('stats', {}).get('pad', '—')} MTP-padded, {f.get('stats', {}).get('pass', '—')} passed through), and the folded MLP reproduces the reference
+forward pass to float32 rounding (max relative error: {g3}). The full log is in `recipe/verify_fold.log`."""
+
+
+def serve_block(model_file):
+    env = ("env LD_LIBRARY_PATH=$PWD/build/bin:/opt/rocm/lib HSA_OVERRIDE_GFX_VERSION=11.5.1 "
+           "GGML_HIP_ENABLE_UNIFIED_MEMORY=1 \\\n")
+    mtp = "  --spec-type draft-mtp \\\n  --spec-draft-n-max 4 --spec-draft-n-min 0 --spec-draft-p-min 0.0 \\\n"
+    mm = f"  --mmproj ~/models/agnes/mmproj-{N}-BF16.gguf \\\n"
+    tail = "  -ngl 999 -fa on -dio --jinja -fit off --parallel 1 -dev ROCm0 \\\n  -c 65536 --host 127.0.0.1 --port 8080"
+    head = f"build/bin/llama-server \\\n  -m ~/models/agnes/{model_file} \\\n"
+    if "vision_quickstart_warning" not in JUDGE:
+        return f"```bash\n{env}{head}{mtp}{mm}{tail}\n```"
+    return (f"Text and tool calls, with MTP:\n\n```bash\n{env}{head}{mtp}{tail}\n```\n\n"
+            f"Images (`--mmproj`), **without** MTP:\n\n```bash\n{env}{head}{mm}{tail}\n```\n\n"
+            + JUDGE["vision_quickstart_warning"])
+
+
+def quick_start(model_file, repo):
+    rb = bench("p-infile-rocm-n4")
+    return f"""**1. Download**
+
+```bash
+hf download {repo} --local-dir ~/models/agnes
+```
+
+**2. Build `llama-server` with the prompt-cache patch** — ROCmFPX at the measured commit (ROCm and Vulkan
+prerequisites: the project's [build guide](https://github.com/charlie12345/ROCmFPX/blob/{B['commit']}/docs/build.md))
+
+```bash
+git clone https://github.com/charlie12345/ROCmFPX.git && cd ROCmFPX
+git checkout {B['commit']}
+git apply ~/models/agnes/recipe/patches/mtp-prompt-cache-fix.patch
+HIPCXX="$(hipconfig -l)/clang" HIP_PATH="$(hipconfig -R)" \\
+cmake -S . -B build -DCMAKE_BUILD_TYPE=Release \\
+  -DGGML_HIP=ON -DGGML_VULKAN=ON -DGPU_TARGETS=gfx1151 \\
+  -DGGML_HIP_GRAPHS=ON -DGGML_HIP_NO_VMM=ON -DLLAMA_CURL=OFF
+cmake --build build --target llama-server -j
+```
+
+(The CMake options of the measured build are listed in [Reproduction](#reproduction).)
+
+`d3ca537` is also in the history of the official [ROCmFPX/ROCmFPX](https://github.com/ROCmFPX/ROCmFPX) repository; the
+patch was verified against `d3ca537` only.
+
+**3. Serve** — the MTP head is inside the model file, so there is no `--model-draft`:
+
+{serve_block(model_file)}
+
+(`LD_LIBRARY_PATH` avoids a soname clash on machines that also have a Vulkan-only llama.cpp build.) The benchmarks used
+the MTP command without `--mmproj`; the exact measured argv is in [Reproduction](#reproduction).
+
+| Flag | Why |
+| --- | --- |
+| `--spec-type draft-mtp` | Uses the MTP layer inside the model file; no second file to download or place. |
+| `--spec-draft-n-max 4` | llama.cpp's default is 16. On this model the measured optimum is about 4 — see [the table](#speculative-decoding-mtp). |
+| `--spec-draft-p-min 0.0` | The default 0.75 gate throttles drafting. |
+| `--jinja` | Already on by default in this build; keep it on — the thinking controls (`chat_template_kwargs`) and tool calling rely on the Jinja chat template. |
+| `-fit off` | Autofit reads `MemAvailable` on integrated GPUs and can silently shrink context or push tensors to CPU. |
+| `-cram <MiB>` | Not set above (default 8 GiB of host RAM for saved prompts). Set it on a shared box — see [Known issues](#known-issues-and-limits). |
+
+> ⚠️ **On an unpatched build the in-file head still works, but prompt caching turns off while it is loaded** — every
+> turn re-prefills the whole conversation ([measured](#prompt-caching-with-mtp)). For multi-turn / agentic use there,
+> drop the two `--spec-*` lines.
+
+Requires a llama.cpp build with ROCmFP4 / ROCmFPX tensor-type support; stock llama.cpp rejects these tensor types."""
+
+
+REASONING = """## Reasoning controls
+
+⛔ **Do not pass `reasoning_effort: "high"`.**
+
+The upstream model card lists `high` as the default effort. **The chat template shipped with the checkpoint does not
+accept it.** It defaults to `xhigh` and accepts only `xhigh`, `medium`, `low`; anything else raises:
+
+```
+Unexpected reasoning effort high. Supported types are xhigh (default), medium, and low.
+```
+
+Verified by rendering the template with each value (`xhigh` / `medium` / `low` render; `high` raises; `medium` injects
+no instruction at all). Use:
+
+| goal | request field |
+| --- | --- |
+| default deep reasoning | nothing, or `"chat_template_kwargs": {"reasoning_effort": "xhigh"}` |
+| shorter thinking | `"chat_template_kwargs": {"reasoning_effort": "low"}` |
+| thinking off | `"chat_template_kwargs": {"enable_thinking": false}` |
+
+With thinking on and a small `max_tokens`, the whole budget can go to reasoning and `content` comes back empty — raise
+`max_tokens` before concluding the model is broken.
+
+⚠️ **`preserve_thinking` defaults to true** in this template: earlier turns' reasoning is re-rendered into the prompt
+(Qwen3.5's template drops it). Multi-turn and agentic contexts grow faster than you may expect; pass
+`"preserve_thinking": false` in `chat_template_kwargs` to get the Qwen3.5 behaviour."""
+
+
+def prompt_range():
+    ns = sorted({b_["prompt_n"] for b_ in S["bench"] if b_.get("prompt_n") and b_.get("workload") == "code"
+                 and b_["label"].startswith("p-infile-")})
+    if not ns:
+        return "—"
+    return f"{ns[0]:,}" if ns[0] == ns[-1] else f"{ns[0]:,}–{ns[-1]:,}"
+
+
+def bf16_row():
+    b = S["bf16"]
+    return (f"| *BF16 reference* | 32 | {gib(b['with_mtp_size_bytes'])}³ | 16 | 0 | 100 % | "
+            f"{pm(b.get('ppl_paired'), b.get('ppl_paired_err'))}¹ | — | — |")
+
+
+def footnotes():
+    b = S["bf16"]
+    return (f"{TG_NOTE}\n"
+            f"¹ The BF16 PPL shown is the paired base every \"×\" ratio is computed against (averaged over the same scored tokens "
+            f"in the KL-divergence runs). The standalone BF16 run's own summary line reads {pm(b['ppl'], b['ppl_err'])}.\n"
+            f"² Quality columns: see *Where the quality numbers come from* below.\n"
+            f"³ BF16 conversion including the MTP layer (the graded BF16 trunk without it is {gib(b['trunk_size_bytes'])}); not "
+            f"published.")
+
+
+def methodology_std():
+    return f"""```bash
+# 0. fold the checkpoint (exact) and verify it
+python recipe/fold_agnes.py   Agnes-3.0-Flash  agnes-qwen35
+python recipe/verify_fold.py  Agnes-3.0-Flash  agnes-qwen35      # must print RESULT: PASS
+
+# 1. convert: model WITH its MTP layer (the converter's default), and the vision projector
+python convert_hf_to_gguf.py agnes-qwen35 --outtype bf16 --model-name {N} --outfile {N}-MTP-BF16.gguf
+python convert_hf_to_gguf.py agnes-qwen35 --outtype bf16 --mmproj --model-name {N} --outfile mmproj-{N}-BF16.gguf
+
+# 2. quantize from BF16 only; the LM head is forced up on every tier and read back by exact tensor name
+llama-quantize --output-tensor-type q6_K                            {N}-MTP-BF16.gguf OUT Q4_0_ROCMFP4_STRIX_LEAN 16
+llama-quantize --output-tensor-type q6_K --token-embedding-type q6_K {N}-MTP-BF16.gguf OUT Q4_0_ROCMFP4_COHERENT 16
+llama-quantize --output-tensor-type q8_0 --token-embedding-type q8_0 {N}-MTP-BF16.gguf OUT Q8_0_ROCMFPX_AGENT 16
+llama-quantize --output-tensor-type q8_0 --token-embedding-type q8_0 {N}-MTP-BF16.gguf OUT Q8_0_ROCMFPX 16
+
+# 3. grade each shipped file against BF16 logits from the same session (the trunk-only BF16: same trunk weights)
+llama-perplexity -m {N}-BF16.gguf -f wikitext-2-raw/wiki.test.raw -c 2048 -b 2048 --chunks 40 --kl-divergence-base bf16.kld \\
+  -ngl 999 -fa on -dio -dev ROCm0
+llama-perplexity -m OUT --kl-divergence-base bf16.kld --kl-divergence -c 2048 -b 2048 -ngl 999 -fa on -dio -dev ROCm0
+```
+
+Receipts (the built file is the receipt — exact tensor names, never a substring match; `recipe/logs/`):
+
+{receipts_table(("q106", "q102", "q115", "q111"))}
+
+*trunk vs trunk-only build*: every trunk tensor's type, shape and sha256 compared with the same tier built without
+the MTP layer (`tensor_identity.py`)."""
+
+
+def std_card():
+    L = T["q106"]
+    sh = S.get("separate_head_replaced", {}).get(f"mtp-{N}-Q4_0.gguf", {})
+    return YAML.format(extra="") + f"""
+# Agnes-3.0-Flash Preview — ROCmFP4 for AMD Strix Halo (gfx1151)
+
+ROCmFP4 / ROCmFPX quantizations of **[Agnes-AI/Agnes-3.0-Flash](https://huggingface.co/Agnes-AI/Agnes-3.0-Flash)** —
+the open-weight **Preview** checkpoint (33B dense, hybrid gated-delta-rule + global attention, 262,144-token context,
+text + image) — built and measured on an AMD Ryzen AI Max+ 395 (Radeon 8060S, `gfx1151`).
+
+- **First ROCmFP4 build of this model** — a Hub search on 2026-09-16 found no other ROCm or Strix Halo build of Agnes (21 Agnes repos checked).
+- **MTP head built into every file.** One GGUF per tier; `--spec-type draft-mtp` finds the head inside the model. It
+  adds {min(T[k]['mtp_block']['bytes'] for k in ('q106', 'q102', 'q106i', 'q102i')) / MiB:.0f}–{max(T[k]['mtp_block']['bytes'] for k in ('q106', 'q102', 'q106i', 'q102i')) / MiB:.0f} MiB to a 4-bit file — the separate head file it replaces was {g((sh.get('size_bytes') or 0) / GiB if sh else None)} GiB
+  ([why](#speculative-decoding-mtp)).
+- **Prompt caching works with MTP on**, with the small `llama-server` patch in `recipe/patches/` — without it, a build of
+  this ROCmFPX line reuses nothing while the head is loaded ([measured](#prompt-caching-with-mtp)).
+- **Vision projector included.**
+- **No Agnes-specific patch needed to load the weights** — the parallel FFN branch is folded *exactly* into a standard
+  `qwen35` MLP ([details](#how-the-parallel-ffn-was-handled)).
+- Importance-matrix builds of the 4-bit tiers: **[{IMAT}](https://huggingface.co/{IMAT})**.
+
+> ⚠️ This is the **Preview** checkpoint. Agnes AI states that its production/API model (1M context) is a different
+> checkpoint and that published benchmark results for that model do not apply to these weights.
+
+## Which file should I use?
+
+{PROTOCOL}
+
+{TABLE_HEAD}
+{chr(10).join(tier_row(k) for k in ("q106", "q102", "q115", "q111"))}
+{bf16_row()}
+
+{footnotes()}
+
+{J('std_recommendation')}
+
+{QUALITY}
+
+**Where the quality numbers come from.** {J('quality_provenance')}
+
+## Quick start
+
+{quick_start(JUDGE.get('std_default') or L['file'], STD)}
+
+{REASONING}
+
+## Speculative decoding (MTP)
+
+{J('layout_note')} {Jopt('memory_head_note')}
+
+{head_table()}
+
+{J('head_speed_note')}
+
+Measured on `{L['file']}` (patched server; code prompts of {prompt_range()} tokens — each request carries a unique nonce):
+
+{spec_table()}
+
+{J('std_mtp_notes')}
+
+{range_line()}
+
+{history_line()}
+
+### Is the output the same as without MTP?
+
+{J('identity_note')}
+
+{J('strict_note')}
+
+## Prompt caching with MTP
+
+{patch_block()}
+
+Measured: a second turn that shares the first turn's ~7K-token prefix and differs only in its closing instruction, each
+exchange run warm (`cache_prompt: true`) and then cold (`cache_prompt: false`) and the two replies compared byte for byte.
+
+{cache_table()}
+
+{J('std_cache_note')}
+
+## Memory
+
+{J('memory_note')}
+
+## Tool calling
+
+The template emits the XML-style `<tool_call><function=…><parameter=…>` format, which llama.cpp parses natively
+(through the Jinja chat template, on by default). Suite run through `llama-server` in the deployed configuration (in-file MTP on), at the
+checkpoint's recommended sampling (temperature 1.0, top-p 0.95, top-k 20; file and sampling are set in
+`recipe/pipeline/vplan_b.py` and `recipe/pipeline/agnes_harness.py`):
+
+{tools_block()}
+
+{J('std_tools_note')}
+
+## Vision
+
+`mmproj-{N}-BF16.gguf` is the 27-layer vision tower (shape-identical to Qwen3.5-27B's), loaded with `--mmproj`.
+Its attention follows the server's `-fa` setting, so both settings were checked, with and without the MTP head.
+
+{vision_block()}
+
+## How the parallel FFN was handled
+
+{fold_block()}
+
+## Quantization methodology
+
+{methodology_std()}
+
+## Reproduction
+
+{repro(L['file'], 'p-infile-rocm-n4')}
+
+## Files
+
+{files_table([(T[k]['file'], T[k]['size_bytes']) for k in ('q106', 'q102', 'q115', 'q111')] + [(k, v) for k, v in S['aux'].items()])}
+
+`SHA256SUMS` covers every model file. `recipe/` holds the fold and verification scripts, the prompt-cache patch
+(`recipe/patches/`), the full measurement pipeline (`recipe/pipeline/`), raw per-run results (`recipe/raw/`), build and
+receipt logs (`recipe/logs/`), and `results_summary.json` with every measured value on this card plus the binary hashes.
+Architecture facts (layer counts, vocabulary, vision depth) come from the checkpoint's `config.json`.
+
+## Known issues and limits
+
+{J('std_known_issues')}
+
+## License and attribution
+
+Apache-2.0, inherited from the base model. Weights and architecture: **Agnes AI**
+([Agnes-AI/Agnes-3.0-Flash](https://huggingface.co/Agnes-AI/Agnes-3.0-Flash)). ROCmFP4 / ROCmFPX quantization format and
+runtime: the ROCmFPX project. MTP prompt-cache patch: kingjones777. Folding, quantization, and measurements: kingjones777.
+"""
+
+
+def imat_card():
+    t = T
+    def d(a, b, k, ek):
+        x, y = t[a].get(k), t[b].get(k); ex, ey = (t[a].get(ek), t[b].get(ek)) if ek else (None, None)
+        if None in (x, y): return "—"
+        s = f"{(y - x) / x * 100:+.1f} %"
+        if ex is not None and ey is not None:
+            s += f" ({abs(y - x) / math.sqrt(ex * ex + ey * ey):.1f}σ)"
+        return s
+    rows = ["| Tier | Build | Size | KLD vs BF16 ↓ | Same top-1 ↑ | PPL (× BF16) | 99th-pct KLD |",
+            "| --- | --- | ---: | ---: | ---: | ---: | ---: |"]
+    for base, imat in (("q106", "q106i"), ("q102", "q102i")):
+        for tag, lab in ((base, "standard"), (imat, "**imatrix**")):
+            x = t[tag]
+            rows.append(f"| {NAMES[tag]} | {lab} | {gib(x['size_bytes'])} | {pm(x['kld_mean'], x['kld_err'])} | "
+                        f"{g(x['same_top_p'], '{:.2f} %')} | {pm(x['ppl'], x['ppl_err'])} "
+                        f"({'—' if x['ppl_ratio'] is None else '×%.4f' % x['ppl_ratio']}) | {g(x['kld_p99'], '{:.4f}')} |")
+        dpp = ("—" if None in (t[base]['same_top_p'], t[imat]['same_top_p'])
+               else f"{t[imat]['same_top_p'] - t[base]['same_top_p']:+.2f} pp")
+        rows.append(f"| | *Δ imatrix* | | {d(base, imat, 'kld_mean', 'kld_err')} | {dpp} | "
+                    f"{d(base, imat, 'ppl', 'ppl_err')} | {d(base, imat, 'kld_p99', None)} |")
+    im = S["imatrix"]; Li = t["q106i"]
+    return YAML.format(extra="\n  - imatrix") + f"""
+# Agnes-3.0-Flash Preview — ROCmFP4 **imatrix** for AMD Strix Halo (gfx1151)
+
+Importance-matrix-calibrated ROCmFP4 quantizations of
+**[Agnes-AI/Agnes-3.0-Flash](https://huggingface.co/Agnes-AI/Agnes-3.0-Flash)** (Preview, 33B dense, 262,144-token context,
+text + image). Companion to the standard build **[{STD}](https://huggingface.co/{STD})** — the same two 4-bit tiers,
+same fold, same built-in MTP head, same vision projector and the same prompt-cache patch; the only difference is how each
+4-bit block's scale was chosen.
+
+## What the imatrix changes
+
+ROCmFP4 has an importance-weighted quantizer path: with `--imatrix`, each block's scale is chosen by an exhaustive
+search that minimises error **weighted by how strongly the calibration activations use each weight**, instead of the
+unweighted default. It changes **which** scales are picked at the **same** bit width and tensor types — so it moves
+quality, not size, and per-token compute is identical.
+
+| | |
+| --- | --- |
+| calibration text | {im['calibration']} (the widely used community calibration set) |
+| computed on | BF16 GGUF, {im['chunks']} chunks × {im['n_ctx']} tokens, ROCm0 |
+| coverage | **558 entries = every matmul weight in the 72 trunk layers** (54 delta-rule layers × 8 + 18 global layers × 7); the output projection and the MTP block have none |
+| file | `{im['file']}` (GGUF format), sha256 `{im['sha256']}` |
+
+## Measured effect
+
+{QUALITY} The calibration text and the grading text are different corpora.
+
+{chr(10).join(rows)}
+
+σ = difference divided by the two runs' combined standard error. The two runs score the **same** tokens, so this is
+conservative (paired noise is smaller).
+
+{J('imat_verdict')}
+
+**Where the quality numbers come from.** {J('quality_provenance')}
+
+## Which file should I use?
+
+{J('imat_recommendation')}
+
+{TABLE_HEAD}
+{tier_row('q106i')}
+{tier_row('q102i')}
+{bf16_row()}
+
+{footnotes()}
+
+{Jopt('imat_speed_note')}
+
+Full speed tables (depth check, strict mode, both backends, workload range), prompt-cache and tool-calling results are on
+the [standard card]({'https://huggingface.co/' + STD}). The 8-bit tiers are not duplicated here; they are on the standard repo.
+
+## Quick start
+
+{quick_start(JUDGE.get('imat_default') or Li['file'], IMAT)}
+
+{REASONING}
+
+## MTP, prompt caching, tool calling, vision
+
+Same architecture, template, in-file MTP block ({Li['mtp_block']['bytes'] / MiB:.0f} MiB in STRIX_LEAN,
+{t['q102i']['mtp_block']['bytes'] / MiB:.0f} MiB in COHERENT) and projector as the standard build. The prompt-cache patch
+is included here too (`recipe/patches/`). The tool-calling suite ran on this repo's STRIX_LEAN — see
+[the standard card]({'https://huggingface.co/' + STD}#tool-calling).
+
+## How the parallel FFN was handled
+
+{fold_block()}
+
+## Quantization methodology
+
+```bash
+llama-imatrix -m {N}-BF16.gguf -f calibration_datav3.txt -o {N}.imatrix \\
+  -ngl 999 -c 512 -b 512 -fa on -dio -dev ROCm0
+llama-quantize --imatrix {N}.imatrix --output-tensor-type q6_K \\
+  {N}-MTP-BF16.gguf {N}-MTP-imatrix-Q4_0_ROCMFP4_STRIX_LEAN.gguf Q4_0_ROCMFP4_STRIX_LEAN 16
+llama-quantize --imatrix {N}.imatrix --output-tensor-type q6_K --token-embedding-type q6_K \\
+  {N}-MTP-BF16.gguf {N}-MTP-imatrix-Q4_0_ROCMFP4_COHERENT.gguf Q4_0_ROCMFP4_COHERENT 16
+```
+
+The imatrix was computed on the trunk-only BF16; the MTP layer is quantized from the BF16 that includes it
+(`{N}-MTP-BF16.gguf`), with the unweighted path for its tensors.
+
+Receipts that the weighted path was actually taken, and that the shipped trunk is the graded one:
+
+{receipts_table(("q106i", "q102i"), imat=True)}
+
+## Reproduction
+
+{repro(Li['file'], 't-q106i-rocm-n4')}
+
+## Files
+
+{files_table([(t[k]['file'], t[k]['size_bytes']) for k in ('q106i', 'q102i')] + [(im['file'], im.get('size_bytes'))] + [(k, v) for k, v in S['aux'].items()])}
+
+## Known issues and limits
+
+{J('imat_known_issues')}
+
+## License and attribution
+
+Apache-2.0, inherited from the base model. Weights and architecture: **Agnes AI**. Calibration text:
+bartowski's `calibration_datav3`. ROCmFP4 / ROCmFPX: the ROCmFPX project. MTP prompt-cache patch, folding, imatrix,
+quantization and measurements: kingjones777.
+"""
+
+
+open(os.path.join(OUT, "README_std.md"), "w").write(std_card())
+def fix_anchors(md, other_repo):
+    """Links to sections that exist only on the other card point there instead of to a missing anchor."""
+    import re
+    slugs = {re.sub(r"[^a-z0-9 -]", "", h.strip().lower()).replace(" ", "-")
+             for h in re.findall(r"^#{1,6} (.+)$", md, flags=re.M)}
+    return re.sub(r"\]\(#([a-z0-9-]+)\)",
+                  lambda m: m.group(0) if m.group(1) in slugs else f"](https://huggingface.co/{other_repo}#{m.group(1)})", md)
+
+
+open(os.path.join(OUT, "README_imat.md"), "w").write(fix_anchors(imat_card(), STD))
+cards = open(os.path.join(OUT, "README_std.md")).read() + open(os.path.join(OUT, "README_imat.md")).read()
+pend = sorted(set(x.split("JUDGMENT PENDING: ")[1].split("]")[0] for x in cards.split("**[")[1:] if "JUDGMENT PENDING" in x))
+if "vision_blocker" in JUDGE:
+    pend.append("BLOCKER: " + JUDGE["vision_blocker"])
+dash_cells = cards.count("| — |")
+print("rendered | knee =", S.get("knee"), "| bench rows =", len(S["bench"]), "| gates =", len(S["gates"]),
+      "| pending judgments:", pend, "| '—' cells:", dash_cells)

recipe/pipeline/seat_sizing.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+# Measure (not estimate) the host-memory footprint of a candidate Agnes seat on the ICED box.
+# For each ctx: MemAvailable before -> after load -> after one ~8K-token request. Seat argv mirrors :8094 (q8_0 KV, -cram 512).
+set -uo pipefail
+W=/mnt/models/agnes-3.0-flash; B=/opt/llama-rocm/rocmfpx-724/build-hipvk/bin; N=Agnes-3.0-Flash-Preview; cd $W
+export LD_LIBRARY_PATH=$B:/opt/rocm-7.2.4/lib HSA_OVERRIDE_GFX_VERSION=11.5.1 GGML_HIP_ENABLE_UNIFIED_MEMORY=1
+KNEE=$(python3 -c "import json;print(json.load(open('results/knee.json'))['knee'])")
+MODEL=${1:-out-imat/$N-imatrix-Q4_0_ROCMFP4_COHERENT.gguf}
+avail(){ awk '/MemAvailable/{printf "%.2f", $2/1048576}' /proc/meminfo; }
+for C in 65536 131072 262144; do
+  a0=$(avail)
+  $B/llama-server -m $MODEL -dev ROCm0 -ngl 999 -fa on --no-mmap -fit off -c $C -ctk q8_0 -ctv q8_0 -np 1 \
+     -b 2048 -ub 1024 -t 16 --jinja --spec-type draft-mtp -md out/mtp-$N-Q4_0.gguf --spec-draft-n-max $KNEE \
+     --spec-draft-ngl 99 --spec-draft-p-min 0.0 --mmproj out/mmproj-$N-BF16.gguf -cram 512 \
+     --host 127.0.0.1 --port 18700 > logs/sizing_c$C.log 2>&1 &
+  pid=$!
+  ok=0; for i in $(seq 1 120); do curl -s -m 2 http://127.0.0.1:18700/health | grep -q ok && { ok=1; break; }; kill -0 $pid 2>/dev/null || break; sleep 3; done
+  if [ $ok -ne 1 ]; then echo "{\"ctx\":$C,\"result\":\"LOAD_FAIL\",\"avail_before\":$a0}" | tee -a results/seat_sizing.jsonl; kill $pid 2>/dev/null; sleep 5; continue; fi
+  a1=$(avail)
+  python3 - <<EOF
+import json,urllib.request
+txt=open("/opt/llama-rocm/rocmfpx-724/convert_hf_to_gguf.py").read()[:30000]
+b={"messages":[{"role":"user","content":txt+"\n\nSummarise this file in one sentence."}],"max_tokens":64,"temperature":0,
+   "chat_template_kwargs":{"enable_thinking":False}}
+urllib.request.urlopen(urllib.request.Request("http://127.0.0.1:18700/v1/chat/completions",data=json.dumps(b).encode(),
+   headers={"Content-Type":"application/json"}),timeout=600).read()
+EOF
+  a2=$(avail)
+  echo "{\"ctx\":$C,\"model\":\"$(basename $MODEL)\",\"avail_before\":$a0,\"avail_loaded\":$a1,\"avail_after_8k\":$a2,\"footprint_loaded_gib\":$(python3 -c "print(round($a0-$a1,2))"),\"footprint_after_8k_gib\":$(python3 -c "print(round($a0-$a2,2))")}" | tee -a results/seat_sizing.jsonl
+  kill $pid; wait $pid 2>/dev/null; sleep 8
+done
+echo SIZING_DONE

recipe/pipeline/seat_sizing2.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+# Apples-to-apples memory: SAME tier (imatrix COHERENT), SAME patched binary, separate head vs in-file head.
+set -uo pipefail
+W=/mnt/models/agnes-3.0-flash; B=/opt/llama-rocm/rocmfpx-724-mtpcache/build-hipvk/bin; N=Agnes-3.0-Flash-Preview; cd $W
+export LD_LIBRARY_PATH=$B:/opt/rocm-7.2.4/lib HSA_OVERRIDE_GFX_VERSION=11.5.1 GGML_HIP_ENABLE_UNIFIED_MEMORY=1
+avail(){ awk '/MemAvailable/{printf "%.2f", $2/1048576}' /proc/meminfo; }
+dl(){ if systemctl list-units --state=active --no-legend --plain 'nex-*.scope' 2>/dev/null | grep -q .; then echo active; else echo inactive; fi; }
+# the next model (Nex-N2.5-mini) downloads/converts/quantizes in capped nex-* scopes; their page-cache churn would
+# skew MemAvailable deltas, so measure only when none is running (max 2 h)
+for i in $(seq 1 360); do [ "$(dl)" = active ] || break; sleep 20; done
+echo "download scope before sizing: $(dl)"; sync; sleep 5
+measure(){  # $1 label  $2 ctx  rest = model/draft args
+  local label=$1 C=$2; shift 2
+  local a0=$(avail)
+  $B/llama-server "$@" -dev ROCm0 -ngl 999 -fa on --no-mmap -fit off -c $C -ctk q8_0 -ctv q8_0 -np 1 -b 2048 -ub 1024 -t 16 \
+     --jinja --spec-type draft-mtp --spec-draft-n-max 4 --spec-draft-p-min 0.0 --mmproj out/mmproj-$N-BF16.gguf -cram 512 \
+     --host 127.0.0.1 --port 18710 > logs/sizing2_${label}_c$C.log 2>&1 &
+  local pid=$! ok=0
+  for i in $(seq 1 120); do curl -s -m 2 http://127.0.0.1:18710/health | grep -q ok && { ok=1; break; }; kill -0 $pid 2>/dev/null || break; sleep 3; done
+  if [ $ok -ne 1 ]; then echo "{\"label\":\"$label\",\"ctx\":$C,\"result\":\"LOAD_FAIL\"}" | tee -a results/seat_sizing2.jsonl; kill $pid 2>/dev/null; sleep 5; return; fi
+  local a1=$(avail)
+  python3 - <<EOF
+import json,urllib.request
+txt=open("/opt/llama-rocm/rocmfpx-724/convert_hf_to_gguf.py").read()[:30000]
+b={"messages":[{"role":"user","content":txt+"\n\nSummarise this file in one sentence."}],"max_tokens":64,"temperature":0,
+   "chat_template_kwargs":{"enable_thinking":False}}
+urllib.request.urlopen(urllib.request.Request("http://127.0.0.1:18710/v1/chat/completions",data=json.dumps(b).encode(),
+   headers={"Content-Type":"application/json"}),timeout=600).read()
+EOF
+  local a2=$(avail)
+  echo "{\"label\":\"$label\",\"ctx\":$C,\"avail_before\":$a0,\"footprint_loaded_gib\":$(python3 -c "print(round($a0-$a1,2))"),\"footprint_after_8k_gib\":$(python3 -c "print(round($a0-$a2,2))"),\"concurrent_download\":\"$(dl)\"}" | tee -a results/seat_sizing2.jsonl
+  kill $pid; wait $pid 2>/dev/null; sleep 8
+}
+measure separate-head 65536 -m out-imat/$N-imatrix-Q4_0_ROCMFP4_COHERENT.gguf --model-draft out/mtp-$N-Q4_0.gguf --spec-draft-ngl 99 --spec-draft-device ROCm0
+for C in 65536 131072 262144; do
+  measure in-file $C -m out-mtp-imat/$N-MTP-imatrix-Q4_0_ROCMFP4_COHERENT.gguf
+done
+measure in-file-lean 65536 -m out-mtp-imat/$N-MTP-imatrix-Q4_0_ROCMFP4_STRIX_LEAN.gguf
+echo SIZING2_DONE

recipe/pipeline/tensor_identity.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+"""Compare every shared tensor (name, type, shape, BYTES) between an old and new GGUF.
+usage: tensor_identity.py <old.gguf> <new.gguf>   -> prints summary JSON; exit 0 iff all shared tensors identical"""
+import hashlib, json, sys
+sys.path.insert(0, "/opt/llama-rocm/rocmfpx-724/gguf-py")
+import gguf
+def index(p):
+    r = gguf.GGUFReader(p); out = {}
+    for t in r.tensors:
+        try: tn = t.tensor_type.name
+        except Exception: tn = f"TYPE_{int(t.tensor_type)}"
+        out[t.name] = (tn, tuple(int(x) for x in t.shape), hashlib.sha256(t.data.tobytes()).hexdigest())
+    return out
+a, b = index(sys.argv[1]), index(sys.argv[2])
+shared = sorted(set(a) & set(b))
+diff_type = [n for n in shared if a[n][0] != b[n][0] or a[n][1] != b[n][1]]
+diff_bytes = [n for n in shared if n not in diff_type and a[n][2] != b[n][2]]
+res = {"old_tensors": len(a), "new_tensors": len(b), "shared": len(shared),
+       "only_in_new": sorted(set(b) - set(a))[:20], "only_in_old": sorted(set(a) - set(b))[:20],
+       "type_or_shape_changed": [(n, a[n][0], b[n][0]) for n in diff_type][:20],
+       "bytes_changed": diff_bytes[:20], "n_type_changed": len(diff_type), "n_bytes_changed": len(diff_bytes)}
+res["result"] = "IDENTICAL" if not diff_type and not diff_bytes and not res["only_in_old"] else "DIFFERENT"
+print(json.dumps(res))
+sys.exit(0 if res["result"] == "IDENTICAL" else 1)

recipe/pipeline/trunk_receipt.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+# Receipt that the grades carry over to the in-file-MTP files: run the EXACT grading command on each new file and
+# compare its first 4 per-chunk rows (cumulative PPL, ln ratio, KLD, dp RMS, same-top-p) with the graded run.
+# kl_divergence always walks every chunk of the base file (it ignores --chunks), so the run is stopped after row 4.
+# Then the image path on the shipped file: MTP on/off x -fa on/off (the quick start uses MTP on + -fa on).
+set -uo pipefail
+W=/mnt/models/agnes-3.0-flash; B=/opt/llama-rocm/rocmfpx-724/build-hipvk/bin; N=Agnes-3.0-Flash-Preview; cd $W
+export LD_LIBRARY_PATH=$B:/opt/rocm-7.2.4/lib HSA_OVERRIDE_GFX_VERSION=11.5.1 GGML_HIP_ENABLE_UNIFIED_MEMORY=1
+rows(){  # first 4 COMPLETE per-chunk rows (a row is written by several log calls; never compare a half-written one)
+  python3 - "$1" <<'EOF'
+import sys
+out = []
+for l in open(sys.argv[1], errors="replace"):
+    t = l.split()
+    if len(t) > 2 and t[0].isdigit() and l.rstrip().endswith("%") and l.count("%") == 2:
+        out.append(" ".join(t))
+    if len(out) == 4:
+        break
+print("\n".join(out))
+EOF
+}
+check(){  # $1 tag  $2 new file
+  local L=logs/R_kld4_$1.log
+  $B/llama-perplexity -m "$2" --kl-divergence-base kld/bf16.kld --kl-divergence -c 2048 -b 2048 \
+      -ngl 999 -fa on -dio -dev ROCm0 > $L 2>&1 &
+  local pid=$!
+  for i in $(seq 1 900); do
+    [ "$(rows $L | grep -c .)" -ge 4 ] && break
+    kill -0 $pid 2>/dev/null || break
+    sleep 1
+  done
+  kill $pid 2>/dev/null; wait $pid 2>/dev/null
+  local a b v n
+  a=$(rows $L); b=$(rows logs/B5_kld_$1.log); n=$(printf "%s\n" "$a" | grep -c .)
+  if [ "$n" -eq 4 ] && [ "$a" = "$b" ]; then v=MATCH; else v=MISMATCH; fi
+  echo "{\"tag\":\"$1\",\"file\":\"$(basename "$2")\",\"rows_compared\":$n,\"result\":\"$v\"}" | tee -a results/trunk_receipt.jsonl
+}
+check q106  out-mtp/$N-MTP-Q4_0_ROCMFP4_STRIX_LEAN.gguf
+check q102  out-mtp/$N-MTP-Q4_0_ROCMFP4_COHERENT.gguf
+check q115  out-mtp/$N-MTP-Q8_0_ROCMFPX_AGENT.gguf
+check q111  out-mtp/$N-MTP-Q8_0_ROCMFPX.gguf
+check q106i out-mtp-imat/$N-MTP-imatrix-Q4_0_ROCMFP4_STRIX_LEAN.gguf
+check q102i out-mtp-imat/$N-MTP-imatrix-Q4_0_ROCMFP4_COHERENT.gguf
+vis(){  # $1 label  $2 -fa  $3 MTP head (yes|no); patched server, shipped STRIX_LEAN file, vision projector
+  local extra=()
+  [ "$3" = yes ] && extra=(--mtp-infile --nmax 4)
+  AGNES_BIN=/opt/llama-rocm/rocmfpx-724-mtpcache/build-hipvk/bin python3 agnes_harness.py vision \
+    --model out-mtp/$N-MTP-Q4_0_ROCMFP4_STRIX_LEAN.gguf ${extra[@]+"${extra[@]}"} --fa "$2" \
+    --mmproj out/mmproj-$N-BF16.gguf --image calib/vision_probe.png \
+    --question "Describe the two shapes in this image and their colors." --expect "red,blue,circle,square" \
+    --label "$1" --jsonl results/mtpfix.jsonl --serverlog logs/v_$1.log
+  echo "vision $1 exit=$?"
+}
+vis vision-q106-infile       on  yes   # the quick-start configuration
+vis vision-q106-infile-faoff off yes
+vis vision-q106-nomtp        on  no
+vis vision-q106-nomtp-faoff  off no
+echo TRUNK_RECEIPT_DONE

recipe/pipeline/verify_download.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+"""Byte + sha256 gate for a pinned HF snapshot. Compares against the tree API's lfs.oid
+(the real content sha256) -- never the plain etag, which is a Xet/CAS id.
+usage: verify_download.py <repo> <revision> <local_dir>"""
+import hashlib, json, os, sys, urllib.request
+from concurrent.futures import ThreadPoolExecutor
+repo, rev, d = sys.argv[1:4]
+tree = json.load(urllib.request.urlopen(
+    f"https://huggingface.co/api/models/{repo}/tree/{rev}?recursive=true", timeout=60))
+files = [t for t in tree if t.get("type") == "file"]
+def sha(p):
+    h = hashlib.sha256()
+    with open(p, "rb") as f:
+        for b in iter(lambda: f.read(64 << 20), b""): h.update(b)
+    return h.hexdigest()
+def one(t):
+    p = os.path.join(d, t["path"])
+    if not os.path.exists(p): return (t["path"], "MISSING")
+    if os.path.getsize(p) != t["size"]: return (t["path"], f"SIZE {os.path.getsize(p)} != {t['size']}")
+    lfs = t.get("lfs")
+    if lfs:
+        got = sha(p)
+        return (t["path"], "ok-sha" if got == lfs["oid"] else f"SHA {got[:12]} != {lfs['oid'][:12]}")
+    return (t["path"], "ok-size")
+with ThreadPoolExecutor(6) as ex: res = list(ex.map(one, files))
+bad = [r for r in res if not r[1].startswith("ok")]
+print(f"files={len(res)} sha-verified={sum(r[1]=='ok-sha' for r in res)} size-only={sum(r[1]=='ok-size' for r in res)} bad={len(bad)}")
+for r in bad: print("  BAD", r)
+print("RESULT:", "PASS" if not bad else "FAIL")
+sys.exit(0 if not bad else 1)

recipe/pipeline/vplan_a.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+"""Verification of the two MTP fixes (in-file head + prompt-cache patch). Sequential GPU, box iced.
+Writes results/mtpfix.jsonl. Each step is an isolated harness process."""
+import json, os, subprocess, time
+W = "/mnt/models/agnes-3.0-flash"; N = "Agnes-3.0-Flash-Preview"
+J = f"{W}/results/mtpfix.jsonl"
+PB = "/opt/llama-rocm/rocmfpx-724-mtpcache/build-hipvk/bin"      # patched
+OB = "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin"                # original d3ca537
+T = {"q106": f"{W}/out-mtp/{N}-MTP-Q4_0_ROCMFP4_STRIX_LEAN.gguf", "q102": f"{W}/out-mtp/{N}-MTP-Q4_0_ROCMFP4_COHERENT.gguf",
+     "q115": f"{W}/out-mtp/{N}-MTP-Q8_0_ROCMFPX_AGENT.gguf", "q111": f"{W}/out-mtp/{N}-MTP-Q8_0_ROCMFPX.gguf",
+     "q106i": f"{W}/out-mtp-imat/{N}-MTP-imatrix-Q4_0_ROCMFP4_STRIX_LEAN.gguf",
+     "q102i": f"{W}/out-mtp-imat/{N}-MTP-imatrix-Q4_0_ROCMFP4_COHERENT.gguf"}
+def log(m): print(f"[{time.strftime('%FT%TZ', time.gmtime())}] {m}", flush=True)
+def run(cmd, label, model, binp=PB, dev="ROCm0", infile=False, strict=False, nmax=4, reps=3, workload="code", extra=()):
+    a = ["python3", f"{W}/agnes_harness.py", cmd, "--model", model, "--dev", dev, "--label", label,
+         "--reps", str(reps), "--jsonl", J, "--serverlog", f"{W}/logs/v_{label}.log", "--workload", workload, *extra]
+    if infile: a += ["--mtp-infile", "--nmax", str(nmax)]
+    if strict: a += ["--strict"]
+    log(f"{cmd} {label}")
+    r = subprocess.run(a, capture_output=True, text=True, env=dict(os.environ, AGNES_BIN=binp))
+    print(r.stdout.strip()[-900:], flush=True)
+    if r.returncode != 0: print("  RC", r.returncode, r.stderr.strip()[-500:], flush=True)
+    return r.returncode
+def rows(label):
+    out = [json.loads(l) for l in open(J)] if os.path.exists(J) else []
+    return [d for d in out if d.get("label") == label]
+# wait until the STRIX_LEAN in-file-MTP file is fully written
+while "quantize time" not in (open(f"{W}/logs/S2_q106.log").read() if os.path.exists(f"{W}/logs/S2_q106.log") else ""):
+    time.sleep(15)
+# --- 1. output identity: plain decode vs MTP (default) vs MTP (strict) ---
+run("identity", "id-nospec-orig", T["q106"], binp=OB)
+run("identity", "id-nospec-patched", T["q106"])
+run("identity", "id-mtp-default", T["q106"], infile=True)
+run("identity", "id-mtp-strict", T["q106"], infile=True, strict=True)
+ref = rows("id-nospec-orig")[-1]["text"]
+for lab in ("id-nospec-patched", "id-mtp-default", "id-mtp-strict"):
+    t = rows(lab)[-1]["text"]
+    k = next((i for i, (x, y) in enumerate(zip(ref, t)) if x != y), None if len(ref) == len(t) else min(len(ref), len(t)))
+    log(f"IDENTITY {lab} vs plain-decode: {'IDENTICAL' if t == ref else f'DIVERGES at char {k}'}")
+    with open(J, "a") as f: f.write(json.dumps({"label": f"cmp-{lab}", "identical_to_plain": t == ref, "first_diff_char": k}) + "\n")
+# --- 3. prompt cache: unpatched vs patched, reuse + warm/cold identity ---
+run("cachegate", "c-orig-infile", T["q106"], binp=OB, infile=True, reps=1)
+run("cachegate2", "c2-patched-infile", T["q106"], infile=True, reps=3)
+run("cachegate2", "c2-patched-infile-strict", T["q106"], infile=True, strict=True, reps=3)
+run("cachegate2", "c2-patched-nospec", T["q106"], reps=3)
+log("VPLAN_A_DONE")

recipe/pipeline/vplan_b.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+"""Verification of the two MTP fixes (in-file head + prompt-cache patch). Sequential GPU, box iced.
+Writes results/mtpfix.jsonl. Each step is an isolated harness process."""
+import json, os, subprocess, time
+W = "/mnt/models/agnes-3.0-flash"; N = "Agnes-3.0-Flash-Preview"
+J = f"{W}/results/mtpfix.jsonl"
+PB = "/opt/llama-rocm/rocmfpx-724-mtpcache/build-hipvk/bin"      # patched
+OB = "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin"                # original d3ca537
+T = {"q106": f"{W}/out-mtp/{N}-MTP-Q4_0_ROCMFP4_STRIX_LEAN.gguf", "q102": f"{W}/out-mtp/{N}-MTP-Q4_0_ROCMFP4_COHERENT.gguf",
+     "q115": f"{W}/out-mtp/{N}-MTP-Q8_0_ROCMFPX_AGENT.gguf", "q111": f"{W}/out-mtp/{N}-MTP-Q8_0_ROCMFPX.gguf",
+     "q106i": f"{W}/out-mtp-imat/{N}-MTP-imatrix-Q4_0_ROCMFP4_STRIX_LEAN.gguf",
+     "q102i": f"{W}/out-mtp-imat/{N}-MTP-imatrix-Q4_0_ROCMFP4_COHERENT.gguf"}
+def log(m): print(f"[{time.strftime('%FT%TZ', time.gmtime())}] {m}", flush=True)
+def run(cmd, label, model, binp=PB, dev="ROCm0", infile=False, strict=False, nmax=4, reps=3, workload="code", extra=()):
+    a = ["python3", f"{W}/agnes_harness.py", cmd, "--model", model, "--dev", dev, "--label", label,
+         "--reps", str(reps), "--jsonl", J, "--serverlog", f"{W}/logs/v_{label}.log", "--workload", workload, *extra]
+    if infile: a += ["--mtp-infile", "--nmax", str(nmax)]
+    if strict: a += ["--strict"]
+    log(f"{cmd} {label}")
+    r = subprocess.run(a, capture_output=True, text=True, env=dict(os.environ, AGNES_BIN=binp))
+    print(r.stdout.strip()[-900:], flush=True)
+    if r.returncode != 0: print("  RC", r.returncode, r.stderr.strip()[-500:], flush=True)
+    return r.returncode
+def rows(label):
+    out = [json.loads(l) for l in open(J)] if os.path.exists(J) else []
+    return [d for d in out if d.get("label") == label]
+# speed numbers need a quiet box: wait for the whole shrink pipeline (all quants + identity checks)
+while "SHRINK_DONE" not in (open(f"{W}/logs/mtp_bake.log").read() if os.path.exists(f"{W}/logs/mtp_bake.log") else ""):
+    time.sleep(20)
+log("box quiet: " + os.popen("uptime").read().strip())
+# --- 0. chunk-ALIGNED prompt-cache identity: the fair test of the patch, run before any speed work ---
+def gate3(label, model, infile=False, strict=False, reps=10):
+    a = ["python3", f"{W}/cachegate3.py", "--model", model, "--label", label, "--reps", str(reps),
+         "--jsonl", J, "--serverlog", f"{W}/logs/v_{label}.log"]
+    if infile: a += ["--mtp-infile"]
+    if strict: a += ["--strict"]
+    log(f"cachegate3 {label}")
+    r = subprocess.run(a, capture_output=True, text=True, env=dict(os.environ, AGNES_BIN=PB))
+    print(r.stdout.strip()[-3000:], flush=True)
+    if r.returncode != 0: print("  RC", r.returncode, r.stderr.strip()[-800:], flush=True)
+    return r.returncode
+gate3("c3-patched-infile", T["q106"], infile=True, reps=10)
+gate3("c3-patched-infile-strict", T["q106"], infile=True, strict=True, reps=5)
+gate3("c3-patched-nospec", T["q106"], reps=5)
+# --- 2. speed: in-file head, patched binary; knee check n3/n4/n5, both modes ---
+run("bench", "p-nospec-rocm", T["q106"])
+for n in (3, 4, 5):
+    run("bench", f"p-infile-rocm-n{n}", T["q106"], infile=True, nmax=n)
+run("bench", "p-infile-strict-rocm-n4", T["q106"], infile=True, strict=True)
+run("bench", "p-nospec-vk", T["q106"], dev="Vulkan0")
+run("bench", "p-infile-vk-n4", T["q106"], dev="Vulkan0", infile=True)
+run("bench", "p-infile-strict-vk-n4", T["q106"], dev="Vulkan0", infile=True, strict=True)
+# --- 4. every other tier in the new layout, both backends, default MTP at n4 ---
+for tag in ("q106i", "q102", "q102i", "q115", "q111"):
+    for dev, sh in (("ROCm0", "rocm"), ("Vulkan0", "vk")):
+        run("bench", f"t-{tag}-{sh}-n4", T[tag], dev=dev, infile=True)
+# --- 5. prose range + tools on the recommended config ---
+run("bench", "p-infile-rocm-n4-prose", T["q106i"], infile=True, workload="prose")
+run("tools", "tools-infile-patched", T["q106i"], infile=True)
+log("VPLAN_B_DONE")

recipe/verify_fold.log

@@ -0,0 +1,19 @@
+G4 config: ok
+G1 names: 1305 out vs 1305 expected
+  G2 progress 0/1521
+  G2 progress 150/1521
+  G2 progress 300/1521
+  G2 progress 450/1521
+  G2 progress 750/1521
+  G2 progress 900/1521
+  G2 progress 1050/1521
+  G2 progress 1200/1521
+  G2 progress 1350/1521
+  G2 progress 1500/1521
+G2 bit-exact: folded=216 mtp_padded=3 passthrough=1086
+G3 layer  0: max rel err 8.535e-10  (|ref|max 139.673)
+G3 layer 36: max rel err 5.425e-08  (|ref|max 4.395)
+G3 layer 71: max rel err 2.481e-08  (|ref|max 9.612)
+G3 mtp     : max rel err 3.030e-08
+
+RESULT: PASS  (0 failures)