Card: GGUF / llama.cpp / Ollama section (EN + ZH), tags; wald-serve 0.1.1 (llama.cpp backend, vLLM path unchanged); MANIFEST with the GGUF files

MANIFEST.json

@@ -20,8 +20,8 @@
     "sha256": "69225fb813578b0d4f9e453a9f0c140c848ec354cf6cca877bcae828c082d3d7"
   },
   "README.md": {
-    "bytes": 28587,
-    "sha256": "7dde057672062641b000143ffaeab5848b19d17fa395c846b79b378cb3746e76"
+    "bytes": 29939,
+    "sha256": "5f02a79558567e33186c0d1e4d462b70a0fa2d960080c2af27b2e08567eb8b98"
   },
   "RUNBOOK.md": {
     "bytes": 4980,
@@ -44,8 +44,8 @@
     "sha256": "e385b92e333eaaa9395a09daf26c09420645bee242456552a55500db45919bc7"
   },
   "docs/readmes/README.zh.md": {
-    "bytes": 24299,
-    "sha256": "3754868c44ea219cc16220d84c70dbfb170e6a8e699b20ab57584d39f366b421"
+    "bytes": 25676,
+    "sha256": "667a7db8482decc260fdf2538de52365f3e130984a173554fb7483e1f5f02d26"
   },
   "evaluation/benchmark-summary.json": {
     "bytes": 36014,
@@ -140,48 +140,48 @@
     "sha256": "d5b7b3e62d43c5f0dae4779f723b1eb4408aa9c21fcbdb6ed65e1da049dd050a"
   },
   "server/README.md": {
-    "bytes": 849,
-    "sha256": "e32014b15ad7e0aa37623d7a5c4561d68f5cd25c9f5514339a8e36d886c1b36f"
+    "bytes": 1678,
+    "sha256": "2ca738dcacd770e7c490ceeb59f2f11df7515f5bc56784eb9d476af4c7145b4d"
   },
   "server/pyproject.toml": {
-    "bytes": 565,
-    "sha256": "f5d1763d77797fab431a8d05b1a457278d5468f1487636545ae290c4b6e7ff53"
+    "bytes": 578,
+    "sha256": "0fd8155bfa466e0f7a95fddce986ab881cce9260187c1620884fa27e5008dfbe"
   },
   "server/src/wald_serve/__init__.py": {
     "bytes": 139,
-    "sha256": "6794303c203fd5ae4dc9246eb4ce75be8006a6e058c9c7f572896e1b880ab961"
+    "sha256": "7db760d45fae0f1d9cea6dbd3e1421a2627a92020981ae3cd56ab57181423aac"
   },
   "server/src/wald_serve/__main__.py": {
     "bytes": 51,
     "sha256": "540fd0a5992ca535482a9d44b2e268ce3a39bb47a0576f301214afa5ef4494b9"
   },
   "server/src/wald_serve/engine.py": {
-    "bytes": 13300,
-    "sha256": "09672d5a560ab5f5f2b9f432d176f6a151b7fcce5ea7729763e8f50af1d6231a"
+    "bytes": 14814,
+    "sha256": "40fbea84c4c5b93ef888da44508d4fec2677c58732a09d731f0bcf17c2b48dd6"
   },
   "server/src/wald_serve/prompt.py": {
     "bytes": 4192,
     "sha256": "bb45a10734f2546eb1ea1bd7e8b5717ecd8305a4aded0b816f80cd7e85188f7a"
   },
   "server/src/wald_serve/server.py": {
-    "bytes": 7195,
-    "sha256": "6eaf25b7b9eecf3d375b7f9b87ed4b16de211af7e5689966287c76e1e7733cfd"
+    "bytes": 8878,
+    "sha256": "49fb06ee52eb2ebe7059404432386d02e049556003acb7d2f6494ba92d4b9703"
   },
   "server/src/wald_serve/wire.py": {
     "bytes": 3670,
     "sha256": "836ce54ce93016a98f94a6beaa508cc03982e24c3d3b4d62b8ef667389cd14af"
   },
   "server/tests/fakevllm.py": {
-    "bytes": 2895,
-    "sha256": "e1a6c2cdf113d491bad0cb8efc292d1323c16ee70609ee0dfe81a8fab222a08e"
+    "bytes": 4167,
+    "sha256": "9fb222a21bc2b17221e5a1a3605973b4f71730db9bbfede439bd3388b91e2570"
   },
   "server/tests/test_parity.py": {
     "bytes": 4392,
     "sha256": "cd1e4142090076114f2b07efce49fdc5466e302a5c79996b993f3833a3457566"
   },
   "server/tests/test_server.py": {
-    "bytes": 11924,
-    "sha256": "8f8fecbcff18abb6b44da743f6e041472124ba4188881f9e75845348e46ca405"
+    "bytes": 13844,
+    "sha256": "46f2007e9743bfeb0f7f79fca093cf444a70df1bfc348229f34cd251f88a19b3"
   },
   "serving.json": {
     "bytes": 128,
@@ -242,5 +242,21 @@
   "evaluation/v1.2/release-check.json": {
     "bytes": 3036,
     "sha256": "0de642d7c1ddb2ececafce16c06ae2d2eb0e663ec94b4afcfab605a61aadfb70"
+  },
+  "Wald-4B-v1.2-Q4_K_M.gguf": {
+    "bytes": 2708804000,
+    "sha256": "e843f7658793b9f3effb3a4941cd67f80707a07b468aaaf94d440613751d76fa"
+  },
+  "Wald-4B-v1.2-Q5_K_M.gguf": {
+    "bytes": 3074986400,
+    "sha256": "6fecc4e7655adb71e8c642ce5018f967564ba259852d562ecd3109386542b0cb"
+  },
+  "Wald-4B-v1.2-Q6_K.gguf": {
+    "bytes": 3464055200,
+    "sha256": "5870f15f3b2eef773f672e56bb36e343a95e32fd09baaf7c9ce8b0da1e5dd897"
+  },
+  "Wald-4B-v1.2-Q8_0.gguf": {
+    "bytes": 4482402720,
+    "sha256": "09c02494ebdfb7d088a2c76726eb2b290a438dbc5658c3be16da8333fddd6038"
   }
 }

README.md

@@ -31,6 +31,9 @@ tags:
 - qwen3.5
 - 4b
 - vllm
+- gguf
+- llama.cpp
+- ollama
 - reasoning
 model-index:
 - name: Wald-Q4B v1.1 (revision v1.1)
@@ -202,6 +205,27 @@ curl http://localhost:8000/v1/systemone \
 
 The answer for `route` contains the chosen key and a probability for each of `returns`, `delivery` and `other`. Request and response fields, yes/no and score questions, and a clarification example: [API reference](https://huggingface.co/org2ai/Wald-4B/blob/main/docs/api.md). `GET /health` reports the effective policy. The server uses vLLM 0.30.0 and the included `wald-serve` package; Docker and exact evaluation settings are in [RUNBOOK.md](https://huggingface.co/org2ai/Wald-4B/blob/main/RUNBOOK.md). Generic text-generation calls do not reproduce the decision API's readout.
 
+## GGUF: llama.cpp, Ollama, LM Studio
+
+`main` also carries v1.2 as GGUF files for CPUs, Apple Silicon and consumer GPUs (llama.cpp `b11312`, the same weights). Parity on the JevBench public set (231 items, `none`), against the BF16 weights on vLLM (204/231):
+
+| File | Size | JevBench public | Same option as BF16 |
+|---|---:|---:|---:|
+| `Wald-4B-v1.2-Q8_0.gguf` | 4.5 GB | 206/231 | 229/231 |
+| `Wald-4B-v1.2-Q6_K.gguf` | 3.5 GB | 205/231 | 227/231 |
+| `Wald-4B-v1.2-Q5_K_M.gguf` | 3.1 GB | 205/231 | 227/231 |
+| `Wald-4B-v1.2-Q4_K_M.gguf` | 2.7 GB | 202/231 | 223/231 |
+
+For calibrated probabilities, run the included server on llama.cpp (`llama-server` on your `PATH`):
+
+```sh
+hf download org2ai/Wald-4B --include "Wald-4B-v1.2-Q8_0.gguf" "serving.json" "temperature.json" "server/*" --local-dir ./wald-gguf
+pip install ./wald-gguf/server
+wald-serve --gguf ./wald-gguf/Wald-4B-v1.2-Q8_0.gguf --max-model-len 32768 --port 8000
+```
+
+Ollama: `ollama run hf.co/org2ai/Wald-4B:Q4_K_M`. LM Studio: search for `Wald-4B`. Neither app was tested with these files, and a chat session returns text, not the option probabilities. Download a single file with `--include`; a plain `hf download org2ai/Wald-4B` of `main` also fetches all four GGUFs (14 GB). Details: [org2ai/Wald-4B-GGUF](https://huggingface.co/org2ai/Wald-4B-GGUF).
+
 ## Thinking effort
 
 | Effort | When it thinks | Thought budget |

docs/readmes/README.zh.md

@@ -83,6 +83,27 @@ curl http://localhost:8000/v1/systemone \
 
 `route` 的答案包含选中的键，以及 `returns`、`delivery`、`other` 各自的概率。请求与响应字段、是非题和评分题、澄清判断示例见 [API 说明](https://huggingface.co/org2ai/Wald-4B/blob/main/docs/api.md)。`GET /health` 返回当前生效策略。服务使用 vLLM 0.30.0 和仓库内的 `wald-serve`；Docker 与精确评测配置见 [RUNBOOK.md](https://huggingface.co/org2ai/Wald-4B/blob/main/RUNBOOK.md)。普通文本生成接口不会复现决策 API 的读出流程。
 
+## GGUF：llama.cpp、Ollama、LM Studio
+
+`main` 上另有 v1.2 的 GGUF 文件，适合 CPU、Apple Silicon 和消费级显卡（llama.cpp `b11312`，同一份权重）。在 JevBench 公开集（231 题，`none`）上与 vLLM 上的 BF16 权重（204/231）对照：
+
+| 文件 | 大小 | JevBench 公开集 | 与 BF16 选同一选项 |
+|---|---:|---:|---:|
+| `Wald-4B-v1.2-Q8_0.gguf` | 4.5 GB | 206/231 | 229/231 |
+| `Wald-4B-v1.2-Q6_K.gguf` | 3.5 GB | 205/231 | 227/231 |
+| `Wald-4B-v1.2-Q5_K_M.gguf` | 3.1 GB | 205/231 | 227/231 |
+| `Wald-4B-v1.2-Q4_K_M.gguf` | 2.7 GB | 202/231 | 223/231 |
+
+要拿到校准后的概率，用自带的服务跑在 llama.cpp 上（`llama-server` 需在 `PATH` 里）：
+
+```sh
+hf download org2ai/Wald-4B --include "Wald-4B-v1.2-Q8_0.gguf" "serving.json" "temperature.json" "server/*" --local-dir ./wald-gguf
+pip install ./wald-gguf/server
+wald-serve --gguf ./wald-gguf/Wald-4B-v1.2-Q8_0.gguf --max-model-len 32768 --port 8000
+```
+
+Ollama：`ollama run hf.co/org2ai/Wald-4B:Q4_K_M`。LM Studio：搜索 `Wald-4B`。这两个应用都没有用这些文件实测过；聊天界面返回的是文字，不是各选项的概率。请用 `--include` 只下载一个文件；直接 `hf download org2ai/Wald-4B` 下载 `main` 会把四个 GGUF（14 GB）一起拉下来。详见 [org2ai/Wald-4B-GGUF](https://huggingface.co/org2ai/Wald-4B-GGUF)。
+
 ## 思考程度
 
 | Effort | 何时思考 | 思考预算 |

server/README.md

@@ -1,14 +1,28 @@
 # wald-serve
 
-The TypeSafe `POST /v1/systemone` server for Wald-4B: the option-letter readout over a vLLM OpenAI-compatible server,
-the effort gate (none / low / medium / high / high-k), the knockout for more than 26 options, and the temperature table.
-It needs only public packages: `pydantic`, plus `vllm==0.30.0` on the GPU machine.
+The TypeSafe `POST /v1/systemone` server for Wald-4B: the option-letter readout over a vLLM OpenAI-compatible server or
+llama.cpp's `llama-server`, the effort gate (none / low / medium / high / high-k), the knockout for more than 26
+options, and the temperature table. It needs only public packages: `pydantic`, plus `vllm==0.30.0` or llama.cpp on the
+machine that runs the weights.
 
 ```sh
 pip install ".[vllm]"                              # on a Linux CUDA machine
 wald-serve --model /path/to/Wald-4B --port 8000    # starts vLLM on the weights and serves /v1/systemone
 ```
 
-Tests (no model, no GPU; a fake vLLM stands in): `pip install ".[test]" && cd tests && pytest -q`.
+GGUF weights (CPU, Apple Silicon or any GPU llama.cpp supports):
+
+```sh
+pip install .
+wald-serve --gguf ./Wald-4B-Q8_0.gguf --port 8000                     # starts llama-server (on PATH) on the file
+wald-serve --llamacpp http://127.0.0.1:8080 --temperature temperature.json \
+  --prompt-format repeat_state_plain --effort none --port 8000        # or attach to a running llama-server
+```
+
+With `--gguf`, `serving.json` and `temperature.json` are read from the GGUF's folder. The llama.cpp backend sends token
+ids to `/completion` and reads the letters from its top 100 next-token logprobs (`n_probs`); a letter outside them gets
+the same floor as in vLLM. llama.cpp's context is `--max-model-len` × `--llama-parallel` (default 4 slots).
+
+Tests (no model, no GPU; a fake server stands in for vLLM and llama-server): `pip install ".[test]" && cd tests && pytest -q`.
 `tests/test_parity.py` compares prompts and answers byte for byte with the reference implementation the published
 numbers were measured with; it is skipped unless that implementation is importable (WALD_PARITY_SRC set).

server/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "wald-serve"
-version = "0.1.0"
-description = "TypeSafe POST /v1/systemone server for Wald-4B: option-letter readout over vLLM, effort gate, knockout, calibration"
+version = "0.1.1"
+description = "TypeSafe POST /v1/systemone server for Wald-4B: option-letter readout over vLLM or llama.cpp, effort gate, knockout, calibration"
 readme = "README.md"
 license = { text = "Apache-2.0" }
 requires-python = ">=3.10"

server/src/wald_serve/__init__.py

@@ -1,2 +1,2 @@
 """Wald-4B serving: TypeSafe `POST /v1/systemone` over vLLM (letter readout, effort gate, knockout, calibration)."""
-__version__ = "0.1.0"
+__version__ = "0.1.1"

server/src/wald_serve/engine.py

@@ -136,16 +136,18 @@ class Client:
     def ids(self, text: str) -> list[int]:
         return self.post("/tokenize", {"model": self.served, "prompt": text, "add_special_tokens": False})["tokens"]
 
-    def readout(self, ids: list[int], n: int):
-        """-> (softmax over the first n letters, total letter mass)."""
-        if len(ids) + 1 > self.max_len:
-            raise Capacity(f"prompt of {len(ids)} tokens is longer than the maximum context length {self.max_len}")
-        want = [t for L in self.letter_ids[:n] for t in L]
+    def letter_logprobs(self, ids: list[int], want: list[int]) -> dict:
         body = {"model": self.served, "prompt": ids, "max_tokens": 1, "temperature": 0.0, "logprobs": 20,
                 "logprob_token_ids": want, "return_tokens_as_token_ids": True}
         r = self.post("/v1/completions", body)
         top = r["choices"][0]["logprobs"]["top_logprobs"][0]
-        lp = {int(k.split(":", 1)[1]): v for k, v in top.items() if k.startswith("token_id:") and v is not None and v > -9999}
+        return {int(k.split(":", 1)[1]): v for k, v in top.items() if k.startswith("token_id:") and v is not None and v > -9999}
+
+    def readout(self, ids: list[int], n: int):
+        """-> (softmax over the first n letters, total letter mass)."""
+        if len(ids) + 1 > self.max_len:
+            raise Capacity(f"prompt of {len(ids)} tokens is longer than the maximum context length {self.max_len}")
+        lp = self.letter_logprobs(ids, [t for L in self.letter_ids[:n] for t in L])
         floor = min(lp.values()) - 2.0 if lp else -30.0
         z = [math.log(sum(math.exp(lp.get(t, floor)) for t in L)) for L in self.letter_ids[:n]]
         m = max(z)
@@ -167,6 +169,33 @@ class Client:
         return c.get("text") or "", ntok
 
 
+class LlamaCppClient(Client):
+    """The same reads over llama.cpp's `llama-server` (GGUF weights): token ids in, raw next-token logprobs out.
+    llama-server reports the top `n_probs` tokens only, so a letter outside them gets the same floor as in vLLM."""
+
+    N_PROBS = 100
+
+    def ids(self, text: str) -> list[int]:
+        return self.post("/tokenize", {"content": text, "add_special": False, "parse_special": False})["tokens"]
+
+    def letter_logprobs(self, ids: list[int], want: list[int]) -> dict:
+        body = {"prompt": ids, "n_predict": 1, "temperature": 0.0, "n_probs": self.N_PROBS, "post_sampling_probs": False,
+                "cache_prompt": True, "samplers": ["top_k"], "top_k": 1}
+        r = self.post("/completion", body)
+        top = r["completion_probabilities"][0]["top_logprobs"]
+        return {int(t["id"]): float(t["logprob"]) for t in top}
+
+    def generate(self, ids: list[int], max_tokens: int, seed: int, n: int = 1):
+        texts, ntok = [], 0
+        for i in range(n):
+            body = {"prompt": ids, "n_predict": max_tokens, "temperature": 0.6, "top_p": 0.95, "top_k": 20, "min_p": 0.0,
+                    "seed": seed + i, "stop": [STOP], "cache_prompt": True}
+            r = self.post("/completion", body)
+            texts.append(r.get("content") or "")
+            ntok += int(r.get("tokens_predicted") or 0)
+        return (texts[0] if n == 1 else texts), ntok
+
+
 # --- one question -----------------------------------------------------------------------------------------------------
 def chunks(n: int, size: int = 26) -> list[list[int]]:
     k = math.ceil(n / size)

server/src/wald_serve/server.py

@@ -8,6 +8,11 @@ or attaches to a vLLM server that is already running:
 
     wald-serve --vllm http://127.0.0.1:8011 --served wald --temperature /path/to/temperature.json --port 8000
 
+GGUF weights run on llama.cpp instead (`llama-server` on PATH, or --llama-server):
+
+    wald-serve --gguf Wald-4B-Q8_0.gguf --temperature temperature.json --port 8000
+    wald-serve --llamacpp http://127.0.0.1:8080 --temperature temperature.json --port 8000
+
 Defaults come from `<model>/serving.json` when present (`effort`, `prompt_format`, `max_model_len`), else the built-in
 ones below; command-line flags override both. The response carries TypeSafe's answer keys plus `mode` (A = one pass,
 B = after a thought, K = knockout) and `usage`.
@@ -30,7 +35,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from pathlib import Path
 
 from . import __version__
-from .engine import Capacity, Client, answer, load_tables, policy
+from .engine import Capacity, Client, LlamaCppClient, answer, load_tables, policy
 from .prompt import PROMPT_FORMATS
 
 DEFAULTS = {"effort": "medium", "prompt_format": "plain", "max_model_len": 131072}
@@ -47,6 +52,16 @@ def launch_vllm(model: str, served: str, port: int, max_len: int, mem: float, ex
     cmd = [sys.executable, "-m", "vllm.entrypoints.openai.api_server", "--model", model, "--served-model-name", served,
            "--host", "127.0.0.1", "--port", str(port), "--max-model-len", str(max_len), "--gpu-memory-utilization",
            str(mem), "--max-num-seqs", "256", "--seed", "0", *shlex.split(extra)]
+    return launch(cmd, port, "vLLM")
+
+
+def launch_llama(binary: str, gguf: str, port: int, max_len: int, parallel: int, extra: str) -> subprocess.Popen:
+    cmd = [binary, "-m", gguf, "--host", "127.0.0.1", "--port", str(port), "-c", str(max_len * parallel),
+           "-np", str(parallel), "-ngl", "999", "--seed", "0", *shlex.split(extra)]
+    return launch(cmd, port, "llama-server")
+
+
+def launch(cmd: list, port: int, name: str) -> subprocess.Popen:
     print(json.dumps({"launching": cmd}), flush=True)
     proc = subprocess.Popen(cmd, start_new_session=True)
 
@@ -58,13 +73,13 @@ def launch_vllm(model: str, served: str, port: int, max_len: int, mem: float, ex
     url = f"http://127.0.0.1:{port}/health"
     for _ in range(1200):
         if proc.poll() is not None:
-            raise SystemExit(f"vLLM exited with code {proc.returncode}")
+            raise SystemExit(f"{name} exited with code {proc.returncode}")
         try:
             with urllib.request.urlopen(url, timeout=5):
                 return proc
         except OSError:
             time.sleep(2)
-    raise SystemExit("vLLM did not become healthy within 40 minutes")
+    raise SystemExit(f"{name} did not become healthy within 40 minutes")
 
 
 def make_handler(cl: Client, pol: dict, tables: dict, info: dict, workers: int):
@@ -109,6 +124,9 @@ def main(argv=None):
     src = ap.add_mutually_exclusive_group(required=True)
     src.add_argument("--model", help="weights directory: start vLLM on it (serving.json / temperature.json read from it)")
     src.add_argument("--vllm", help="URL of a running vLLM OpenAI-compatible server")
+    src.add_argument("--gguf", help="GGUF weights file: start llama.cpp's llama-server on it (serving.json / temperature.json "
+                     "read from its folder)")
+    src.add_argument("--llamacpp", help="URL of a running llama.cpp llama-server")
     ap.add_argument("--served", default="wald", help="vLLM served model name")
     ap.add_argument("--model-name", default="wald-4b", help="name reported in responses")
     ap.add_argument("--effort", default=None, help="none | low | medium | high | high-k<k> (default: serving.json, else medium)")
@@ -119,16 +137,20 @@ def main(argv=None):
     ap.add_argument("--vllm-port", type=int, default=8011)
     ap.add_argument("--gpu-memory-utilization", type=float, default=0.90)
     ap.add_argument("--vllm-args", default="", help="extra `vllm serve` arguments, e.g. \"--quantization fp8\"")
+    ap.add_argument("--llama-server", default="llama-server", help="llama-server binary for --gguf")
+    ap.add_argument("--llama-parallel", type=int, default=4, help="llama-server slots for --gguf (context = max-model-len x slots)")
+    ap.add_argument("--llama-args", default="", help="extra llama-server arguments for --gguf")
     ap.add_argument("--host", default="0.0.0.0")
     ap.add_argument("--port", type=int, default=8000)
     a = ap.parse_args(argv)
 
-    cfg = {**DEFAULTS, **read_serving(a.model)}
+    home = a.model or (str(Path(a.gguf).parent) if a.gguf else None)
+    cfg = {**DEFAULTS, **read_serving(home)}
     effort = a.effort or cfg["effort"]
     fmt = a.prompt_format or cfg["prompt_format"]
     max_len = a.max_model_len or int(cfg["max_model_len"])
     pol = policy(effort)
-    temps = a.temperature or (str(Path(a.model) / cfg.get("temperature", "temperature.json")) if a.model else None)
+    temps = a.temperature or (str(Path(home) / cfg.get("temperature", "temperature.json")) if home else None)
     if temps and not Path(temps).is_file():
         raise SystemExit(f"temperature table not found: {temps}")
     tables = load_tables(temps, pol["budget"])
@@ -136,10 +158,14 @@ def main(argv=None):
     if a.model:
         launch_vllm(a.model, a.served, a.vllm_port, max_len, a.gpu_memory_utilization, a.vllm_args)
         endpoint = f"http://127.0.0.1:{a.vllm_port}"
+    elif a.gguf:
+        launch_llama(a.llama_server, a.gguf, a.vllm_port, max_len, a.llama_parallel, a.llama_args)
+        endpoint = f"http://127.0.0.1:{a.vllm_port}"
     else:
-        endpoint = a.vllm
-    cl = Client(endpoint, a.served, max_len, prompt_format=fmt)
-    info = {"model": a.model_name, "version": __version__, "effort": pol["name"], "gate": pol["gate"],
+        endpoint = a.vllm or a.llamacpp
+    cl = (LlamaCppClient if a.gguf or a.llamacpp else Client)(endpoint, a.served, max_len, prompt_format=fmt)
+    info = {"model": a.model_name, "version": __version__, "backend": "llama.cpp" if a.gguf or a.llamacpp else "vllm",
+            "effort": pol["name"], "gate": pol["gate"],
             "budget": pol["budget"], "think_k": pol["k"], "prompt_format": fmt, "max_model_len": max_len,
             "wide": "knockout", "max_options": 676, "temperature": bool(tables)}
     ThreadingHTTPServer.daemon_threads = True

server/tests/fakevllm.py

@@ -3,6 +3,9 @@
 Tokenizer: one token per character (so "A" is one token and " A" is two). Completions: the logprob of every requested
 token id is a deterministic function of the prompt ids, so a read is reproducible; generation returns a short fixed
 thought per seed. A prompt longer than `max_len` answers 400 "maximum context length", as vLLM does.
+
+The same server also answers llama.cpp's llama-server shapes (`/tokenize` with `content`, `/completion` with `n_probs`),
+reporting the same logprobs for the printable-ASCII token ids, so both clients see one model.
 """
 from __future__ import annotations
 
@@ -44,7 +47,9 @@ class FakeVLLM:
                 body = json.loads(self.rfile.read(int(self.headers["content-length"])))
                 if self.path == "/tokenize":
                     fake.calls["tokenize"] += 1
-                    return self.send(200, {"tokens": [ord(c) for c in body["prompt"]]})
+                    return self.send(200, {"tokens": [ord(c) for c in body.get("prompt", body.get("content"))]})
+                if self.path == "/completion":
+                    return self.llama(body)
                 prompt = body["prompt"]
                 if len(prompt) + body["max_tokens"] > fake.max_len:
                     return self.send(400, {"message": f"This model's maximum context length is {fake.max_len} tokens."})
@@ -57,6 +62,20 @@ class FakeVLLM:
                 texts = [f" thought {body['seed'] % 997} #{i}: the evidence points one way." for i in range(n)]
                 return self.send(200, {"choices": [{"text": t} for t in texts], "usage": {"completion_tokens": 9 * n}})
 
+            def llama(self, body):
+                prompt = body["prompt"]
+                if len(prompt) + body["n_predict"] > fake.max_len:
+                    return self.send(400, {"error": {"message": "the request exceeds the available context size"}})
+                if body["n_predict"] == 1:
+                    fake.calls["read"] += 1
+                    top = sorted(({"id": t, "token": chr(t), "logprob": _lp(prompt, t)} for t in range(32, 127)),
+                                 key=lambda x: -x["logprob"])[: body["n_probs"]]
+                    return self.send(200, {"content": top[0]["token"],
+                                           "completion_probabilities": [{**top[0], "top_logprobs": top}]})
+                fake.calls["generate"] += 1
+                return self.send(200, {"content": f" thought {body['seed'] % 997}: the evidence points one way.",
+                                       "tokens_predicted": 9})
+
         self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), H)
         self.httpd.daemon_threads = True
         self.url = f"http://127.0.0.1:{self.httpd.server_address[1]}"

server/tests/test_server.py

@@ -12,7 +12,7 @@ from pathlib import Path
 import pytest
 
 from fakevllm import FakeVLLM
-from wald_serve.engine import (POLICIES, Client, answer, bucket_key, chunks, load_tables, policy, temper, temperature,
+from wald_serve.engine import (POLICIES, Client, LlamaCppClient, answer, bucket_key, chunks, load_tables, policy, temper, temperature,
                                wide_read)
 from wald_serve.prompt import STATE_REPEAT, format_prompt, question_prompt
 from wald_serve.server import make_handler
@@ -249,3 +249,47 @@ def test_thought_that_does_not_fit_falls_back(fake):
     cl = Client(fake.url, "wald", 400)   # the prompt fits, prompt + 512-token budget does not
     a, u = answer(cl, q, policy("high"), {})
     assert a["q"]["mode"] == "A" and u["output_tokens"] == 0
+
+
+def test_llamacpp_client_matches_vllm_client(fake):
+    vl, lc = Client(fake.url, "wald", 100_000), LlamaCppClient(fake.url, "wald", 100_000)
+    assert lc.letter_ids == vl.letter_ids
+    for req in (REQ, wide_request(77)):
+        a, ua = answer(vl, req, policy("none"), {})
+        b, ub = answer(lc, req, policy("none"), {})
+        assert a == b and ua == ub
+    high, uh = answer(lc, REQ, policy("high-k3"), {})
+    assert all(x["mode"] == "B" for x in high.values()) and uh["output_tokens"] > 0
+    check_wire(REQ, {"answers": high})
+
+
+def test_llamacpp_capacity_is_422(fake):
+    cl = LlamaCppClient(fake.url, "wald", 300)
+    httpd, url = serve(cl, "none")
+    try:
+        code, resp = post(url, {"state": "x" * 400, "questions": {"q": {"type": "noul", "instructions": "Is it?"}}})
+        assert code == 422 and "maximum context length" in resp["error"]
+    finally:
+        httpd.shutdown()
+
+
+def test_gguf_reads_serving_json_beside_the_file(tmp_path, monkeypatch):
+    import wald_serve.server as srv
+    (tmp_path / "serving.json").write_text(json.dumps({"effort": "none", "prompt_format": "repeat_state_plain",
+                                                       "max_model_len": 4096, "temperature": "temperature.json"}))
+    (tmp_path / "temperature.json").write_text(json.dumps(TABLE))
+    seen = {}
+    monkeypatch.setattr(srv, "launch_llama", lambda *a: seen.update(launch=a))
+    monkeypatch.setattr(srv, "LlamaCppClient", lambda *a, **k: seen.update(client=(a, k)))
+
+    class Stop(Exception):
+        pass
+
+    def fake_http(addr, handler):
+        seen["handler"] = handler
+        raise Stop
+    monkeypatch.setattr(srv, "ThreadingHTTPServer", fake_http)
+    with pytest.raises(Stop):
+        srv.main(["--gguf", str(tmp_path / "Wald-4B-Q8_0.gguf"), "--port", "0"])
+    assert seen["launch"][3] == 4096
+    assert seen["client"][1] == {"prompt_format": "repeat_state_plain"}