Update Dockerfile

Dockerfile

@@ -1,25 +1,68 @@
-FROM ghcr.io/ggml-org/llama.cpp:full
+# Switch away from ghcr.io/ggml-org/llama.cpp:full —
+# MTP requires a custom build from the mtp-clean branch.
+FROM ubuntu:24.04
 
 WORKDIR /app
 
-RUN apt update && apt install -y python3 python3-pip python3-venv
+# ── 1. System deps for building llama.cpp (CPU-only) ──────────────────────────
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        build-essential cmake git curl \
+        libcurl4-openssl-dev ca-certificates \
+        python3 python3-pip python3-venv \
+    && rm -rf /var/lib/apt/lists/*
+
+# ── 2. Build llama.cpp from the MTP branch (CPU, no CUDA) ─────────────────────
+# The standard ghcr.io llama.cpp image does NOT include MTP support.
+# mtp-clean is the upstream PR branch that adds --spec-type draft-mtp.
+RUN git clone --depth 1 -b mtp-clean \
+        https://github.com/am17an/llama.cpp.git /opt/llama.cpp && \
+    cmake /opt/llama.cpp -B /opt/llama.cpp/build \
+        -DBUILD_SHARED_LIBS=OFF \
+        -DGGML_CUDA=OFF \        
+        -DLLAMA_CURL=ON && \
+    cmake --build /opt/llama.cpp/build \
+        --config Release -j$(nproc) \
+        --clean-first \
+        --target llama-server && \
+    cp /opt/llama.cpp/build/bin/llama-server /usr/local/bin/llama-server && \
+    rm -rf /opt/llama.cpp   # free disk space after build
+
+# ── 3. Python env + huggingface_hub ───────────────────────────────────────────
 RUN python3 -m venv /opt/venv
 ENV PATH="/opt/venv/bin:$PATH"
+RUN pip install -U pip huggingface_hub hf_transfer
 
-RUN pip install -U pip huggingface_hub
+# ── 4. Download MTP GGUF (4-bit dynamic, ~5.97 GB) ────────────────────────────
+# NOTE: --mmproj is NOT supported with MTP — do NOT download it.
+# The MTP repo embeds the draft model heads inside the single GGUF file.
+ENV HF_HUB_ENABLE_HF_TRANSFER=1
+RUN python3 -c '\
+from huggingface_hub import hf_hub_download; \
+hf_hub_download( \
+    repo_id="unsloth/Qwen3.5-9B-MTP-GGUF", \
+    filename="Qwen3.5-9B-UD-Q4_K_XL.gguf", \
+    local_dir="/app" \
+)'
 
-# Download MTP model (verify exact filename in repo if different)
-RUN python3 -c 'from huggingface_hub import hf_hub_download; \
-    repo="unsloth/Qwen3.5-9B-MTP-GGUF"; \
-    hf_hub_download(repo_id=repo, filename="Qwen3.5-9B-MTP-Q4_K_M.gguf", local_dir="/app")'
+EXPOSE 7860
 
+# ── 5. Launch server with MTP speculative decoding ────────────────────────────
+# Changes vs your original:
+#   • -m  → new MTP GGUF filename
+#   • removed --mmproj   (unsupported with MTP)
+#   • removed -ngl       (CPU has no GPU layers)
+#   • added --spec-type draft-mtp  (MTP speculative decoding, ~1.5-2x faster)
+#   • added --spec-draft-n-max 2   (draft 2 tokens ahead; raise to 4 for ngram)
+#   • -c reduced to 16384          (64 k ctx is very RAM-heavy on CPU)
+#   • -n reduced to 8192           (generation budget; tune as needed)
 CMD ["llama-server", \
-     "--model", "/app/Qwen3.5-9B-MTP-Q4_K_M.gguf", \
+     "-m", "/app/Qwen3.5-9B-UD-Q4_K_XL.gguf", \
      "--host", "0.0.0.0", \
      "--port", "7860", \
      "-t", "2", \
      "--cache-type-k", "q8_0", \
      "--cache-type-v", "iq4_nl", \
-     "-c", "64000", \
-     "--n-predict", "38912", \
-     "--mtp"]
+     "-c", "64l84", \
+     "-n", "8192", \
+     "--spec-type", "draft-mtp", \
+     "--spec-draft-n-max", "2"]