compose: A16-scale-fix image (~1.03M-token KV pool) + JIT-cache persistence

docker-compose.yml

@@ -2,7 +2,7 @@
 # (e.g. RTX PRO 6000, SM120). Works for both the -504B (code-calib) and -504B-term variants.
 #
 # Best config (defaults below): DCP4 + MTP5 + global-topk + DCP-sharded draft + use_index_cache
-#   → 474k-token KV pool; 70-94 tok/s codegen, ~50 tok/s @256k (flat at depth),
+#   → ~1.03M-token KV pool (A16-scale-fix image); 70-94 tok/s codegen, ~50 tok/s @256k (flat at depth),
 #     256k prefill no-wedge, ~200 tok/s aggregate @ concurrency 4.
 #
 # Quickstart:
@@ -11,12 +11,13 @@
 #
 # Tuning via env (no file edits):
 #   max decode speed, short ctx :  DCP_SIZE=1 MTP=1 MAX_MODEL_LEN=125000
-#   absolute max context        :  raise MAX_MODEL_LEN toward the KV pool (~474k @ DCP4)
+#   absolute max context        :  raise MAX_MODEL_LEN toward the KV pool (~1.03M @ DCP4)
 #   no speculative decode       :  MTP=0
 # Sampling: temperature 0.6, top_p 0.95, repetition_penalty 1.0 (do NOT exceed 1.0).
 services:
   glm52-reap:
-    image: ${IMAGE:-madeby561/vllm:dark-devotion-df8ad3b-b12x5af873a-dcpglobaltopk-cu132-20260621-mtpdcpfix}
+    # A16-scale-fix build: frees ~7 GiB/GPU vs the prior image (no stale dequant scales in VRAM) -> ~1.03M-token KV pool on TP4.
+    image: ${IMAGE:-voipmonitor/vllm:glm52-dark-devotion-pr31-pr15-w4a16scale-vllm79f154c-b12xaecc88f-cu132-20260622}
     container_name: ${NAME:-glm52-reap}
     network_mode: host
     ipc: host
@@ -52,6 +53,13 @@ services:
       VLLM_USE_B12X_SPARSE_INDEXER: "1"
       VLLM_DCP_GLOBAL_TOPK: "1"
       VLLM_DCP_SHARD_DRAFT: "1"
+      # JIT/compile-cache persistence -> artifacts land in the mounted ./cache and survive restarts = much faster reboots
+      VLLM_CACHE_DIR: /cache/jit/vllm
+      TRITON_CACHE_DIR: /cache/jit/triton
+      TORCH_EXTENSIONS_DIR: /cache/jit/torch_extensions
+      TORCHINDUCTOR_CACHE_DIR: /cache/jit/torchinductor
+      FLASHINFER_WORKSPACE_BASE: /cache/jit/flashinfer
+      XDG_CACHE_HOME: /cache/jit
       VLLM_USE_V2_MODEL_RUNNER: "1"
       VLLM_ENABLE_PCIE_ALLREDUCE: "1"
       VLLM_PCIE_ALLREDUCE_BACKEND: b12x