File size: 5,344 Bytes
68f3404 dadc89e 5097530 68f3404 5097530 68f3404 5097530 68f3404 5097530 68f3404 e341fbb 68f3404 dadc89e 68f3404 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 | # GLM-5.2-NVFP4-REAP (168 experts / ~504B) — vLLM serving on 4× 96 GB Blackwell GPUs
# (e.g. RTX PRO 6000, SM120). Works for both the -504B (code-calib) and -504B-term variants.
#
# Best config (defaults below): DCP4 + MTP5 + global-topk + DCP-sharded draft
# → 474k-token KV pool; 70-94 tok/s codegen, ~50 tok/s @256k (flat at depth),
# 256k prefill no-wedge, ~200 tok/s aggregate @ concurrency 4.
#
# Quickstart:
# MODEL_DIR=/path/to/GLM-5.2-NVFP4-REAP-504B-term docker compose up -d
# # serves an OpenAI API on :5001, model id "GLM-5.2"
#
# Tuning via env (no file edits):
# max decode speed, short ctx : DCP_SIZE=1 MTP=1 MAX_MODEL_LEN=125000
# absolute max context : raise MAX_MODEL_LEN toward the KV pool (~474k @ DCP4)
# no speculative decode : MTP=0
# Sampling: temperature 0.6, top_p 0.95, repetition_penalty 1.0 (do NOT exceed 1.0).
services:
glm52-reap:
image: ${IMAGE:-madeby561/vllm:dark-devotion-df8ad3b-b12x5af873a-dcpglobaltopk-cu132-20260621-mtpdcpfix}
container_name: ${NAME:-glm52-reap}
network_mode: host
ipc: host
shm_size: 32g
runtime: nvidia
gpus: all
restart: "no"
ulimits:
memlock: -1
stack: 67108864
volumes:
- ${MODEL_DIR:?set MODEL_DIR to the model directory}:/model:ro
- ./cache:/cache
- ./cache/huggingface:/root/.cache/huggingface
environment:
CUDA_VISIBLE_DEVICES: ${CUDA_VISIBLE_DEVICES:-0,1,2,3}
CUDA_DEVICE_ORDER: PCI_BUS_ID
CUDA_DEVICE_MAX_CONNECTIONS: "32"
OMP_NUM_THREADS: "16"
CUTE_DSL_ARCH: sm_120a
NCCL_IB_DISABLE: "1"
NCCL_P2P_LEVEL: SYS
NCCL_PROTO: LL,LL128,Simple
PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
SAFETENSORS_FAST_GPU: "1"
VLLM_WORKER_MULTIPROC_METHOD: spawn
VLLM_USE_AOT_COMPILE: "1"
VLLM_USE_BREAKABLE_CUDAGRAPH: "0"
VLLM_USE_MEGA_AOT_ARTIFACT: "1"
VLLM_USE_FLASHINFER_SAMPLER: "1"
VLLM_USE_B12X_FP8_GEMM: "1"
VLLM_USE_B12X_MOE: "1"
VLLM_USE_B12X_SPARSE_INDEXER: "1"
VLLM_DCP_GLOBAL_TOPK: "1"
VLLM_DCP_SHARD_DRAFT: "1"
VLLM_USE_V2_MODEL_RUNNER: "1"
VLLM_ENABLE_PCIE_ALLREDUCE: "1"
VLLM_PCIE_ALLREDUCE_BACKEND: b12x
VLLM_PCIE_ONESHOT_ALLREDUCE_MAX_SIZE: 64KB
USES_B12X: "True"
B12X_DENSE_SPLITK_TURBO: "1"
B12X_W4A16_TC_DECODE: "1"
B12X_MOE_FORCE_A16: "1"
MODEL: ${MODEL:-/model}
MTP_MODEL: ${MTP_MODEL:-/model}
SERVED_MODEL_NAME: ${SERVED_MODEL_NAME:-GLM-5.2}
PORT: ${PORT:-5001}
TP_SIZE: ${TP_SIZE:-4}
DCP_SIZE: ${DCP_SIZE:-4}
MTP: ${MTP:-1}
GPU_MEMORY_UTILIZATION: ${GPU_MEMORY_UTILIZATION:-0.95}
MAX_MODEL_LEN: ${MAX_MODEL_LEN:-300000}
MAX_NUM_SEQS: ${MAX_NUM_SEQS:-8}
MAX_NUM_BATCHED_TOKENS: ${MAX_NUM_BATCHED_TOKENS:-8192}
NUM_SPECULATIVE_TOKENS: ${NUM_SPECULATIVE_TOKENS:-5}
MAX_CUDAGRAPH_CAPTURE_SIZE: ${MAX_CUDAGRAPH_CAPTURE_SIZE:-}
SPEC_CONFIG: ${SPEC_CONFIG:-}
HF_OVERRIDES: '{"index_topk_pattern":"FFFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSS"}'
entrypoint:
- bash
- -lc
- |
set -euo pipefail
unset NCCL_GRAPH_FILE NCCL_GRAPH_DUMP_FILE VLLM_B12X_MLA_EXTEND_MAX_CHUNKS
if [ -z "$${MAX_CUDAGRAPH_CAPTURE_SIZE:-}" ]; then
if [ "$${MTP}" = "1" ]; then
MAX_CUDAGRAPH_CAPTURE_SIZE=$$((MAX_NUM_SEQS * (NUM_SPECULATIVE_TOKENS + 1)))
else
MAX_CUDAGRAPH_CAPTURE_SIZE="$${MAX_NUM_SEQS}"
fi
fi
SPEC_ARGS=()
if [ "$${MTP}" = "1" ]; then
SPEC_CONFIG="$${SPEC_CONFIG:-$$(printf '{"model":"%s","method":"mtp","num_speculative_tokens":%s,"moe_backend":"b12x","draft_sample_method":"probabilistic"}' "$${MTP_MODEL}" "$${NUM_SPECULATIVE_TOKENS}")}"
SPEC_ARGS=(--speculative-config "$${SPEC_CONFIG}")
fi
echo "[glm52-reap] TP$${TP_SIZE} DCP$${DCP_SIZE} MTP=$${MTP} ctx=$${MAX_MODEL_LEN} util=$${GPU_MEMORY_UTILIZATION}"
cd /
exec /opt/venv/bin/python -m vllm.entrypoints.cli.main serve "$${MODEL}" \
--served-model-name "$${SERVED_MODEL_NAME}" \
--trust-remote-code \
--host 0.0.0.0 \
--port "$${PORT}" \
--tensor-parallel-size "$${TP_SIZE}" \
--pipeline-parallel-size 1 \
--decode-context-parallel-size "$${DCP_SIZE}" \
--dcp-comm-backend ag_rs \
--dcp-kv-cache-interleave-size 1 \
--enable-chunked-prefill \
--enable-prefix-caching \
--max-model-len "$${MAX_MODEL_LEN}" \
--load-format fastsafetensors \
--async-scheduling \
-cc.pass_config.fuse_allreduce_rms=True \
--gpu-memory-utilization "$${GPU_MEMORY_UTILIZATION}" \
--max-num-batched-tokens "$${MAX_NUM_BATCHED_TOKENS}" \
--max-num-seqs "$${MAX_NUM_SEQS}" \
--max-cudagraph-capture-size "$${MAX_CUDAGRAPH_CAPTURE_SIZE}" \
--quantization modelopt_fp4 \
--attention-backend B12X_MLA_SPARSE \
--moe-backend b12x \
--kv-cache-dtype fp8 \
--tool-call-parser glm47 \
--enable-auto-tool-choice \
--reasoning-parser glm45 \
--hf-overrides "$${HF_OVERRIDES}" \
"$${SPEC_ARGS[@]}"
|