docker-compose.nvfp4.yml

# Validated v20 quality-first NVFP4 MLA-KV profile.
#
# Long-context NVFP4 MLA-KV profile for:
#   madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid
#
# This profile keeps the 64-D RoPE record in BF16 for the lower measured KLD.
# The calibrated 512-D latent scale file is shipped with the checkpoint and
# passed explicitly because the final v20 image does not auto-discover a
# checkpoint-relative scale artifact.
#
# Download without Xet, then launch:
#   HF_HUB_DISABLE_XET=1 hf download \
#     madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid \
#     --local-dir ./GLM-5.2-MXFP8-NVFP4-NF3-Hybrid
#   cd ./GLM-5.2-MXFP8-NVFP4-NF3-Hybrid
#   MODEL_DIR=$PWD docker compose -f docker-compose.nvfp4.yml up -d
#
# The 700k capacity profile defaults to MTP0. MTP3 requires a lower context
# pin on stock v20; its tested limit is documented in the model card.

name: glm52-nf3-v20-nvfp4

services:
  glm52:
    image: voipmonitor/vllm:gilded-gnosis-v20-vllm0c79e41-sie603f74-fi801d57a-cu132-20260726
    entrypoint: ["/usr/local/bin/serve-gilded-gnosis.sh"]
    restart: unless-stopped
    network_mode: host
    ipc: host
    privileged: true
    init: true
    shm_size: 32gb
    gpus: all
    ulimits:
      memlock: -1
      stack: 67108864
      nofile:
        soft: 1048576
        hard: 1048576
    environment:
      MODEL_FAMILY: glm52-hybrid
      MODEL: /root/models/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid
      SERVED_MODEL_NAME: GLM-5.2
      GPUS: 0,1,2,3
      PORT: "${PORT:-5001}"
      TP: "4"
      DCP: "4"
      DCP_BACKEND: a2a
      DCP_A2A_MAX_TOKENS: "16"
      DCP_A2A_LARGE_BACKEND: ag_rs
      DCP_PREFILL_WORKSPACE: auto
      # Leave the v20 helper in control by default. These high-level knobs are
      # exposed for reproducible topology A/B tests without forcing its
      # lower-level VLLM_* implementation variables.
      DCP_QUERY_SPLIT: "${DCP_QUERY_SPLIT:-auto}"
      DCP_CKV_GATHER: "${DCP_CKV_GATHER:-auto}"
      DCP_TOPK_OWNER_MERGE: "${DCP_TOPK_OWNER_MERGE:-auto}"
      DCP_INDEXER_SHARDS: "${DCP_INDEXER_SHARDS:-auto}"
      DCP_CKV_PREFETCH_DEPTH: "${DCP_CKV_PREFETCH_DEPTH:-auto}"
      PCIE_CALIBRATION: "${PCIE_CALIBRATION:-auto}"
      PCIE_DMA_MIN_BYTES: "${PCIE_DMA_MIN_BYTES:-auto}"
      # v20 defaults this capacity workspace to 524,288 gathered tokens.
      # Keep that default generally; the explicit 775k/MTP3 profile uses the
      # proven v19 ceiling of 131,072 to reclaim ~345 MiB/GPU without disabling
      # CKV gather.
      VLLM_B12X_MLA_CKV_GATHER_MAX_TOKENS: "${VLLM_B12X_MLA_CKV_GATHER_MAX_TOKENS:-524288}"
      F8_DMA: "0"
      MTP: "${MTP:-0}"
      MOE_MODE: a16
      ONLINE_QUANT: custom
      QUANTIZATION_CONFIG_JSON: >-
        {"linear":{"weight":"mxfp8"},"shared_experts":{"weight":"mxfp8"},"ignore":["re:.*\\.fused_qkv_a_proj$","re:.*\\.q_a_proj$","re:.*kv_a_proj_with_mqa","re:.*\\.mlp\\.gate$","model.layers.78.eh_proj","lm_head"]}
      # Maintained generic successor to the v19 QBMM experiment. Keep disabled
      # in the quality-first base profile so its measured posture remains
      # explicit. The 775k overlay enables it: the 184-shard target consumes
      # the contiguous online-MXFP8 kv_b pack directly, and its reviewed
      # same-checkpoint inheritance patch gives the MTP draft the same pack.
      VLLM_B12X_ABSORB_BMM: "${VLLM_B12X_ABSORB_BMM:-0}"
      KV_CACHE_DTYPE: nvfp4_ds_mla
      # Leave empty for v20 automatic profiling. The separately certified
      # 775k/MTP3 profile uses the historical 6,198,000,000-byte explicit pin.
      KV_CACHE_MEMORY_BYTES: "${KV_CACHE_MEMORY_BYTES:-}"
      # Quality default: 0 keeps the 64-D RoPE record in BF16. Set
      # KV_FP8_ROPE=1 for the separately certified long-context profile.
      KV_FP8_ROPE: "${KV_FP8_ROPE:-0}"
      VLLM_NVFP4_MLA_SCALES_FILE: /root/models/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid/kv-scales/glm52-nvfp4-nf3-hybrid_mla_outer_scales_v1.json
      MAX_NUM_SEQS: "${MAX_NUM_SEQS:-8}"
      GRAPH: "${GRAPH:-64}"
      CUDAGRAPH_SIZES_LIST: "${CUDAGRAPH_SIZES_LIST:-}"
      MAX_MODEL_LEN: "${MAX_MODEL_LEN:-700000}"
      MAX_BATCHED_TOKENS: "${MAX_BATCHED_TOKENS:-2048}"
      # This selective long-context profile quantizes shared experts but keeps
      # q-a/kv-a BF16. It fits the tested 700k pin without the aggressive
      # preset's much larger KLD regression.
      GPU_MEMORY_UTILIZATION: "${GPU_MEMORY_UTILIZATION:-0.983}"
      LOAD_FORMAT: instanttensor
      INSTANTTENSOR_BACKEND: BUFFERED
      # The validated v19 775k profile used the default, non-expandable CUDA
      # allocator to avoid load-time fragmentation. The single-dash Compose
      # expansion deliberately permits an explicit empty override:
      #   PYTORCH_CUDA_ALLOC_CONF= docker compose ...
      PYTORCH_CUDA_ALLOC_CONF: "${PYTORCH_CUDA_ALLOC_CONF-expandable_segments:True}"
    volumes:
      - ${MODEL_DIR:?set MODEL_DIR to the downloaded checkpoint directory}:/root/models/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid:ro
      - ${CACHE_DIR:-./cache/glm52-nf3-v20-nvfp4}:/cache
      - ${CACHE_DIR:-./cache/glm52-nf3-v20-nvfp4}/tmp:/container-tmp