# Validated v20 quality-first FP8 MLA-KV profile.
#
# Quality-first FP8 MLA-KV profile for:
#   madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid
#
# Download without Xet, then launch:
#   HF_HUB_DISABLE_XET=1 hf download \
#     madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid \
#     --local-dir ./GLM-5.2-MXFP8-NVFP4-NF3-Hybrid
#   cd ./GLM-5.2-MXFP8-NVFP4-NF3-Hybrid
#   MODEL_DIR=$PWD docker compose -f docker-compose.fp8.yml up -d
#
# The standalone 400k capacity profile defaults to MTP0. For the certified
# shared-expert/MTP3 capacity posture, layer this file with
# docker-compose.fp8-450k-mtp3.yml.

name: glm52-nf3-v20-fp8

services:
  glm52:
    image: voipmonitor/vllm:gilded-gnosis-v20-vllm0c79e41-sie603f74-fi801d57a-cu132-20260726
    entrypoint: ["/usr/local/bin/serve-gilded-gnosis.sh"]
    restart: unless-stopped
    network_mode: host
    ipc: host
    privileged: true
    init: true
    shm_size: 32gb
    gpus: all
    ulimits:
      memlock: -1
      stack: 67108864
      nofile:
        soft: 1048576
        hard: 1048576
    environment:
      MODEL_FAMILY: glm52-hybrid
      MODEL: /root/models/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid
      SERVED_MODEL_NAME: GLM-5.2
      GPUS: 0,1,2,3
      PORT: "${PORT:-5001}"
      TP: "4"
      DCP: "4"
      DCP_BACKEND: a2a
      DCP_A2A_MAX_TOKENS: "16"
      DCP_A2A_LARGE_BACKEND: ag_rs
      DCP_PREFILL_WORKSPACE: auto
      PCIE_CALIBRATION: auto
      F8_DMA: "0"
      MTP: "${MTP:-0}"
      MOE_MODE: a16
      ONLINE_QUANT: custom
      QUANTIZATION_CONFIG_JSON: >-
        {"linear":{"weight":"mxfp8"},"ignore":["re:.*\\.fused_qkv_a_proj$","re:.*\\.q_a_proj$","re:.*kv_a_proj_with_mqa","re:.*\\.mlp\\.gate$","model.layers.78.eh_proj","lm_head"]}
      KV_CACHE_DTYPE: fp8_ds_mla
      MAX_NUM_SEQS: "${MAX_NUM_SEQS:-8}"
      GRAPH: "${GRAPH:-64}"
      MAX_MODEL_LEN: "${MAX_MODEL_LEN:-400000}"
      MAX_BATCHED_TOKENS: "${MAX_BATCHED_TOKENS:-2048}"
      # 0.96 only exposes ~297k tokens with this quality-preserving MXFP8
      # membership on 4x96 GiB. 0.98 is required for the tested 400k pin.
      GPU_MEMORY_UTILIZATION: "${GPU_MEMORY_UTILIZATION:-0.98}"
      LOAD_FORMAT: instanttensor
      INSTANTTENSOR_BACKEND: BUFFERED
      PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
    volumes:
      - ${MODEL_DIR:?set MODEL_DIR to the downloaded checkpoint directory}:/root/models/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid:ro
      - ${CACHE_DIR:-./cache/glm52-nf3-v20-fp8}:/cache
      - ${CACHE_DIR:-./cache/glm52-nf3-v20-fp8}/tmp:/container-tmp