| set -euo pipefail | |
| : "${BASE_MODEL_PATH:=/models/Qwen3.5-35B-A3B}" | |
| : "${SERVED_MODEL_NAME:=Qwen3.5-35B-A3B}" | |
| : "${VLLM_HOST:=0.0.0.0}" | |
| : "${VLLM_PORT:=8001}" | |
| : "${CUDA_VISIBLE_DEVICES:=0,1,2,3,4,5,6,7}" | |
| export CUDA_VISIBLE_DEVICES | |
| python -m vllm.entrypoints.openai.api_server \ | |
| --model "${BASE_MODEL_PATH}" \ | |
| --served-model-name "${SERVED_MODEL_NAME}" \ | |
| --tensor-parallel-size 8 \ | |
| --enable-expert-parallel \ | |
| --gpu-memory-utilization 0.90 \ | |
| --host "${VLLM_HOST}" \ | |
| --port "${VLLM_PORT}" \ | |
| --max-num-seqs 64 \ | |
| --max-model-len 262144 \ | |
| --enable-chunked-prefill \ | |
| --enable-auto-tool-choice \ | |
| --tool-call-parser qwen3_coder | |