File size: 1,262 Bytes
2a85415 48876b9 2a85415 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 | #!/bin/bash
# entrypoint.sh — vLLM server for LFM2.5-8B-A1B-NVFP4 (modelopt / NVFP4, single GPU).
set -e
MODEL_DIR="${MODEL_DIR:-/model}"
PORT="${PORT:-8000}"
HOST="${HOST:-0.0.0.0}"
# --- tunables (all env-overridable) ---
MAX_MODEL_LEN="${MAX_MODEL_LEN:-32768}" # up to 128000
MAX_NUM_SEQS="${MAX_NUM_SEQS:-16}" # ~10 full-128K足軽 + headroom for shorter ones (fp8 KV)
GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.90}"
KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-auto}" # set "fp8" to ~double KV capacity
TP_SIZE="${TP_SIZE:-1}" # this model fits 1 GPU — keep 1
DTYPE="${DTYPE:-auto}"
REASONING_PARSER="${REASONING_PARSER:-deepseek_r1}" # <think>…</think> -> reasoning_content; "" to disable
SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-lfm25-8b-a1b}"
ARGS=(
"$MODEL_DIR"
--served-model-name "$SERVED_MODEL_NAME"
--quantization modelopt
--tensor-parallel-size "$TP_SIZE"
--max-model-len "$MAX_MODEL_LEN"
--max-num-seqs "$MAX_NUM_SEQS"
--gpu-memory-utilization "$GPU_MEM_UTIL"
--kv-cache-dtype "$KV_CACHE_DTYPE"
--dtype "$DTYPE"
--chat-template "$MODEL_DIR/chat_template.jinja"
--host "$HOST"
--port "$PORT"
)
[ -n "$REASONING_PARSER" ] && ARGS+=(--reasoning-parser "$REASONING_PARSER")
exec vllm serve "${ARGS[@]}"
|