entrypoint.sh

#!/bin/bash
# entrypoint.sh — vLLM server for LFM2.5-8B-A1B-NVFP4 (modelopt / NVFP4, single GPU).
set -e

MODEL_DIR="${MODEL_DIR:-/model}"
PORT="${PORT:-8000}"
HOST="${HOST:-0.0.0.0}"

# --- tunables (all env-overridable) ---
MAX_MODEL_LEN="${MAX_MODEL_LEN:-32768}"     # up to 128000
MAX_NUM_SEQS="${MAX_NUM_SEQS:-16}"          # ~10 full-128K足軽 + headroom for shorter ones (fp8 KV)
GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.90}"
KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-auto}"    # set "fp8" to ~double KV capacity
TP_SIZE="${TP_SIZE:-1}"                     # this model fits 1 GPU — keep 1
DTYPE="${DTYPE:-auto}"
REASONING_PARSER="${REASONING_PARSER:-deepseek_r1}"  # <think>…</think> -> reasoning_content; "" to disable
SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-lfm25-8b-a1b}"

ARGS=(
  "$MODEL_DIR"
  --served-model-name "$SERVED_MODEL_NAME"
  --quantization modelopt
  --tensor-parallel-size "$TP_SIZE"
  --max-model-len "$MAX_MODEL_LEN"
  --max-num-seqs "$MAX_NUM_SEQS"
  --gpu-memory-utilization "$GPU_MEM_UTIL"
  --kv-cache-dtype "$KV_CACHE_DTYPE"
  --dtype "$DTYPE"
  --chat-template "$MODEL_DIR/chat_template.jinja"
  --host "$HOST"
  --port "$PORT"
)
[ -n "$REASONING_PARSER" ] && ARGS+=(--reasoning-parser "$REASONING_PARSER")

exec vllm serve "${ARGS[@]}"