run_gguf_server.sh

#!/bin/bash
set -e

BASE=./Confucius4_GGUF
LLAMA_CPP_DIR=<your-llama.cpp-install-path>

# defaults
MODEL=""
PORT=8080
N_CTX=20000
USE_GPU="${GGML_CUDA:-ON}"
GPU_LAYERS=99
GPUS="0"

usage() {
    echo "Usage: sh run_gguf_server.sh [OPTIONS] [MODEL]"
    echo ""
    echo "MODEL: quant type (Q4_K_M, IQ3_M), file path, or empty for auto-detect"
    echo ""
    echo "Options:"
    echo "  -p PORT       port (default 8080)"
    echo "  -c CTX        context length (default 4096)"
    echo "  -g GPUS       GPU devices (default 0, e.g. 0,1)"
    echo "  -ngl LAYERS   GPU layers (default 99)"
    echo "  --cpu         disable GPU"
    echo ""
    echo "Examples:"
    echo "  sh run_gguf_server.sh Q4_K_M"
    echo "  sh run_gguf_server.sh Q4_K_M -p 8080 -g 0,1 -c 8192"
    echo "  sh run_gguf_server.sh bf16/Confucius4-Q4_K_M.gguf --cpu"
    exit 1
}

# parse args
while [ $# -gt 0 ]; do
    case "$1" in
        -p) PORT="$2"; shift 2 ;;
        -c) N_CTX="$2"; shift 2 ;;
        -g) GPUS="$2"; shift 2 ;;
        -ngl) GPU_LAYERS="$2"; shift 2 ;;
        --cpu) USE_GPU=OFF; shift ;;
        -h|--help) usage ;;
        -*) echo "Unknown: $1"; usage ;;
        *) MODEL="$1"; shift ;;
    esac
done

# resolve model
if [ -z "$MODEL" ]; then
    # auto-detect: prefer quantized, then f16
    MODEL=$(ls "$BASE"/*/Confucius4-*.gguf 2>/dev/null | head -1)
    if [ -z "$MODEL" ]; then
        MODEL=$(ls "$BASE"/bf16/Confucius4-00001-of-*.gguf 2>/dev/null | head -1)
    fi
else
    # if it's a quant type like Q4_K_M, look in that dir
    if [ -d "$BASE/$MODEL" ] && [ -f "$BASE/$MODEL/Confucius4-$MODEL.gguf" ]; then
        MODEL="$BASE/$MODEL/Confucius4-$MODEL.gguf"
    elif [ -d "$BASE/$MODEL" ]; then
        # dir exists, find first gguf inside
        MODEL=$(ls "$BASE/$MODEL"/Confucius4-*.gguf 2>/dev/null | head -1)
        [ -z "$MODEL" ] && { echo "ERROR: no gguf in $BASE/$MODEL/"; exit 1; }
    fi
fi

[ -f "$MODEL" ] || { echo "ERROR: model not found: $MODEL"; exit 1; }

# auto-detect mmproj
MMPROJ=""
MODEL_DIR=$(dirname "$MODEL")
# Look for mmproj in the same directory as the model, or in f16 dir
for candidate in "$MODEL_DIR"/mmproj-*.gguf "$BASE"/bf16/mmproj-*.gguf; do
    [ -f "$candidate" ] && { MMPROJ="$candidate"; break; }
done

# build llama-server if needed
SERVER_BIN="$LLAMA_CPP_DIR/build/bin/llama-server"
CUDA_BUILD_DIR="$LLAMA_CPP_DIR/build-cuda"
if [ "$USE_GPU" = "ON" ]; then
    SERVER_BIN="$CUDA_BUILD_DIR/bin/llama-server"
    if [ ! -f "$SERVER_BIN" ]; then
        echo "=== Building llama-server (CUDA) ==="
        cmake -S "$LLAMA_CPP_DIR" -B "$CUDA_BUILD_DIR" \
            -DBUILD_SHARED_LIBS=OFF -DLLAMA_CURL=OFF \
            -DGGML_CUDA=ON 2>&1 | tail -1
        cmake --build "$CUDA_BUILD_DIR" --target llama-server -j$(nproc) 2>&1 | tail -3
    fi
else
    if [ ! -f "$SERVER_BIN" ]; then
        echo "=== Building llama-server (CPU) ==="
        cmake -S "$LLAMA_CPP_DIR" -B "$LLAMA_CPP_DIR/build" -DBUILD_SHARED_LIBS=OFF -DLLAMA_CURL=OFF 2>&1 | tail -1
        cmake --build "$LLAMA_CPP_DIR/build" --target llama-server -j$(nproc) 2>&1 | tail -3
    fi
fi

echo "=== Starting server ==="
echo "  model: $MODEL"
echo "  port:  $PORT"
echo "  ctx:   $N_CTX"
if [ -n "$MMPROJ" ]; then
    echo "  mmproj: $MMPROJ"
fi
if [ "$USE_GPU" = "ON" ]; then
    echo "  GPU:   layers=$GPU_LAYERS  devices=$GPUS"
    export CUDA_VISIBLE_DEVICES=$GPUS
else
    echo "  GPU:   off"
fi
echo "  API:   http://localhost:$PORT/v1/chat/completions"
echo ""

MMPROJ_ARG=()
[ -n "$MMPROJ" ] && MMPROJ_ARG=(--mmproj "$MMPROJ")

if [ "$USE_GPU" = "ON" ]; then
    "$SERVER_BIN" -m "$MODEL" --port "$PORT" --host 0.0.0.0 -c "$N_CTX" -ngl "$GPU_LAYERS" --parallel 1 "${MMPROJ_ARG[@]}"
else
    "$SERVER_BIN" -m "$MODEL" --port "$PORT" --host 0.0.0.0 -c "$N_CTX" --parallel 1 "${MMPROJ_ARG[@]}"
fi