Upload run_gguf_server.sh with huggingface_hub

run_gguf_server.sh

@@ -0,0 +1,119 @@
+#!/bin/bash
+set -e
+
+BASE=./Confucius4_GGUF
+LLAMA_CPP_DIR=<your-llama.cpp-install-path>
+
+# defaults
+MODEL=""
+PORT=8080
+N_CTX=20000
+USE_GPU="${GGML_CUDA:-ON}"
+GPU_LAYERS=99
+GPUS="0"
+
+usage() {
+    echo "Usage: sh run_gguf_server.sh [OPTIONS] [MODEL]"
+    echo ""
+    echo "MODEL: quant type (Q4_K_M, IQ3_M), file path, or empty for auto-detect"
+    echo ""
+    echo "Options:"
+    echo "  -p PORT       port (default 8080)"
+    echo "  -c CTX        context length (default 4096)"
+    echo "  -g GPUS       GPU devices (default 0, e.g. 0,1)"
+    echo "  -ngl LAYERS   GPU layers (default 99)"
+    echo "  --cpu         disable GPU"
+    echo ""
+    echo "Examples:"
+    echo "  sh run_gguf_server.sh Q4_K_M"
+    echo "  sh run_gguf_server.sh Q4_K_M -p 8080 -g 0,1 -c 8192"
+    echo "  sh run_gguf_server.sh bf16/Confucius4-Q4_K_M.gguf --cpu"
+    exit 1
+}
+
+# parse args
+while [ $# -gt 0 ]; do
+    case "$1" in
+        -p) PORT="$2"; shift 2 ;;
+        -c) N_CTX="$2"; shift 2 ;;
+        -g) GPUS="$2"; shift 2 ;;
+        -ngl) GPU_LAYERS="$2"; shift 2 ;;
+        --cpu) USE_GPU=OFF; shift ;;
+        -h|--help) usage ;;
+        -*) echo "Unknown: $1"; usage ;;
+        *) MODEL="$1"; shift ;;
+    esac
+done
+
+# resolve model
+if [ -z "$MODEL" ]; then
+    # auto-detect: prefer quantized, then f16
+    MODEL=$(ls "$BASE"/*/Confucius4-*.gguf 2>/dev/null | head -1)
+    if [ -z "$MODEL" ]; then
+        MODEL=$(ls "$BASE"/bf16/Confucius4-00001-of-*.gguf 2>/dev/null | head -1)
+    fi
+else
+    # if it's a quant type like Q4_K_M, look in that dir
+    if [ -d "$BASE/$MODEL" ] && [ -f "$BASE/$MODEL/Confucius4-$MODEL.gguf" ]; then
+        MODEL="$BASE/$MODEL/Confucius4-$MODEL.gguf"
+    elif [ -d "$BASE/$MODEL" ]; then
+        # dir exists, find first gguf inside
+        MODEL=$(ls "$BASE/$MODEL"/Confucius4-*.gguf 2>/dev/null | head -1)
+        [ -z "$MODEL" ] && { echo "ERROR: no gguf in $BASE/$MODEL/"; exit 1; }
+    fi
+fi
+
+[ -f "$MODEL" ] || { echo "ERROR: model not found: $MODEL"; exit 1; }
+
+# auto-detect mmproj
+MMPROJ=""
+MODEL_DIR=$(dirname "$MODEL")
+# Look for mmproj in the same directory as the model, or in f16 dir
+for candidate in "$MODEL_DIR"/mmproj-*.gguf "$BASE"/bf16/mmproj-*.gguf; do
+    [ -f "$candidate" ] && { MMPROJ="$candidate"; break; }
+done
+
+# build llama-server if needed
+SERVER_BIN="$LLAMA_CPP_DIR/build/bin/llama-server"
+CUDA_BUILD_DIR="$LLAMA_CPP_DIR/build-cuda"
+if [ "$USE_GPU" = "ON" ]; then
+    SERVER_BIN="$CUDA_BUILD_DIR/bin/llama-server"
+    if [ ! -f "$SERVER_BIN" ]; then
+        echo "=== Building llama-server (CUDA) ==="
+        cmake -S "$LLAMA_CPP_DIR" -B "$CUDA_BUILD_DIR" \
+            -DBUILD_SHARED_LIBS=OFF -DLLAMA_CURL=OFF \
+            -DGGML_CUDA=ON 2>&1 | tail -1
+        cmake --build "$CUDA_BUILD_DIR" --target llama-server -j$(nproc) 2>&1 | tail -3
+    fi
+else
+    if [ ! -f "$SERVER_BIN" ]; then
+        echo "=== Building llama-server (CPU) ==="
+        cmake -S "$LLAMA_CPP_DIR" -B "$LLAMA_CPP_DIR/build" -DBUILD_SHARED_LIBS=OFF -DLLAMA_CURL=OFF 2>&1 | tail -1
+        cmake --build "$LLAMA_CPP_DIR/build" --target llama-server -j$(nproc) 2>&1 | tail -3
+    fi
+fi
+
+echo "=== Starting server ==="
+echo "  model: $MODEL"
+echo "  port:  $PORT"
+echo "  ctx:   $N_CTX"
+if [ -n "$MMPROJ" ]; then
+    echo "  mmproj: $MMPROJ"
+fi
+if [ "$USE_GPU" = "ON" ]; then
+    echo "  GPU:   layers=$GPU_LAYERS  devices=$GPUS"
+    export CUDA_VISIBLE_DEVICES=$GPUS
+else
+    echo "  GPU:   off"
+fi
+echo "  API:   http://localhost:$PORT/v1/chat/completions"
+echo ""
+
+MMPROJ_ARG=()
+[ -n "$MMPROJ" ] && MMPROJ_ARG=(--mmproj "$MMPROJ")
+
+if [ "$USE_GPU" = "ON" ]; then
+    "$SERVER_BIN" -m "$MODEL" --port "$PORT" --host 0.0.0.0 -c "$N_CTX" -ngl "$GPU_LAYERS" --parallel 1 "${MMPROJ_ARG[@]}"
+else
+    "$SERVER_BIN" -m "$MODEL" --port "$PORT" --host 0.0.0.0 -c "$N_CTX" --parallel 1 "${MMPROJ_ARG[@]}"
+fi