Dockerfile

FROM ubuntu:22.04

# Install system dependencies
RUN apt-get update && \
    apt-get install -y \
    build-essential \
    libssl-dev \
    zlib1g-dev \
    libboost-math-dev \
    libboost-python-dev \
    libboost-timer-dev \
    libboost-thread-dev \
    libboost-system-dev \
    libboost-filesystem-dev \
    libopenblas-dev \
    libomp-dev \
    cmake \
    pkg-config \
    git \
    python3-pip \
    curl \
    libcurl4-openssl-dev \
    wget && \
    rm -rf /var/lib/apt/lists/*

# Build llama.cpp with OpenBLAS
RUN git clone https://github.com/ggerganov/llama.cpp && \
    cd llama.cpp && \
    cmake -B build -S . \
        -DLLAMA_BUILD_SERVER=ON \
        -DLLAMA_BUILD_EXAMPLES=ON \
        -DGGML_BLAS=ON \
        -DGGML_BLAS_VENDOR=OpenBLAS \
        -DCMAKE_BUILD_TYPE=Release && \
    cmake --build build --config Release --target llama-server -j $(nproc)
    
RUN cd /llama.cpp/build && ./bin/llama-server --list-devices

# Download model
RUN mkdir -p /models && \
    wget -O /models/model.q8_0.gguf https://huggingface.co/hugging-quants/Llama-3.2-3B-Instruct-Q8_0-GGUF/resolve/main/llama-3.2-3b-instruct-q8_0.gguf


RUN pip install fastapi uvicorn openai

# Copy app and startup script
COPY app.py /app.py
COPY start.sh /start.sh
RUN chmod +x /start.sh

# Expose ports
EXPOSE 7860 8080

# Start services
CMD ["/start.sh"]