| """ |
| HuggingFace Space — Gemma 4 26B A4B Coding API |
| Model : unsloth/gemma-4-26B-A4B-it-GGUF → UD-IQ3_XXS (11.2 GB) |
| RAM : fits in 16 GB with ~4 GB left for KV cache at ctx=4096 |
| Params: temp=0.3, top_p=0.9, min_p=0.1, top_k=20 (tuned for coding per reddit) |
| |
| Endpoints |
| GET / → landing page |
| GET /health → status (also used by self-ping) |
| GET /v1/models → OpenAI model list |
| POST /v1/chat/completions → OpenAI-compatible |
| POST /v1/messages → Anthropic-compatible ← Claude Code uses this |
| """ |
|
|
| import os, json, time, uuid, asyncio, threading |
| from contextlib import asynccontextmanager |
| from typing import Optional, List, Union, Any, Dict |
|
|
| import httpx |
| from fastapi import FastAPI, HTTPException |
| from fastapi.responses import HTMLResponse, StreamingResponse, JSONResponse |
| from fastapi.middleware.cors import CORSMiddleware |
| from pydantic import BaseModel |
|
|
| |
| MODEL_REPO = os.getenv("MODEL_REPO", "unsloth/gemma-4-26B-A4B-it-GGUF") |
| MODEL_FILE = os.getenv("MODEL_FILE", "gemma-4-26B-A4B-it-UD-IQ3_XXS.gguf") |
| MODEL_DIR = "/app/models" |
| MODEL_PATH = f"{MODEL_DIR}/{MODEL_FILE}" |
| SPACE_URL = os.getenv("SPACE_URL", "") |
|
|
| |
| N_CTX = int(os.getenv("N_CTX", "4096")) |
| N_THREADS = int(os.getenv("N_THREADS", "2")) |
|
|
| |
| DEFAULT_TEMP = float(os.getenv("DEFAULT_TEMP", "0.3")) |
| DEFAULT_TOP_P = float(os.getenv("DEFAULT_TOP_P", "0.9")) |
| DEFAULT_MIN_P = float(os.getenv("DEFAULT_MIN_P", "0.1")) |
| DEFAULT_TOP_K = int(os.getenv("DEFAULT_TOP_K", "20")) |
|
|
| MODEL_ALIAS = "gemma-4-26b" |
| llm = None |
|
|
| |
| def download_model(): |
| from huggingface_hub import hf_hub_download |
| os.makedirs(MODEL_DIR, exist_ok=True) |
| if not os.path.exists(MODEL_PATH): |
| print(f"[model] Downloading {MODEL_FILE} (~11.2 GB)...") |
| hf_hub_download( |
| repo_id=MODEL_REPO, |
| filename=MODEL_FILE, |
| local_dir=MODEL_DIR, |
| ) |
| print("[model] Download complete.") |
|
|
| def load_model(): |
| global llm |
| from llama_cpp import Llama |
| download_model() |
| print("[model] Loading Gemma 4 26B IQ3_XXS into RAM...") |
| llm = Llama( |
| model_path = MODEL_PATH, |
| n_ctx = N_CTX, |
| n_threads = N_THREADS, |
| n_batch = 512, |
| n_gpu_layers = 0, |
| verbose = False, |
| chat_format = None, |
| ) |
| print(f"[model] Gemma 4 26B ready — ctx={N_CTX}, threads={N_THREADS}") |
|
|
| |
| async def self_ping_loop(): |
| while True: |
| await asyncio.sleep(25 * 60) |
| if SPACE_URL: |
| try: |
| async with httpx.AsyncClient(timeout=15) as c: |
| r = await c.get(f"{SPACE_URL}/health") |
| print(f"[ping] {r.status_code}") |
| except Exception as e: |
| print(f"[ping] failed: {e}") |
|
|
| |
| @asynccontextmanager |
| async def lifespan(app: FastAPI): |
| threading.Thread(target=load_model, daemon=True).start() |
| asyncio.create_task(self_ping_loop()) |
| yield |
|
|
| app = FastAPI(title="Gemma 4 Coding API", lifespan=lifespan) |
| app.add_middleware( |
| CORSMiddleware, |
| allow_origins=["*"], |
| allow_methods=["*"], |
| allow_headers=["*"], |
| allow_credentials=True, |
| ) |
|
|
| |
| def _check_model(): |
| if llm is None: |
| raise HTTPException( |
| 503, |
| detail="Model still loading — first boot downloads ~11 GB, wait ~5-10 min" |
| ) |
|
|
| def _extract_text(content) -> str: |
| if isinstance(content, str): |
| return content |
| if isinstance(content, list): |
| parts = [] |
| for block in content: |
| if isinstance(block, dict): |
| if block.get("type") == "text": |
| parts.append(block.get("text", "")) |
| elif block.get("type") == "tool_result": |
| parts.append(_extract_text(block.get("content", ""))) |
| else: |
| parts.append(str(block)) |
| return "".join(parts) |
| return str(content) |
|
|
| |
| @app.get("/health") |
| async def health(): |
| return { |
| "status": "ok", |
| "model_loaded": llm is not None, |
| "model": MODEL_FILE, |
| "ctx": N_CTX, |
| } |
|
|
| |
| class OAIMessage(BaseModel): |
| role: str |
| content: Union[str, List[Any]] |
|
|
| class OAIRequest(BaseModel): |
| model: str = MODEL_ALIAS |
| messages: List[OAIMessage] |
| temperature: float = DEFAULT_TEMP |
| top_p: float = DEFAULT_TOP_P |
| min_p: float = DEFAULT_MIN_P |
| top_k: int = DEFAULT_TOP_K |
| max_tokens: int = 2048 |
| stream: bool = False |
| stop: Optional[List[str]] = None |
|
|
| @app.get("/v1/models") |
| async def oai_models(): |
| return { |
| "object": "list", |
| "data": [{ |
| "id": MODEL_ALIAS, |
| "object": "model", |
| "created": int(time.time()), |
| "owned_by": "google-deepmind", |
| }], |
| } |
|
|
| @app.post("/v1/chat/completions") |
| async def oai_chat(req: OAIRequest): |
| _check_model() |
| msgs = [ |
| {"role": m.role, "content": _extract_text(m.content)} |
| for m in req.messages |
| ] |
| kwargs = dict( |
| messages = msgs, |
| temperature = req.temperature, |
| top_p = req.top_p, |
| min_p = req.min_p, |
| top_k = req.top_k, |
| max_tokens = req.max_tokens, |
| stop = req.stop, |
| ) |
|
|
| if req.stream: |
| async def gen(): |
| rid = f"chatcmpl-{uuid.uuid4().hex[:8]}" |
| ts = int(time.time()) |
| for chunk in llm.create_chat_completion(**kwargs, stream=True): |
| data = { |
| "id": rid, |
| "object": "chat.completion.chunk", |
| "created": ts, |
| "model": req.model, |
| "choices": [{ |
| "index": 0, |
| "delta": chunk["choices"][0]["delta"], |
| "finish_reason": chunk["choices"][0]["finish_reason"], |
| }], |
| } |
| yield f"data: {json.dumps(data)}\n\n" |
| yield "data: [DONE]\n\n" |
| return StreamingResponse(gen(), media_type="text/event-stream") |
|
|
| result = llm.create_chat_completion(**kwargs, stream=False) |
| return JSONResponse(result) |
|
|
| |
| class AnthropicMessage(BaseModel): |
| role: str |
| content: Union[str, List[Dict]] |
|
|
| class AnthropicRequest(BaseModel): |
| model: str = MODEL_ALIAS |
| messages: List[AnthropicMessage] |
| system: Optional[str] = None |
| max_tokens: int = 2048 |
| temperature: float = DEFAULT_TEMP |
| top_p: float = DEFAULT_TOP_P |
| top_k: int = DEFAULT_TOP_K |
| stream: bool = False |
| stop_sequences: Optional[List[str]] = None |
|
|
| @app.post("/v1/messages") |
| async def anthropic_messages(req: AnthropicRequest): |
| _check_model() |
| msgs = [] |
| if req.system: |
| msgs.append({"role": "system", "content": req.system}) |
| for m in req.messages: |
| msgs.append({"role": m.role, "content": _extract_text(m.content)}) |
|
|
| kwargs = dict( |
| messages = msgs, |
| temperature = req.temperature, |
| top_p = req.top_p, |
| min_p = DEFAULT_MIN_P, |
| top_k = req.top_k, |
| max_tokens = req.max_tokens, |
| stop = req.stop_sequences, |
| ) |
|
|
| if req.stream: |
| async def gen(): |
| msg_id = f"msg_{uuid.uuid4().hex[:20]}" |
| yield f"data: {json.dumps({'type':'message_start','message':{'id':msg_id,'type':'message','role':'assistant','content':[],'model':req.model,'stop_reason':None,'usage':{'input_tokens':0,'output_tokens':0}}})}\n\n" |
| yield f"data: {json.dumps({'type':'content_block_start','index':0,'content_block':{'type':'text','text':''}})}\n\n" |
| full = "" |
| for chunk in llm.create_chat_completion(**kwargs, stream=True): |
| dt = chunk["choices"][0]["delta"].get("content", "") |
| if dt: |
| full += dt |
| yield f"data: {json.dumps({'type':'content_block_delta','index':0,'delta':{'type':'text_delta','text':dt}})}\n\n" |
| yield f"data: {json.dumps({'type':'content_block_stop','index':0})}\n\n" |
| yield f"data: {json.dumps({'type':'message_delta','delta':{'stop_reason':'end_turn','stop_sequence':None},'usage':{'output_tokens':len(full.split())}})}\n\n" |
| yield f"data: {json.dumps({'type':'message_stop'})}\n\n" |
| return StreamingResponse( |
| gen(), |
| media_type="text/event-stream", |
| headers={"anthropic-version": "2023-06-01"}, |
| ) |
|
|
| result = llm.create_chat_completion(**kwargs, stream=False) |
| text = result["choices"][0]["message"]["content"] |
| usage = result.get("usage", {}) |
| return JSONResponse({ |
| "id": f"msg_{uuid.uuid4().hex[:20]}", |
| "type": "message", |
| "role": "assistant", |
| "content": [{"type": "text", "text": text}], |
| "model": req.model, |
| "stop_reason": "end_turn", |
| "stop_sequence": None, |
| "usage": { |
| "input_tokens": usage.get("prompt_tokens", 0), |
| "output_tokens": usage.get("completion_tokens", 0), |
| }, |
| }) |
|
|
| |
| @app.get("/", response_class=HTMLResponse) |
| async def landing(): |
| sc = "#22c55e" if llm is not None else "#f59e0b" |
| st = "Model ready" if llm is not None else "Loading model... (~5-10 min on first boot)" |
| return LANDING_HTML.replace("{{SC}}", sc).replace("{{ST}}", st) |
|
|
| LANDING_HTML = r"""<!DOCTYPE html> |
| <html lang="en"> |
| <head> |
| <meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1"> |
| <title>Gemma 4 26B Coding API</title> |
| <style> |
| *{box-sizing:border-box;margin:0;padding:0} |
| body{font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif;background:#0d0d12;color:#e2e2ed;min-height:100vh;display:flex;flex-direction:column;align-items:center;padding:3.5rem 1.5rem 4rem} |
| h1{font-size:2.1rem;font-weight:700;background:linear-gradient(130deg,#818cf8 20%,#34d399 80%);-webkit-background-clip:text;-webkit-text-fill-color:transparent;margin-bottom:.35rem;letter-spacing:-.5px} |
| .tagline{color:#6b7280;font-size:.93rem;margin-bottom:2.5rem;text-align:center;line-height:1.5} |
| .badge{display:inline-flex;align-items:center;gap:.45rem;background:#151520;border:1px solid #2a2a3a;border-radius:999px;padding:.3rem .9rem;font-size:.8rem;margin:.25rem} |
| .dot{width:7px;height:7px;border-radius:50%;background:{{SC}};flex-shrink:0} |
| .badges{display:flex;flex-wrap:wrap;justify-content:center;margin-bottom:2.8rem} |
| .cards{display:grid;grid-template-columns:repeat(auto-fit,minmax(290px,1fr));gap:1.1rem;width:100%;max-width:920px;margin-bottom:2.8rem} |
| .card{background:#13131c;border:1px solid #252535;border-radius:14px;padding:1.3rem 1.5rem} |
| .card-title{font-size:.72rem;font-weight:600;text-transform:uppercase;letter-spacing:.1em;color:#6b7280;margin-bottom:.75rem} |
| pre{background:#090910;border:1px solid #1e1e2e;border-radius:9px;padding:.85rem 1rem;font-family:'JetBrains Mono','Fira Code',monospace;font-size:.78rem;color:#a5b4fc;line-height:1.65;overflow-x:auto;white-space:pre-wrap;word-break:break-all} |
| .ep-table{width:100%;max-width:920px;border-collapse:collapse;margin-bottom:2rem} |
| .ep-table thead th{font-size:.72rem;text-transform:uppercase;letter-spacing:.08em;color:#4b5563;padding:.5rem .8rem;border-bottom:1px solid #1e1e2e;text-align:left} |
| .ep-table tbody tr{border-bottom:1px solid #161622} |
| .ep-table tbody td{padding:.7rem .8rem;font-size:.84rem} |
| .method{display:inline-block;font-size:.68rem;font-weight:700;padding:.18rem .5rem;border-radius:5px;min-width:42px;text-align:center} |
| .get{background:#064e3b;color:#34d399}.post{background:#1e3a5f;color:#60a5fa} |
| .path{font-family:monospace;color:#e2e8f0;font-size:.85rem} |
| .note{font-size:.78rem;color:#4b5563} |
| .tip{background:#131a1f;border:1px solid #1d3040;border-radius:10px;padding:1rem 1.25rem;width:100%;max-width:920px;font-size:.82rem;color:#7dd3fc;line-height:1.6;margin-bottom:1.2rem} |
| footer{margin-top:2.5rem;font-size:.75rem;color:#374151;text-align:center;line-height:1.8} |
| </style> |
| </head> |
| <body> |
| <h1>Gemma 4 26B A4B</h1> |
| <p class="tagline">Coding-tuned · Anthropic & OpenAI compatible · HuggingFace Spaces</p> |
| |
| <div class="badges"> |
| <span class="badge"><span class="dot"></span>{{ST}}</span> |
| <span class="badge" style="color:#9ca3af">IQ3_XXS · 11.2 GB</span> |
| <span class="badge" style="color:#9ca3af">ctx 4096 · 2 vCPU · 16 GB RAM</span> |
| <span class="badge" style="color:#9ca3af">temp 0.3 · top-k 20 · min-p 0.1</span> |
| </div> |
| |
| <div class="cards"> |
| <div class="card"> |
| <div class="card-title">Claude Code setup</div> |
| <pre>export ANTHROPIC_BASE_URL=\ |
| https://YOUR-USER-space-name.hf.space |
| export ANTHROPIC_API_KEY=gemma4-local |
| |
| claude --model gemma-4-26b</pre> |
| </div> |
| <div class="card"> |
| <div class="card-title">OpenAI Python client</div> |
| <pre>from openai import OpenAI |
| client = OpenAI( |
| base_url="https://YOUR-SPACE.hf.space/v1", |
| api_key="gemma4-local", |
| ) |
| r = client.chat.completions.create( |
| model="gemma-4-26b", |
| messages=[{"role":"user", |
| "content":"write binary search"}], |
| )</pre> |
| </div> |
| <div class="card"> |
| <div class="card-title">curl quick test</div> |
| <pre>curl YOUR-SPACE.hf.space/v1/chat/completions \ |
| -H "Content-Type: application/json" \ |
| -d '{ |
| "model": "gemma-4-26b", |
| "messages": [ |
| {"role":"user", |
| "content":"hello"} |
| ] |
| }'</pre> |
| </div> |
| </div> |
| |
| <div class="tip"> |
| <strong>First boot:</strong> The model (~11.2 GB) downloads from HuggingFace on first start — allow 5–10 min. |
| <code style="background:#0d1b26;padding:1px 5px;border-radius:4px">/health</code> returns |
| <code style="background:#0d1b26;padding:1px 5px;border-radius:4px">model_loaded: false</code> |
| until ready. Subsequent restarts load from disk in ~60 s. Self-pings every 25 min to prevent sleep. |
| </div> |
| |
| <table class="ep-table"> |
| <thead><tr><th>Method</th><th>Path</th><th>Notes</th></tr></thead> |
| <tbody> |
| <tr><td><span class="method get">GET</span></td><td class="path">/health</td><td class="note">Status + model_loaded</td></tr> |
| <tr><td><span class="method get">GET</span></td><td class="path">/v1/models</td><td class="note">Model list (OpenAI)</td></tr> |
| <tr><td><span class="method post">POST</span></td><td class="path">/v1/chat/completions</td><td class="note">OpenAI-compatible · streaming supported</td></tr> |
| <tr><td><span class="method post">POST</span></td><td class="path">/v1/messages</td><td class="note">Anthropic-compatible · used by Claude Code</td></tr> |
| </tbody> |
| </table> |
| |
| <footer> |
| Gemma 4 26B A4B · unsloth UD-IQ3_XXS · llama-cpp-python + OpenBLAS<br> |
| Self-pings /health every 25 min · April 2026 |
| </footer> |
| </body> |
| </html>""" |
