Spaces:
Paused
Paused
File size: 4,238 Bytes
5803f54 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 | from fastapi import FastAPI, Request
from fastapi.responses import StreamingResponse, JSONResponse
from llm_manager import LLMManager
import json
import time
import asyncio
from typing import Optional
app = FastAPI()
llm = None
@app.on_event("startup")
async def startup_event():
global llm
llm = LLMManager()
@app.get("/")
async def root():
return {"status": "running", "model": llm.model_file if llm else "loading"}
@app.post("/api/generate")
async def generate(request: Request):
data = await request.json()
prompt = data.get("prompt")
stream = data.get("stream", True)
model_name = data.get("model", "qwen3")
if not prompt:
return JSONResponse({"error": "Prompt is required"}, status_code=400)
def stream_response():
response = llm.generate(prompt, stream=True)
for chunk in response:
yield json.dumps({
"model": model_name,
"created_at": time.strftime("%Y-%m-%dT%H:%M:%S.000Z", time.gmtime()),
"response": chunk["choices"][0]["text"],
"done": False
}) + "\n"
yield json.dumps({
"model": model_name,
"created_at": time.strftime("%Y-%m-%dT%H:%M:%S.000Z", time.gmtime()),
"done": True,
"context": [], # Placeholder
"total_duration": 0,
"load_duration": 0,
"prompt_eval_count": 0,
"prompt_eval_duration": 0,
"eval_count": 0,
"eval_duration": 0
}) + "\n"
if stream:
return StreamingResponse(stream_response(), media_type="application/x-ndjson")
else:
response = llm.generate(prompt, stream=False)
return {
"model": model_name,
"created_at": time.strftime("%Y-%m-%dT%H:%M:%S.000Z", time.gmtime()),
"response": response["choices"][0]["text"],
"done": True,
"context": [],
"total_duration": 0,
"load_duration": 0,
"prompt_eval_count": 0,
"prompt_eval_duration": 0,
"eval_count": 0,
"eval_duration": 0
}
@app.post("/api/chat")
async def chat(request: Request):
data = await request.json()
messages = data.get("messages", [])
stream = data.get("stream", True)
model_name = data.get("model", "qwen3")
def stream_chat():
response = llm.chat_completion(messages, stream=True)
for chunk in response:
if "choices" in chunk and len(chunk["choices"]) > 0:
delta = chunk["choices"][0].get("delta", {})
content = delta.get("content", "")
yield json.dumps({
"model": model_name,
"created_at": time.strftime("%Y-%m-%dT%H:%M:%S.000Z", time.gmtime()),
"message": {"role": "assistant", "content": content},
"done": False
}) + "\n"
yield json.dumps({
"model": model_name,
"created_at": time.strftime("%Y-%m-%dT%H:%M:%S.000Z", time.gmtime()),
"done": True
}) + "\n"
if stream:
return StreamingResponse(stream_chat(), media_type="application/x-ndjson")
else:
response = llm.chat_completion(messages, stream=False)
return {
"model": model_name,
"created_at": time.strftime("%Y-%m-%dT%H:%M:%S.000Z", time.gmtime()),
"message": response["choices"][0]["message"],
"done": True
}
@app.get("/api/tags")
async def tags():
return {
"models": [
{
"name": "qwen3",
"modified_at": time.strftime("%Y-%m-%dT%H:%M:%S.000Z", time.gmtime()),
"size": 0, # TBD
"digest": "qwen3-digest",
"details": {
"format": "gguf",
"family": "qwen",
"families": ["qwen"],
"parameter_size": "14B",
"quantization_level": "Q4_K_M"
}
}
]
}
if __name__ == "__main__":
import uvicorn
uvicorn.run(app, host="0.0.0.0", port=7860)
|