Update gen.py

gen.py

@@ -6,7 +6,7 @@ from urllib.parse import quote
 from fastapi import APIRouter, Request, HTTPException, Header
 from fastapi.responses import Response, JSONResponse, StreamingResponse
 import re
-from typing import Optional
+from typing import Optional, Any
 import json
 from helper.assets import (
     save_base64_image,
@@ -34,6 +34,10 @@ from helper.ratelimit import (
     get_usage_snapshot_for_subject,
 )
 from helper.keywords import *
+from uuid import uuid4
+from time import time
+from typing import Dict, List, Optional, Tuple
+
 router = APIRouter(prefix="/gen")
 
 PKEY = os.getenv("POLLINATIONS_KEY", "")
@@ -48,45 +52,207 @@ ratios = {"3:2", "2:3", "1:1"}
 valid_modes = {"normal", "fun", "", None}
 modes = {"normal", "fun"}
 
-MAX_VIDEO_RETRIES = 6
+MODEL_MAP = {
+    "llama-3.1-8b-instant": "Meta Llama 3.1 8B Instant",
+    "gpt-4o-mini": "OpenAI GPT 4o Mini",
+    "nemotron-3-super": "NVIDIA Nemotron 3 Super",
+    "openai/gpt-oss-120b": "OpenAI GPT-OSS 120B",
+    "openai/gpt-oss-20b": "OpenAI GPT-OSS 20B",
+    "qwen-3-235b-a22b-instruct-2507": "Qwen3 Instruct",
+    "llama-3.3-70b-versatile": "Meta Llama 3.3 70B Versatile",
+    "meta-llama/llama-4-scout-17b-16e-instruct": "Meta Llama 4 Scout",
+}
+
+FALLBACK_MODEL = "meta-llama/llama-4-scout-17b-16e-instruct"
+FALLBACK_PROVIDER = "groq"
 
-def is_cinematic_image_prompt(prompt: str) -> bool:
-    for kw in CREATIVE_KEYWORDS:
-        if kw in prompt.lower():
-            return True
-    return False
 
-def is_complex_reasoning(prompt: str) -> bool:
-    if len(prompt) > 800:
-        return True
+# ──────────────────────────────────────────────
+# CENTRAL ROUTING LOGIC
+# ──────────────────────────────────────────────
 
+def route_chat(
+    messages: List[Dict[str, Any]],
+    uses_tools: bool = False,
+) -> Tuple[str, str]:
+    """
+    Inspect messages and return (chosen_model, provider).
+
+    This is the single source of truth for model selection.
+    No API calls, no side-effects — pure routing logic.
+    """
+    total_chars, total_bytes = calculate_messages_size(messages)
+    prompt_text = extract_user_text(messages)
+
+    long_context   = is_long_context(messages)
+    code_present   = contains_code(prompt_text)
+    math_heavy     = is_math_heavy(prompt_text)
+    structured_task = is_structured_task(prompt_text)
+    multi_q        = multiple_questions(prompt_text)
+    code_heavy     = is_code_heavy(prompt_text, code_present, long_context)
+    has_images     = contains_images(messages)
+
+    score = 0
+    if long_context:    score += 3
+    if math_heavy:      score += 3
+    if structured_task: score += 2
+    if code_present:    score += 2
+    if multi_q:         score += 1
     for kw in REASONING_KEYWORDS:
-        if kw in prompt:
-            return True
+        if kw in prompt_text:
+            score += 1
+    score = min(score, 10)
 
-    if re.search(r"\b(if|therefore|assume|let x|given that)\b", prompt):
-        return True
+    # ── multimodal fast-path ──────────────────
+    if has_images:
+        return "gpt-4o-mini", "navy vision"
 
-    return False
+    # ── tool-use branch ──────────────────────
+    if uses_tools:
+        if score >= 6:
+            return "nemotron-3-super", "navy"
+        if score >= 4:
+            return "openai/gpt-oss-120b", "groq"
+        return "openai/gpt-oss-20b", "groq"
+
+    # ── code branch ──────────────────────────
+    if code_present:
+        if code_heavy and score >= 6:
+            return "o3-mini", "navy"
+        if score >= 4:
+            return "llama-3.3-70b-versatile", "groq"
+
+    # ── general reasoning branch ─────────────
+    if score >= 6:
+        return "sonar", "navy"
+    if score >= 4:
+        return "meta-llama/llama-4-scout-17b-16e-instruct", "groq"
+
+    # ── default ──────────────────────────────
+    chosen_model, provider = "llama-3.1-8b-instant", "groq"
+
+    # Groq context-size guard — promote to navy if too large
+    if provider == "groq" and (
+        total_chars > MAX_GROQ_PROMPT_CHARS or total_bytes > MAX_GROQ_PROMPT_BYTES
+    ):
+        return "gpt-4o-mini", "navy"
+
+    return chosen_model, provider
+
+
+def _log_routing(
+    chosen_model: str,
+    provider: str,
+    messages: List[Dict[str, Any]],
+    uses_tools: bool,
+) -> None:
+    prompt_text = extract_user_text(messages)
+    long_context    = is_long_context(messages)
+    code_present    = contains_code(prompt_text)
+    math_heavy      = is_math_heavy(prompt_text)
+    structured_task = is_structured_task(prompt_text)
+    multi_q         = multiple_questions(prompt_text)
+    has_images      = contains_images(messages)
+    print(
+        f"\n[ADVANCED ROUTER]\n"
+        f"  Uses tools:    {uses_tools}\n"
+        f"  Long context:  {long_context}\n"
+        f"  Code present:  {code_present}\n"
+        f"  Math heavy:    {math_heavy}\n"
+        f"  Structured:    {structured_task}\n"
+        f"  Multi-question:{multi_q}\n"
+        f"  Has images:    {has_images}\n"
+        f"  → Selected:    {chosen_model} ({provider})\n"
+    )
+
+
+# ──────────────────────────────────────────────
+# CENTRAL HTTP CALL
+# ──────────────────────────────────────────────
+
+def _get_provider_url_and_key(provider: str) -> Tuple[str, str]:
+    """Return (url, api_key) for the given provider, raising on misconfiguration."""
+    if provider == "groq":
+        keys = [k.strip() for k in os.getenv("GROQ_KEY", "").split(",") if k.strip()]
+        if not keys:
+            raise HTTPException(500, "Missing GROQ_KEY(s)")
+        return "https://api.groq.com/openai/v1/chat/completions", random.choice(keys)
+
+    if provider == "cerebras":
+        keys = [k.strip() for k in os.getenv("CER_KEY", "").split(",") if k.strip()]
+        if not keys:
+            raise HTTPException(500, "Missing CER_KEY(s)")
+        return "https://api.cerebras.ai/v1/chat/completions", random.choice(keys)
+
+    if provider == "navy vision":
+        keys = [k.strip() for k in os.getenv("NAVY_KEY", "").split(",") if k.strip()]
+        if not keys:
+            raise HTTPException(500, "Missing NAVY_KEY(s)")
+        return "https://api.navy/v1/chat/completions", random.choice(keys)
+
+    if provider == "navy":
+        keys = [k.strip() for k in os.getenv("NAVY_TEXT_ONLY", "").split(",") if k.strip()]
+        if not keys:
+            raise HTTPException(500, "Missing NAVY_TEXT_ONLY key(s)")
+        return "https://api.navy/v1/chat/completions", random.choice(keys)
+
+    raise HTTPException(500, f"Unknown provider: {provider!r}")
+
+
+async def call_chat_completions(
+    messages: List[Dict[str, Any]],
+    model: str,
+    provider: str,
+    extra_body: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+    """
+    Non-streaming chat-completions call.
+
+    Returns the full upstream JSON payload.
+    Raises HTTPException on upstream errors.
+    """
+    url, api_key = _get_provider_url_and_key(provider)
+    headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
+    body = {"model": model, "messages": messages, "stream": False}
+    if extra_body:
+        body.update(extra_body)
+
+    async with httpx.AsyncClient(timeout=None) as client:
+        r = await client.post(url, json=body, headers=headers)
+
+    if r.status_code != 200:
+        raise HTTPException(status_code=r.status_code, detail=r.text[:1000])
+
+    return r.json()
 
 
-def is_lightweight(prompt: str) -> bool:
-    if len(prompt) < 100:
-        for kw in LIGHTWEIGHT_KEYWORDS:
-            if kw in prompt:
-                return True
+def _extract_text_from_response(data: Dict[str, Any]) -> str:
+    try:
+        return data["choices"][0]["message"]["content"] or ""
+    except Exception:
+        return ""
+
+
+def _extract_usage(data: Dict[str, Any]) -> Tuple[int, int]:
+    usage = data.get("usage", {})
+    return usage.get("prompt_tokens", 0), usage.get("completion_tokens", 0)
+
+
+# ──────────────────────────────────────────────
+# HELPER: image generation
+# ──────────────────────────────────────────────
+
+def is_cinematic_image_prompt(prompt: str) -> bool:
+    for kw in CREATIVE_KEYWORDS:
+        if kw in prompt.lower():
+            return True
     return False
 
-async def check_chat_rate_limit(
-    request: Request,
-    authorization: Optional[str],
-    client_id: Optional[str] = None,
-):
-    return await enforce_rate_limit(request, authorization, "cloudChatDaily", client_id)
 
-# -----------------------------
+# ──────────────────────────────────────────────
 # IMAGE GENERATION
-# -----------------------------
+# ──────────────────────────────────────────────
+
 @router.post("/image")
 @router.get("/image/{prompt}")
 async def generate_image(
@@ -95,21 +261,6 @@ async def generate_image(
     authorization: str = Header(None),
     x_client_id: str = Header(None),
 ):
-    """
-    Image generation endpoint.
-    --------------------------------------------------------------
-    • Accepts a plain‑text prompt (GET or JSON body).
-    • Optional JSON fields:
-        - mode:  "fantasy" | "realistic" (keeps current behaviour)
-        - image_urls: list of up to 2 image URLs or base‑64 strings
-    • If *any* image is supplied we always use the Pollinations
-      model **flux-klein** (the “editing” model). Otherwise the
-      original heuristic (flux / zimage) is retained.
-    • Base‑64 images are saved temporarily with the helper
-      `save_base64_image` and served from the asset CDN exactly
-      like the video endpoint does.
-    --------------------------------------------------------------f
-    """
     timeout = httpx.Timeout(300.0, read=300.0)
 
     if prompt is None:
@@ -173,9 +324,10 @@ async def generate_image(
     return Response(content=resp.content, media_type="image/jpeg")
 
 
-# -----------------------------
+# ──────────────────────────────────────────────
 # SFX GENERATION
-# -----------------------------
+# ──────────────────────────────────────────────
+
 @router.get("/sfx/{prompt}")
 @router.post("/sfx")
 async def gensfx(
@@ -206,9 +358,10 @@ async def gensfx(
     return Response(resp.content, media_type="audio/mpeg")
 
 
-# -----------------------------
+# ──────────────────────────────────────────────
 # TTS GENERATION
-# -----------------------------
+# ──────────────────────────────────────────────
+
 @router.get("/tts/{prompt}")
 @router.post("/tts")
 async def gentts(
@@ -239,13 +392,19 @@ async def gentts(
     return Response(resp.content, media_type="audio/mpeg")
 
 
-# -----------------------------
+# ──────────────────────────────────────────────
 # VIDEO GENERATION (Pollinations)
-# -----------------------------
+# ──────────────────────────────────────────────
+
 @router.get("/video/{prompt}")
 @router.post("/video")
 @router.head("/video")
-async def genvideo(request: Request, prompt: str = None, authorization: str = Header(None), x_client_id: str = Header(None)):
+async def genvideo(
+    request: Request,
+    prompt: str = None,
+    authorization: str = Header(None),
+    x_client_id: str = Header(None),
+):
     if request.method == "HEAD":
         return Response(
             status_code=200,
@@ -270,8 +429,6 @@ async def genvideo(request: Request, prompt: str = None, authorization: str = He
     inputMode = "normal"
     duration = 5
     image_urls = None
-    ratio = None
-    mode = None
 
     if prompt is None:
         user_body = await request.json()
@@ -282,18 +439,12 @@ async def genvideo(request: Request, prompt: str = None, authorization: str = He
         duration = user_body.get("duration", 5)
 
         if ratio not in valid_ratios:
-            raise HTTPException(
-                status_code=400,
-                detail=f"Invalid aspect ratio '{ratio}'. Must be one of 3:2, 2:3, or 1:1.",
-            )
+            raise HTTPException(400, f"Invalid aspect ratio '{ratio}'. Must be one of 3:2, 2:3, or 1:1.")
         if ratio in ratios:
             aspectRatio = ratio
 
         if mode not in valid_modes:
-            raise HTTPException(
-                status_code=400,
-                detail=f"Invalid mode '{mode}'. Must be 'normal' or 'fun'.",
-            )
+            raise HTTPException(400, f"Invalid mode '{mode}'. Must be 'normal' or 'fun'.")
         if mode in modes:
             inputMode = mode
 
@@ -303,23 +454,16 @@ async def genvideo(request: Request, prompt: str = None, authorization: str = He
             if len(image_urls) > 2:
                 raise HTTPException(400, "You may provide at most two image URLs")
 
-        # Clamp duration
         try:
             duration = max(1, min(10, int(duration)))
         except (TypeError, ValueError):
             duration = 5
 
     prompt = normalize_prompt_value(prompt, "prompt")
-    enforce_prompt_size(
-        prompt, MAX_MEDIA_PROMPT_CHARS, MAX_MEDIA_PROMPT_BYTES, "Video prompt"
-    )
+    enforce_prompt_size(prompt, MAX_MEDIA_PROMPT_CHARS, MAX_MEDIA_PROMPT_BYTES, "Video prompt")
     await check_video_rate_limit(request, authorization, x_client_id)
 
-    RATIO_MAP = {
-        "3:2": "16:9",
-        "2:3": "9:16",
-        "1:1": "9:16",
-    }
+    RATIO_MAP = {"3:2": "16:9", "2:3": "9:16", "1:1": "9:16"}
     pollinations_ratio = RATIO_MAP.get(aspectRatio, "16:9")
 
     encoded_prompt = quote(prompt, safe="")
@@ -334,27 +478,23 @@ async def genvideo(request: Request, prompt: str = None, authorization: str = He
 
     if image_urls:
         processed_urls = []
-
         for img in image_urls[:2]:
             if is_base64_image(img):
                 image_id = save_base64_image(img)
                 temp_assets.append(image_id)
-
                 served_url = f"{request.base_url}asset-cdn/assets/{image_id}"
                 processed_urls.append(served_url)
             else:
                 processed_urls.append(img)
-
         params["image"] = "|".join(processed_urls)
 
     if inputMode == "fun":
         params["enhance"] = "true"
 
     query_string = "&".join(f"{k}={quote(str(v), safe='')}" for k, v in params.items())
-    url = f"https://gen.pollinations.ai/image/{encoded_prompt}?{query_string}"
-
+    url = f"https://gen.pollinations.ai/image/{encoded_prompt}?{query_string}&key={PKEY}"
     print(f"[VIDEO GEN] Pollinations URL: {url}")
-    url = url + f"&key={PKEY}"
+
     resp = None
     try:
         async with httpx.AsyncClient(timeout=600) as client:
@@ -362,8 +502,10 @@ async def genvideo(request: Request, prompt: str = None, authorization: str = He
     finally:
         for aid in temp_assets:
             cleanup_image(aid)
+
     if resp is None:
         raise HTTPException(502, "Video generation request failed")
+
     if resp.status_code != 200:
         body_text = ""
         try:
@@ -392,6 +534,11 @@ async def genvideo(request: Request, prompt: str = None, authorization: str = He
         },
     )
 
+
+# ──────────────────────────────────────────────
+# VIDEO GENERATION (Airforce)
+# ──────────────────────────────────────────────
+
 @router.get("/video/airforce/{prompt}")
 @router.post("/video/airforce")
 async def genvideo_airforce(
@@ -404,9 +551,7 @@ async def genvideo_airforce(
         return Response(
             status_code=200,
             headers={
-                # Required field
                 "Y-prompt": "string — required. The text prompt used to generate the video.",
-                # Optional fields
                 "Y-ratio": "string — optional. Aspect ratio of the output video.",
                 "Y-ratio-values": "3:2,2:3,1:1",
                 "Y-ratio-default": "3:2",
@@ -417,9 +562,7 @@ async def genvideo_airforce(
                 "Y-duration-default": "5",
                 "Y-image_urls": "array<string> — optional. Up to 2 image URLs for conditioning.",
                 "Y-image_urls-max": "2",
-                # Response format
                 "Y-response_format": "video/mp4",
-                # Model info
                 "Y-model": "grok-imagine-video",
             },
         )
@@ -427,10 +570,7 @@ async def genvideo_airforce(
     aspectRatio = "3:2"
     inputMode = "normal"
     image_urls = None
-    ratio = None
-    mode = None
 
-    user_body = {}
     if prompt is None:
         user_body = await request.json()
         prompt = user_body.get("prompt")
@@ -439,32 +579,23 @@ async def genvideo_airforce(
         image_urls = user_body.get("image_urls")
 
         if ratio not in valid_ratios:
-            raise HTTPException(
-                status_code=400,
-                detail=f"Invalid aspect ratio {ratio}. Must be one of 3:2, 2:3, or 1:1. Default is 3:2",
-            )
+            raise HTTPException(400, f"Invalid aspect ratio {ratio}. Must be one of 3:2, 2:3, or 1:1. Default is 3:2")
         if ratio in ratios:
             aspectRatio = ratio
 
         if mode not in valid_modes:
-            raise HTTPException(
-                status_code=400,
-                detail=f"Invalid mode {mode}. Must be 'normal' or 'fun'. Default is normal",
-            )
+            raise HTTPException(400, f"Invalid mode {mode}. Must be 'normal' or 'fun'. Default is normal")
         if mode in modes:
             inputMode = mode
 
         if image_urls:
             if not isinstance(image_urls, list):
                 raise HTTPException(400, "image_urls must be a list")
-
             if len(image_urls) > 2:
                 raise HTTPException(400, "You may provide at most two image URLs")
 
     prompt = normalize_prompt_value(prompt, "prompt")
-    enforce_prompt_size(
-        prompt, MAX_MEDIA_PROMPT_CHARS, MAX_MEDIA_PROMPT_BYTES, "Video prompt"
-    )
+    enforce_prompt_size(prompt, MAX_MEDIA_PROMPT_CHARS, MAX_MEDIA_PROMPT_BYTES, "Video prompt")
     await check_video_rate_limit(request, authorization, x_client_id)
 
     payload = {
@@ -484,10 +615,7 @@ async def genvideo_airforce(
     async with httpx.AsyncClient(timeout=600) as client:
         resp = await client.post(
             AIRFORCE_API_URL,
-            headers={
-                "Authorization": f"Bearer {AIRFORCE_KEY}",
-                "Content-Type": "application/json",
-            },
+            headers={"Authorization": f"Bearer {AIRFORCE_KEY}", "Content-Type": "application/json"},
             json=payload,
         )
 
@@ -516,16 +644,20 @@ async def genvideo_airforce(
             "Accept-Ranges": "bytes",
         },
     )
-MODEL_MAP = {
-    "llama-3.1-8b-instant": "Meta Llama 3.1 8B Instant",
-    "gpt-4o-mini": "OpenAI GPT 4o Mini",
-    "nemotron-3-super": "NVIDIA Nemotron 3 Super",
-    "openai/gpt-oss-120b": "OpenAI GPT-OSS 120B",
-    "openai/gpt-oss-20b": "OpenAI GPT-OSS 20B",
-    "qwen-3-235b-a22b-instruct-2507": "Qwen3 Instruct",
-    "llama-3.3-70b-versatile": "Meta Llama 3.3 70B Versatile",
-    "meta-llama/llama-4-scout-17b-16e-instruct": "Meta Llama 4 Scout"
-}
+
+
+# ──────────────────────────────────────────────
+# CHAT COMPLETIONS  (/gen/chat/completions)
+# ──────────────────────────────────────────────
+
+async def _check_chat_rate_limit(
+    request: Request,
+    authorization: Optional[str],
+    client_id: Optional[str] = None,
+):
+    return await enforce_rate_limit(request, authorization, "cloudChatDaily", client_id)
+
+
 @router.post("/chat/completions")
 async def generate_text(
     request: Request,
@@ -537,422 +669,158 @@ async def generate_text(
     if not isinstance(messages, list) or len(messages) == 0:
         raise HTTPException(400, "messages[] is required")
 
-    total_chars, total_bytes = calculate_messages_size(messages)
-    # if total_chars > MAX_CHAT_PROMPT_CHARS or total_bytes > MAX_CHAT_PROMPT_BYTES:
-    #     raise HTTPException(
-    #         status_code=413,
-    #         detail=(
-    #             f"Prompt context too large ({total_chars} chars, {total_bytes} bytes). "
-    #             f"Max allowed is {MAX_CHAT_PROMPT_CHARS} chars or {MAX_CHAT_PROMPT_BYTES} bytes."
-    #         ),
-    #     )
-
-    prompt_text = extract_user_text(messages)
-
     uses_tools = (
         "tools" in body and isinstance(body["tools"], list) and len(body["tools"]) > 0
     ) or ("tool_choice" in body and body["tool_choice"] not in [None, "none"])
 
-    long_context = is_long_context(messages)
-    code_present = contains_code(prompt_text)
-    math_heavy = is_math_heavy(prompt_text)
-    structured_task = is_structured_task(prompt_text)
-    multi_q = multiple_questions(prompt_text)
-    code_heavy = is_code_heavy(prompt_text, code_present, long_context)
-
-    score = 0
-
-    if long_context:
-        score += 3
-
-    if math_heavy:
-        score += 3
-
-    if structured_task:
-        score += 2
-
-    if code_present:
-        score += 2
-
-    if multi_q:
-        score += 1
-
-    for kw in REASONING_KEYWORDS:
-        if kw in prompt_text:
-            score += 1
-
-    chosen_model = "llama-3.1-8b-instant"
-    provider = "groq"
-    has_images = contains_images(messages)
+    chosen_model, provider = route_chat(messages, uses_tools=uses_tools)
+    _log_routing(chosen_model, provider, messages, uses_tools)
 
-    if has_images:
-        chosen_model = "gpt-4o-mini"
-        provider = "navy vision"
-    else:
-        if score > 10:
-            score = 10
-        if uses_tools:
-            if score >= 6:
-                chosen_model = "nemotron-3-super"
-                provider = "navy"
-            elif score >= 4:
-                chosen_model = "openai/gpt-oss-120b"
-                provider = "groq"
-            else:
-                chosen_model = "openai/gpt-oss-20b"
-                provider = "groq"
-    
-        elif code_present:
-    
-            if code_heavy and score >= 6:
-                chosen_model = "o3-mini"
-                provider = "navy"
-    
-            elif score >= 4:
-                chosen_model = "llama-3.3-70b-versatile"
-                provider = "groq"
-    
-        elif score >= 4:
-            chosen_model = "meta-llama/llama-4-scout-17b-16e-instruct"
-            provider = "groq"
-
-        elif score >= 6:
-            chosen_model = "sonar"
-            provider = "navy"
-
-        if provider == "groq" and (
-            total_chars > MAX_GROQ_PROMPT_CHARS or total_bytes > MAX_GROQ_PROMPT_BYTES
-        ):
-            provider = "navy"
-            chosen_model = "gpt-4o-mini"
-
-    await check_chat_rate_limit(request, authorization, x_client_id)
+    await _check_chat_rate_limit(request, authorization, x_client_id)
 
     body["model"] = chosen_model
-    print(
-        f"""
-    [ADVANCED ROUTER]
-      Score: {score}
-      Uses tools: {uses_tools}
-      Long context: {long_context}
-      Code present: {code_present}
-      Math heavy: {math_heavy}
-      Structured: {structured_task}
-      Multi-question: {multi_q}
-      MULTIMODAL REQUIRED: {has_images}
-      → Selected: {chosen_model} ({provider})
-    """
-    )
-
     stream = body.get("stream", False)
-    fallback_model = "meta-llama/llama-4-scout-17b-16e-instruct"
-    fallback_provider = "groq"
-    if provider == "groq":
-        groq_keys = os.getenv("GROQ_KEY", "")
-        print(f"ENV VAR: {groq_keys}")
-        groq_keys_list = [k.strip() for k in groq_keys.split(",") if k.strip()]
-        print(f"PARSED ENV VAR LIST: {groq_keys_list}")
-        if not groq_keys_list:
-            raise HTTPException(500, "Missing GROQ_KEY(s)")
-        API_KEY = random.choice(groq_keys_list)
-        print(f"SELECTED API KEY: {API_KEY}")
-        url = "https://api.groq.com/openai/v1/chat/completions"
-    
-    elif provider == "cerebras":
-        cer_keys = os.getenv("CER_KEY", "")
-        cer_keys_list = [k.strip() for k in cer_keys.split(",") if k.strip()]
-        if not cer_keys_list:
-            raise HTTPException(500, "Missing CER_KEY(s)")
-        API_KEY = random.choice(cer_keys_list)
-    
-        url = "https://api.cerebras.ai/v1/chat/completions"
-
-    elif provider == "navy vision":
-        navy_keys = os.getenv("NAVY_KEY", "")
-        navy_keys_list = [k.strip() for k in navy_keys.split(",") if k.strip()]
-        if not navy_keys_list:
-            raise HTTPException(500, "Missing NAVY Keys(s)")
-        API_KEY = random.choice(navy_keys_list)
-    
-        url = "https://api.navy/v1/chat/completions"
-
-    elif provider == "navy":
-        navy_keys = os.getenv("NAVY_TEXT_ONLY", "")
-        navy_keys_list = [k.strip() for k in navy_keys.split(",") if k.strip()]
-        if not navy_keys_list:
-            raise HTTPException(500, "Missing NAVY TEXT ONLY Keys(s)")
-        API_KEY = random.choice(navy_keys_list)
-    
-        url = "https://api.navy/v1/chat/completions"
-
-    else:
-        raise HTTPException(500, "Unknown provider routing error")
 
-    headers = {"Authorization": f"Bearer {API_KEY}"}
+    url, api_key = _get_provider_url_and_key(provider)
+    headers = {"Authorization": f"Bearer {api_key}"}
 
     if stream:
         body["stream"] = True
-    
-        async def stream_primary(client, url, body, headers):
-            """
-            Handles the primary provider stream (Navy Vision, Groq, Cerebras, etc.)
-            Returns either:
-            - a StreamingResponse generator, OR
-            - triggers fallback if provider fails
-            """
-            try:
-                async with client.stream("POST", url, json=body, headers=headers) as r:
-    
-                    if r.status_code >= 400:
-                        print("[STREAM FALLBACK] Primary provider failed → switching to Groq fallback")
-                        async for chunk in stream_fallback(client, body):
-                            yield chunk
-                        return
-    
-                    async for line in r.aiter_lines():
-                        if not line:
-                            yield "\n"
-                            continue
-                        if line.startswith("event: error"):
-                            fallback()
-                            
-                            if line.startswith("data:"):
-                                try:
-                                    obj = json.loads(line[5:].strip())
-                                    if isinstance(obj, dict) and "error" in obj and isinstance(obj["error"], dict):
-                                        fallback()
-                                except:
-                                    pass
-    
-                        yield line + "\n"
-    
-            except Exception as e:
-                print(f"[STREAM ERROR] {e}")
-                async for chunk in stream_fallback(client, body):
-                    yield chunk
-    
-    
-        async def stream_fallback(client, body):
-            """
-            Clean fallback stream to Groq 17B.
-            This MUST NOT be nested inside another stream.
-            """
+
+        async def stream_fallback(client: httpx.AsyncClient):
             fallback_body = {
-                "model": fallback_model,
+                "model": FALLBACK_MODEL,
                 "messages": body["messages"],
                 "stream": True,
             }
-    
-            groq_keys = os.getenv("GROQ_KEY", "")
-            groq_keys_list = [k.strip() for k in groq_keys.split(",") if k.strip()]
-            fallback_headers = {"Authorization": f"Bearer {random.choice(groq_keys_list)}"}
-    
+            fb_url, fb_key = _get_provider_url_and_key(FALLBACK_PROVIDER)
+            fb_headers = {"Authorization": f"Bearer {fb_key}"}
             print("[FALLBACK] Starting Groq fallback stream")
-    
-            async with client.stream(
-                "POST",
-                "https://api.groq.com/openai/v1/chat/completions",
-                json=fallback_body,
-                headers=fallback_headers,
-            ) as r:
-    
+
+            async with client.stream("POST", fb_url, json=fallback_body, headers=fb_headers) as r:
                 if r.status_code >= 400:
                     err = (await r.aread()).decode("utf-8", errors="replace")
                     yield f'data: {{"error": "Fallback provider failed: {err[:500]}"}}\n\n'
                     return
-    
                 async for line in r.aiter_lines():
                     if not line:
                         yield "\n"
                         continue
-    
-                    if not line.startswith("data:"):
-                        yield f"data: {line}\n\n"
-                    else:
+                    yield (line if line.startswith("data:") else f"data: {line}\n\n") + "\n"
+
+        async def stream_primary(client: httpx.AsyncClient):
+            try:
+                async with client.stream("POST", url, json=body, headers=headers) as r:
+                    if r.status_code >= 400:
+                        print("[STREAM FALLBACK] Primary provider failed → switching to fallback")
+                        async for chunk in stream_fallback(client):
+                            yield chunk
+                        return
+
+                    async for line in r.aiter_lines():
+                        if not line:
+                            yield "\n"
+                            continue
+                        if line.startswith("data:"):
+                            try:
+                                obj = json.loads(line[5:].strip())
+                                if isinstance(obj, dict) and isinstance(obj.get("error"), dict):
+                                    async for chunk in stream_fallback(client):
+                                        yield chunk
+                                    return
+                            except Exception:
+                                pass
                         yield line + "\n"
-    
-    
+            except Exception as e:
+                print(f"[STREAM ERROR] {e}")
+                async for chunk in stream_fallback(client):
+                    yield chunk
+
         async def event_generator():
             sent_metadata = False
-        
             async with httpx.AsyncClient(timeout=None) as client:
-                async for chunk in stream_primary(client, url, body, headers):
-        
+                async for chunk in stream_primary(client):
                     if not sent_metadata:
-                        meta = {
-                            "router_metadata": {
-                                "model_name": MODEL_MAP.get(chosen_model, chosen_model)
-                            }
-                        }
+                        meta = {"router_metadata": {"model_name": MODEL_MAP.get(chosen_model, chosen_model)}}
                         yield f"data: {json.dumps(meta)}\n\n"
                         sent_metadata = True
-        
                     yield chunk
 
         return StreamingResponse(
             event_generator(),
             media_type="text/event-stream",
-            headers={
-                "Cache-Control": "no-cache",
-                "Connection": "keep-alive",
-                "X-Accel-Buffering": "no",
-            },
+            headers={"Cache-Control": "no-cache", "Connection": "keep-alive", "X-Accel-Buffering": "no"},
         )
 
+    # ── non-streaming ─────────────────────────
+    async with httpx.AsyncClient(timeout=None) as client:
+        r = await client.post(url, json=body, headers=headers)
+
+        # navy-vision fallback
+        if provider == "navy vision" and r.status_code >= 400:
+            print("[FALLBACK] Navy vision failed — switching to fallback")
+            fb_url, fb_key = _get_provider_url_and_key(FALLBACK_PROVIDER)
+            fallback_body = dict(body)
+            fallback_body["model"] = FALLBACK_MODEL
+            r = await client.post(fb_url, json=fallback_body, headers={"Authorization": f"Bearer {fb_key}"})
+
+    content_type = (r.headers.get("content-type") or "").lower()
+    if "application/json" in content_type:
+        try:
+            payload = r.json()
+        except Exception:
+            payload = {"error": "Upstream returned invalid JSON"}
     else:
-        async with httpx.AsyncClient(timeout=None) as client:
-            r = await client.post(url, json=body, headers=headers)
-            if provider == "navy vision" and r.status_code >= 400:
-                print("[FALLBACK] Navy vision failed — switching to 17B Groq")
-        
-                groq_keys = os.getenv("GROQ_KEY", "")
-                groq_keys_list = [k.strip() for k in groq_keys.split(",") if k.strip()]
-                if not groq_keys_list:
-                    raise HTTPException(500, "Missing GROQ_KEY(s) for fallback")
-        
-                API_KEY = random.choice(groq_keys_list)
-        
-                fallback_headers = {"Authorization": f"Bearer {API_KEY}"}
-                fallback_body = dict(body)
-                fallback_body["model"] = fallback_model
-        
-                r = await client.post(
-                    "https://api.groq.com/openai/v1/chat/completions",
-                    json=fallback_body,
-                    headers=fallback_headers,
-                )
-        content_type = (r.headers.get("content-type") or "").lower()
-        if "application/json" in content_type:
-            try:
-                payload = r.json()
-            except Exception:
-                payload = {"error": "Upstream returned invalid JSON"}
-        else:
-            payload = {
-                "error": "Upstream returned non-JSON response",
-                "status_code": r.status_code,
-                "message": r.text[:1000],
-            }
+        payload = {
+            "error": "Upstream returned non-JSON response",
+            "status_code": r.status_code,
+            "message": r.text[:1000],
+        }
 
-        return JSONResponse(status_code=r.status_code, content=payload)
+    return JSONResponse(status_code=r.status_code, content=payload)
 
-    raise HTTPException(500, "Unknown provider routing error")
+
+# ──────────────────────────────────────────────
+# PROMPT ANALYZE  (/gen/prompt_analyze)
+# ──────────────────────────────────────────────
 
 @router.post("/prompt_analyze")
-async def analyze_prompt(
-    request: Request
-):
+async def analyze_prompt(request: Request):
     body = await request.json()
     messages = body.get("prompt", [])
     if not isinstance(messages, list) or len(messages) == 0:
         raise HTTPException(400, "messages[] is required")
 
-    total_chars, total_bytes = calculate_messages_size(messages)
-    prompt_text = extract_user_text(messages)
-
     uses_tools = (
         "tools" in body and isinstance(body["tools"], list) and len(body["tools"]) > 0
     ) or ("tool_choice" in body and body["tool_choice"] not in [None, "none"])
 
-    long_context = is_long_context(messages)
-    code_present = contains_code(prompt_text)
-    math_heavy = is_math_heavy(prompt_text)
-    structured_task = is_structured_task(prompt_text)
-    multi_q = multiple_questions(prompt_text)
-    code_heavy = is_code_heavy(prompt_text, code_present, long_context)
-
-    score = 0
+    chosen_model, _ = route_chat(messages, uses_tools=uses_tools)
+    return {MODEL_MAP.get(chosen_model, chosen_model)}
 
-    if long_context:
-        score += 3
 
-    if math_heavy:
-        score += 3
-
-    if structured_task:
-        score += 2
-
-    if code_present:
-        score += 2
-
-    if multi_q:
-        score += 1
-
-    for kw in REASONING_KEYWORDS:
-        if kw in prompt_text:
-            score += 1
-
-    chosen_model = "llama-3.1-8b-instant"
-    provider = "groq"
-    has_images = contains_images(messages)
-
-    if has_images:
-        chosen_model = "gpt-4o-mini"
-        provider = "navy vision"
-    else:
-        if score > 10:
-            score = 10
-        if uses_tools:
-            if score >= 6:
-                chosen_model = "nemotron-3-super"
-                provider = "navy"
-            elif score >= 4:
-                chosen_model = "openai/gpt-oss-120b"
-                provider = "groq"
-            else:
-                chosen_model = "openai/gpt-oss-20b"
-                provider = "groq"
-    
-        elif code_present:
-    
-            if code_heavy and score >= 6:
-                chosen_model = "o3-mini"
-                provider = "navy"
-    
-            elif score >= 4:
-                chosen_model = "llama-3.3-70b-versatile"
-                provider = "groq"
-    
-        elif score >= 4:
-            chosen_model = "meta-llama/llama-4-scout-17b-16e-instruct"
-            provider = "groq"
-
-        elif score >= 6:
-            chosen_model = "sonar"
-            provider = "navy"
-
-        if provider == "groq" and (
-            total_chars > MAX_GROQ_PROMPT_CHARS or total_bytes > MAX_GROQ_PROMPT_BYTES
-        ):
-            provider = "navy"
-            chosen_model = "gpt-4o-mini"
-
-    return { MODEL_MAP[chosen_model] }
+# ──────────────────────────────────────────────
+# MODELS LIST
+# ──────────────────────────────────────────────
 
 @router.get("/models")
 def return_models_openai():
     return {
-      "object": "list",
-      "data": [
-        {
-          "id": "lightning",
-          "object": "model",
-          "created": 1767225600,
-          "owned_by": "inferenceport-ai"
-        }
-      ]
+        "object": "list",
+        "data": [
+            {
+                "id": "lightning",
+                "object": "model",
+                "created": 1767225600,
+                "owned_by": "inferenceport-ai",
+            }
+        ],
     }
 
-from uuid import uuid4
-from time import time
-from typing import Any, Dict, List, Optional
-import json
-import os
-import random
-import httpx
 
-from fastapi import Request, HTTPException, Header
-from fastapi.responses import JSONResponse, StreamingResponse
+# ──────────────────────────────────────────────
+# RESPONSES API  (/gen/responses)
+# ──────────────────────────────────────────────
 
 def _resp_id(prefix: str) -> str:
     return f"{prefix}_{uuid4().hex}"
@@ -966,16 +834,17 @@ def _content_to_text(content: Any) -> str:
     if isinstance(content, list):
         parts = []
         for item in content:
-            if isinstance(item, dict):
-                t = item.get("type")
-                if t in ("input_text", "output_text", "text"):
-                    txt = item.get("text")
-                    if isinstance(txt, str):
-                        parts.append(txt)
+            if isinstance(item, dict) and item.get("type") in ("input_text", "output_text", "text"):
+                txt = item.get("text")
+                if isinstance(txt, str):
+                    parts.append(txt)
         return "".join(parts)
     return ""
 
-def _responses_input_to_messages(input_data: Any, instructions: Optional[str] = None) -> List[Dict[str, Any]]:
+def _responses_input_to_messages(
+    input_data: Any,
+    instructions: Optional[str] = None,
+) -> List[Dict[str, Any]]:
     messages: List[Dict[str, Any]] = []
     if instructions:
         messages.append({"role": "developer", "content": instructions})
@@ -992,16 +861,21 @@ def _responses_input_to_messages(input_data: Any, instructions: Optional[str] =
             if not isinstance(item, dict):
                 continue
             role = item.get("role", "user")
-            content = item.get("content", "")
-            text = _content_to_text(content)
+            text = _content_to_text(item.get("content", ""))
             if text:
                 messages.append({"role": role, "content": text})
 
     return messages
 
-def _openai_responses_payload(model: str, text: str, input_tokens: int = 0, output_tokens: int = 0) -> Dict[str, Any]:
+def _build_responses_payload(
+    model: str,
+    text: str,
+    response_id: str,
+    input_tokens: int = 0,
+    output_tokens: int = 0,
+) -> Dict[str, Any]:
     return {
-        "id": _resp_id("resp"),
+        "id": response_id,
         "object": "response",
         "created_at": _resp_ts(),
         "status": "completed",
@@ -1017,276 +891,99 @@ def _openai_responses_payload(model: str, text: str, input_tokens: int = 0, outp
                 "type": "message",
                 "role": "assistant",
                 "status": "completed",
-                "content": [
-                    {
-                        "type": "output_text",
-                        "text": text,
-                        "annotations": []
-                    }
-                ]
+                "content": [{"type": "output_text", "text": text, "annotations": []}],
             }
         ],
         "output_text": text,
         "usage": {
             "input_tokens": input_tokens,
             "output_tokens": output_tokens,
-            "total_tokens": input_tokens + output_tokens
-        }
+            "total_tokens": input_tokens + output_tokens,
+        },
     }
 
-async def _generate_text_from_messages(
-    request: Request,
-    messages: List[Dict[str, Any]],
-    authorization: Optional[str],
-    xclientid: Optional[str],
-) -> Dict[str, Any]:
-    totalchars, totalbytes = calculate_messages_size(messages)
-    prompttext = extract_user_text(messages)
-
-    usestools = False
-    longcontext = is_long_context(messages)
-    codepresent = contains_code(prompttext)
-    mathheavy = is_math_heavy(prompttext)
-    structuredtask = is_structured_task(prompttext)
-    multiq = multiple_questions(prompttext)
-    codeheavy = is_code_heavy(prompttext, codepresent, longcontext)
-
-    score = 0
-    if longcontext:
-        score += 3
-    if mathheavy:
-        score += 3
-    if structuredtask:
-        score += 2
-    if codepresent:
-        score += 2
-    if multiq:
-        score += 1
-    for kw in REASONING_KEYWORDS:
-        if kw in prompttext:
-            score += 1
-    if score > 10:
-        score = 10
-
-    chosenmodel = "llama-3.1-8b-instant"
-    provider = "groq"
-    hasimages = contains_images(messages)
-
-    if hasimages:
-        chosenmodel = "gpt-4o-mini"
-        provider = "navy vision"
-    else:
-        if usestools:
-            if score >= 6:
-                chosenmodel = "nemotron-3-super"
-                provider = "navy"
-            elif score >= 4:
-                chosenmodel = "openai/gpt-oss-120b"
-                provider = "groq"
-            else:
-                chosenmodel = "openai/gpt-oss-20b"
-                provider = "groq"
-        elif codepresent:
-            if codeheavy and score >= 6:
-                chosenmodel = "o3-mini"
-                provider = "navy"
-            elif score >= 4:
-                chosenmodel = "llama-3.3-70b-versatile"
-                provider = "groq"
-        elif score >= 4:
-            chosenmodel = "meta-llama/llama-4-scout-17b-16e-instruct"
-            provider = "groq"
-        elif score >= 6:
-            chosenmodel = "sonar"
-            provider = "navy"
-
-        if provider == "groq" and (totalchars > MAX_GROQ_PROMPT_CHARS or totalbytes > MAX_GROQ_PROMPT_BYTES):
-            provider = "navy"
-            chosenmodel = "gpt-4o-mini"
-
-    await check_chat_rate_limit(request, authorization, xclientid)
-
-    if provider == "groq":
-        groqkeys = os.getenv("GROQ_KEY")
-        groqkeyslist = [k.strip() for k in groqkeys.split(",") if k.strip()] if groqkeys else []
-        if not groqkeyslist:
-            raise HTTPException(status_code=500, detail="Missing GROQ_KEYs")
-        apikey = random.choice(groqkeyslist)
-        url = "https://api.groq.com/openai/v1/chat/completions"
-        headers = {"Authorization": f"Bearer {apikey}", "Content-Type": "application/json"}
-        payload = {"model": chosenmodel, "messages": messages, "stream": False}
-        async with httpx.AsyncClient(timeout=None) as client:
-            r = await client.post(url, json=payload, headers=headers)
-        if r.status_code != 200:
-            raise HTTPException(status_code=r.status_code, detail=r.text[:1000])
-        data = r.json()
-        text = ""
-        try:
-            text = data["choices"][0]["message"]["content"] or ""
-        except Exception:
-            text = ""
-        return {"text": text, "model": chosenmodel, "provider": provider, "raw": data}
-
-    if provider == "navy vision":
-        navykeys = os.getenv("NAVY_KEY")
-        navykeyslist = [k.strip() for k in navykeys.split(",") if k.strip()] if navykeys else []
-        if not navykeyslist:
-            raise HTTPException(status_code=500, detail="Missing NAVY_KEYs")
-        apikey = random.choice(navykeyslist)
-        url = "https://api.navy/v1/chat/completions"
-        headers = {"Authorization": f"Bearer {apikey}", "Content-Type": "application/json"}
-        payload = {"model": chosenmodel, "messages": messages, "stream": False}
-        async with httpx.AsyncClient(timeout=None) as client:
-            r = await client.post(url, json=payload, headers=headers)
-        if r.status_code != 200:
-            raise HTTPException(status_code=r.status_code, detail=r.text[:1000])
-        data = r.json()
-        text = ""
-        try:
-            text = data["choices"][0]["message"]["content"] or ""
-        except Exception:
-            text = ""
-        return {"text": text, "model": chosenmodel, "provider": provider, "raw": data}
-
-    if provider == "navy":
-        navykeys = os.getenv("NAVY_TEXT_ONLY")
-        navykeyslist = [k.strip() for k in navykeys.split(",") if k.strip()] if navykeys else []
-        if not navykeyslist:
-            raise HTTPException(status_code=500, detail="Missing NAVY TEXT ONLY keys")
-        apikey = random.choice(navykeyslist)
-        url = "https://api.navy/v1/chat/completions"
-        headers = {"Authorization": f"Bearer {apikey}", "Content-Type": "application/json"}
-        payload = {"model": chosenmodel, "messages": messages, "stream": False}
-        async with httpx.AsyncClient(timeout=None) as client:
-            r = await client.post(url, json=payload, headers=headers)
-        if r.status_code != 200:
-            raise HTTPException(status_code=r.status_code, detail=r.text[:1000])
-        data = r.json()
-        text = ""
-        try:
-            text = data["choices"][0]["message"]["content"] or ""
-        except Exception:
-            text = ""
-        return {"text": text, "model": chosenmodel, "provider": provider, "raw": data}
-
-    raise HTTPException(status_code=500, detail="Unknown provider routing error")
 
 @router.post("/responses")
 async def create_responses(
     request: Request,
     authorization: Optional[str] = Header(None),
-    xclientid: Optional[str] = Header(None),
+    x_client_id: Optional[str] = Header(None),
 ):
     body = await request.json()
     model = body.get("model")
     input_data = body.get("input")
     instructions = body.get("instructions")
     stream = body.get("stream", True)
-    response_format = body.get("response_format")
 
     if not model:
-        raise HTTPException(status_code=400, detail="model is required")
+        raise HTTPException(400, "model is required")
     if input_data is None:
-        raise HTTPException(status_code=400, detail="input is required")
+        raise HTTPException(400, "input is required")
 
     messages = _responses_input_to_messages(input_data, instructions=instructions)
     if not messages:
-        raise HTTPException(status_code=400, detail="input could not be parsed")
-
+        raise HTTPException(400, "input could not be parsed")
+
+    # ── shared helper: route + call + return (text, input_tokens, output_tokens) ──
+    async def _generate() -> Tuple[str, int, int]:
+        chosen_model, provider = route_chat(messages)
+        await _check_chat_rate_limit(request, authorization, x_client_id)
+        data = await call_chat_completions(messages, chosen_model, provider)
+        text = _extract_text_from_response(data)
+        input_tokens, output_tokens = _extract_usage(data)
+        return text, input_tokens, output_tokens
+
+    # ── non-streaming ─────────────────────────
     if stream is False:
-        result = await _generate_text_from_messages(
-            request=request,
-            messages=messages,
-            authorization=authorization,
-            xclientid=xclientid,
+        text, input_tokens, output_tokens = await _generate()
+        response_id = _resp_id("resp")
+        return JSONResponse(
+            content=_build_responses_payload(model, text, response_id, input_tokens, output_tokens)
         )
-        if "text" not in result:
-            raise HTTPException(status_code=500, detail="upstream generation failed")
-        return JSONResponse(content=_openai_responses_payload(model, result["text"]))
 
+    # ── streaming ─────────────────────────────
     async def event_stream():
         response_id = _resp_id("resp")
-        created = {
+
+        created_evt = {
             "type": "response.created",
             "response": {
                 "id": response_id,
                 "object": "response",
                 "created_at": _resp_ts(),
                 "status": "in_progress",
-                "model": model
-            }
+                "model": model,
+            },
         }
-        yield f"data: {json.dumps(created)}\n\n"
+        yield f"data: {json.dumps(created_evt)}\n\n"
 
-        result = await _generate_text_from_messages(
-            request=request,
-            messages=messages,
-            authorization=authorization,
-            xclientid=xclientid,
-        )
-
-        if "text" in result:
-            text = result["text"]
-            if text:
-                chunk_size = 64
-                for i in range(0, len(text), chunk_size):
-                    delta = text[i:i + chunk_size]
-                    evt = {
-                        "type": "response.output_text.delta",
-                        "response_id": response_id,
-                        "delta": delta
-                    }
-                    yield f"data: {json.dumps(evt)}\n\n"
-
-            completed = {
-                "type": "response.completed",
-                "response": {
-                    "id": response_id,
-                    "object": "response",
-                    "created_at": _resp_ts(),
-                    "status": "completed",
-                    "completed_at": _resp_ts(),
-                    "model": model,
-                    "output_text": result["text"],
-                    "output": [
-                        {
-                            "id": _resp_id("msg"),
-                            "type": "message",
-                            "role": "assistant",
-                            "status": "completed",
-                            "content": [
-                                {
-                                    "type": "output_text",
-                                    "text": result["text"],
-                                    "annotations": []
-                                }
-                            ]
-                        }
-                    ],
-                    "usage": {
-                        "input_tokens": 0,
-                        "output_tokens": 0,
-                        "total_tokens": 0
-                    }
-                }
-            }
-            yield f"data: {json.dumps(completed)}\n\n"
+        try:
+            text, input_tokens, output_tokens = await _generate()
+        except HTTPException as exc:
+            err_evt = {"type": "response.error", "error": {"message": exc.detail}}
+            yield f"data: {json.dumps(err_evt)}\n\n"
             yield "data: [DONE]\n\n"
             return
 
-        err = {
-            "type": "response.error",
-            "error": result.get("error", {"message": "upstream error"})
+        # Stream text in chunks
+        chunk_size = 64
+        for i in range(0, len(text), chunk_size):
+            delta_evt = {
+                "type": "response.output_text.delta",
+                "response_id": response_id,
+                "delta": text[i : i + chunk_size],
+            }
+            yield f"data: {json.dumps(delta_evt)}\n\n"
+
+        completed_evt = {
+            "type": "response.completed",
+            "response": _build_responses_payload(model, text, response_id, input_tokens, output_tokens),
         }
-        yield f"data: {json.dumps(err)}\n\n"
+        yield f"data: {json.dumps(completed_evt)}\n\n"
         yield "data: [DONE]\n\n"
 
     return StreamingResponse(
         event_stream(),
         media_type="text/event-stream",
-        headers={
-            "Cache-Control": "no-cache",
-            "Connection": "keep-alive",
-            "X-Accel-Buffering": "no",
-        },
+        headers={"Cache-Control": "no-cache", "Connection": "keep-alive", "X-Accel-Buffering": "no"},
     )