Expand to 5 models safely, no secrets

ai_router.py

@@ -2,6 +2,8 @@ from models.blip_model import blip_answer
 from models.reasoning_model import reasoning_answer
 from models.gemini_vision import gemini_vision_answer
 from models.groq_vision import groq_vision_answer
+from models.blip_yolo_model import blip_yolo_answer
+from models.hf_boss_api import hf_boss_answer
 
 try:
     from deep_translator import GoogleTranslator
@@ -19,11 +21,15 @@ def route_model(model_choice, image, question, lang="en"):
 
     # Model Execution
     if model_choice == "local":
-        cap, ans, exp = reasoning_answer(image, question)
+        cap, ans, exp = blip_answer(image, question, lang)
+    elif model_choice == "local_yolo":
+        cap, ans, exp = blip_yolo_answer(image, question, lang)
     elif model_choice == "gemini":
         cap, ans, exp = gemini_vision_answer(image, question, lang)
     elif model_choice == "groq":
         cap, ans, exp = groq_vision_answer(image, question, lang)
+    elif model_choice == "hf_boss":
+        cap, ans, exp = hf_boss_answer(image, question, lang)
     else:
         cap, ans, exp = "Unknown", "Invalid", "Invalid"
 

models/blip_yolo_model.py

@@ -0,0 +1,47 @@
+from models.blip_model import processor_blip, model_blip, tokenizer_t5, model_t5
+try:
+    from ultralytics import YOLO
+    yolo_model = YOLO("yolov8n.pt")
+except Exception as e:
+    print("YOLO initialization failed:", e)
+    yolo_model = None
+
+def blip_yolo_answer(image, question, lang="en"):
+    objects_detected = []
+    if yolo_model is not None:
+        try:
+            results = yolo_model(image)
+            for r in results:
+                for c in r.boxes.cls:
+                    objects_detected.append(yolo_model.names[int(c)])
+        except:
+            pass
+    
+    yolo_context = ""
+    det_str = "None"
+    if objects_detected:
+        unique_objs = list(set(objects_detected))
+        counts = {obj: objects_detected.count(obj) for obj in unique_objs}
+        det_str = ", ".join([f"{count} {obj}(s)" for obj, count in counts.items()])
+        yolo_context = f"YOLO precisely detected: {det_str}. "
+
+    inputs = processor_blip(image, return_tensors="pt")
+    out = model_blip.generate(**inputs, max_new_tokens=50)
+    blip_caption = processor_blip.decode(out[0], skip_special_tokens=True)
+    
+    caption = f"{yolo_context}General context: {blip_caption}".strip()
+
+    prompt = f"""Based on this visual context:
+{caption}
+
+Question: {question}
+
+Provide ONLY the final answer in exact one line. Do not explain here."""
+
+    input_ids = tokenizer_t5(prompt, return_tensors="pt").input_ids
+    out_answer = model_t5.generate(input_ids, max_new_tokens=50)
+    answer = tokenizer_t5.decode(out_answer[0], skip_special_tokens=True)
+
+    explanation = f"YOLO identified specific objects: {det_str}. BLIP provided the overall scene context. FLAN synthesized the concise answer based on these inputs."
+    
+    return caption, answer, explanation

models/hf_boss_api.py

@@ -0,0 +1,84 @@
+import os
+import io
+import base64
+import requests
+import re
+
+def hf_boss_answer(image, question, lang="en"):
+    token = os.getenv("HF_TOKEN")
+    if not token:
+        return "Setup Required", "HF Token is missing.", "Please set HF_TOKEN."
+
+    try:
+        model_id = "meta-llama/Llama-3.2-11B-Vision-Instruct"
+        api_url = f"https://api-inference.huggingface.co/models/{model_id}/v1/chat/completions"
+        
+        headers = {
+            "Authorization": f"Bearer {token}",
+            "Content-Type": "application/json"
+        }
+        
+        if image.mode != 'RGB':
+            image = image.convert('RGB')
+        
+        image.thumbnail((800, 800))
+        buffered = io.BytesIO()
+        image.save(buffered, format="JPEG", quality=85)
+        img_b64 = base64.b64encode(buffered.getvalue()).decode("utf-8")
+        img_data_url = f"data:image/jpeg;base64,{img_b64}"
+
+        prompt = f"""Analyze this image carefully.
+        
+Question: {question}
+
+CRITICAL RULES:
+1. You MUST respond strictly in language code: {lang}. All text in output must be translated to '{lang}'.
+2. You MUST use EXACTLY the format below with these English labels.
+3. Caption MUST be exactly 1 line only.
+4. Final Answer MUST be exactly 1 line only.
+5. Explanation MUST be strictly 2 to 3 lines maximum.
+
+Respond exactly like this:
+Caption: <one-line caption>
+Final Answer: <one-line answer>
+Explanation: <2-3 lines max explanation>"""
+
+        payload = {
+            "model": model_id,
+            "messages": [
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": prompt},
+                        {"type": "image_url", "image_url": {"url": img_data_url}}
+                    ]
+                }
+            ],
+            "max_tokens": 512,
+            "temperature": 0.2
+        }
+        
+        response = requests.post(api_url, headers=headers, json=payload, timeout=60)
+        
+        if response.status_code != 200:
+            return "HF Boss Server Error", f"HTTP {response.status_code}", response.text
+            
+        data = response.json()
+        raw_text = data["choices"][0]["message"]["content"]
+        
+        caption_match = re.search(r'Caption:\s*(.*?)(?=Final Answer:|$)', raw_text, re.IGNORECASE | re.DOTALL)
+        answer_match = re.search(r'Final Answer:\s*(.*?)(?=Explanation:|$)', raw_text, re.IGNORECASE | re.DOTALL)
+        explanation_match = re.search(r'Explanation:\s*(.*)', raw_text, re.IGNORECASE | re.DOTALL)
+
+        caption = caption_match.group(1).strip() if caption_match else "Caption not generated correctly."
+        answer = answer_match.group(1).strip() if answer_match else raw_text.split('\n')[0]
+        explanation = explanation_match.group(1).strip() if explanation_match else "Explanation not generated correctly."
+
+        caption = caption.replace("**", "").replace("\n", " ")
+        answer = answer.replace("**", "").replace("\n", " ")
+        explanation = explanation.replace("**", "")
+
+        return caption, answer, explanation
+        
+    except Exception as e:
+        return "HF Boss API Error", "The request crashed.", repr(e)

requirements.txt

@@ -10,4 +10,5 @@ deep-translator==1.11.4
 einops>=0.6.1
 timm>=0.9.2
 huggingface_hub>=0.24.6
-
+ultralytics==8.1.0
+opencv-python-headless

static/app.js

@@ -2,7 +2,7 @@
 let appState = {
     username: '',
     logs: [],
-    stats: { total: 0, local: 0, gemini: 0, groq: 0 },
+    stats: { total: 0, local: 0, yolo: 0, gemini: 0, groq: 0, hf_boss: 0 },
     geminiEnabled: true
 };
 
@@ -353,9 +353,11 @@ async function fetchLogs() {
         
         appState.logs = data.logs || [];
         appState.stats.total = appState.logs.length;
-        appState.stats.local = appState.logs.filter(l => ['local', 'blip', 'reasoning'].includes(l.model.toLowerCase())).length;
+        appState.stats.local = appState.logs.filter(l => ['local', 'blip'].includes(l.model.toLowerCase())).length;
+        appState.stats.yolo = appState.logs.filter(l => ['yolo', 'local_yolo'].includes(l.model.toLowerCase())).length;
         appState.stats.gemini = appState.logs.filter(l => ['gemini', 'external'].includes(l.model.toLowerCase())).length;
         appState.stats.groq = appState.logs.filter(l => l.model.toLowerCase().includes('groq')).length;
+        appState.stats.hf_boss = appState.logs.filter(l => l.model.toLowerCase().includes('hf_boss') || l.model.toLowerCase().includes('boss')).length;
         
         updateDashboardView();
     } catch(err) {
@@ -409,12 +411,12 @@ function initChart() {
 
 function getChartData() {
     return {
-        labels: ['BLIP + FLAN (Local)', 'Gemini AI (Cloud)', 'Groq Vision (Cloud)'],
+        labels: ['Local BLIP', 'Local YOLO', 'Gemini AI', 'Groq Vision', 'HF Boss API'],
         datasets: [{
             label: 'Queries',
-            data: [appState.stats.local, appState.stats.gemini, appState.stats.groq],
-            backgroundColor: ['rgba(16, 185, 129, 0.6)', 'rgba(99, 102, 241, 0.6)', 'rgba(245, 158, 11, 0.6)'],
-            borderColor: ['rgba(16, 185, 129, 1)', 'rgba(99, 102, 241, 1)', 'rgba(245, 158, 11, 1)'],
+            data: [appState.stats.local, appState.stats.yolo, appState.stats.gemini, appState.stats.groq, appState.stats.hf_boss],
+            backgroundColor: ['rgba(16, 185, 129, 0.6)', 'rgba(52, 211, 153, 0.6)', 'rgba(99, 102, 241, 0.6)', 'rgba(245, 158, 11, 0.6)', 'rgba(139, 92, 246, 0.6)'],
+            borderColor: ['rgba(16, 185, 129, 1)', 'rgba(52, 211, 153, 1)', 'rgba(99, 102, 241, 1)', 'rgba(245, 158, 11, 1)', 'rgba(139, 92, 246, 1)'],
             borderWidth: 1, borderRadius: 6
         }]
     };

templates/index.html

@@ -96,8 +96,10 @@
                                     <label><i class="fa-solid fa-microchip"></i> Select Model</label>
                                     <select id="model-selector">
                                         <option value="local">BLIP + FLAN (Local)</option>
+                                        <option value="local_yolo">BLIP + FLAN + YOLO (Local Advanced)</option>
                                         <option value="gemini" id="gemini-option">Gemini 3.1 Flash Vision (High Quota)</option>
-                                        <option value="groq" id="groq-option">Groq Native Vision: Llama 4 Scout (Low Rate Limit)</option>
+                                        <option value="groq" id="groq-option">Groq Native Vision: Llama 4 Scout</option>
+                                        <option value="hf_boss" id="hf-option">Hugging Face API: Llama 11B Vision (Unlimited Boss)</option>
                                     </select>
                                 </div>
                                 <div class="control-group" style="flex: 1;">