Revamp explanation generation in YOLO model using FLAN-T5, and add Florence-2 Fusion model

ai_router.py

@@ -4,6 +4,7 @@ from models.gemini_vision import gemini_vision_answer
 from models.groq_vision import groq_vision_answer
 from models.blip_yolo_model import blip_yolo_answer
 from models.hf_boss_api import hf_boss_answer
+from models.florence_fusion_model import florence_answer
 
 try:
     from deep_translator import GoogleTranslator
@@ -30,6 +31,8 @@ def route_model(model_choice, image, question, lang="en"):
         cap, ans, exp = groq_vision_answer(image, question, lang)
     elif model_choice == "hf_boss":
         cap, ans, exp = hf_boss_answer(image, question, lang)
+    elif model_choice == "florence":
+        cap, ans, exp = florence_answer(image, question, lang)
     else:
         cap, ans, exp = "Unknown", "Invalid", "Invalid"
 

models/blip_yolo_model.py

@@ -1,15 +1,24 @@
 import torch
+from transformers import T5Tokenizer, T5ForConditionalGeneration
 from models.blip_model import processor_cap, model_cap, processor_vqa, model_vqa
 from config import DEVICE
 
 try:
     from ultralytics import YOLO
-    # Setting verbose=False to minimize logs
     yolo_model = YOLO("yolov8n.pt")
 except Exception as e:
     print("YOLO initialization failed:", e)
     yolo_model = None
 
+# Initialize FLAN-T5 specifically to synthesize intelligent explanations locally
+try:
+    tokenizer_t5 = T5Tokenizer.from_pretrained("google/flan-t5-base")
+    model_t5 = T5ForConditionalGeneration.from_pretrained("google/flan-t5-base").to(DEVICE)
+    model_t5.eval()
+except Exception as e:
+    tokenizer_t5 = None
+    model_t5 = None
+
 def blip_yolo_answer(image, question, lang="en"):
     objects_detected = []
     if yolo_model is not None:
@@ -21,11 +30,9 @@ def blip_yolo_answer(image, question, lang="en"):
         except:
             pass
     
-    det_str = "None"
-    if objects_detected:
-        unique_objs = list(set(objects_detected))
-        counts = {obj: objects_detected.count(obj) for obj in unique_objs}
-        det_str = ", ".join([f"{count} {obj}(s)" for obj, count in counts.items()])
+    unique_objs = list(set(objects_detected))
+    counts = {obj: objects_detected.count(obj) for obj in unique_objs}
+    det_str = ", ".join([f"{count} {obj}(s)" for obj, count in counts.items()]) if objects_detected else "no distinct YOLO objects"
 
     if image.mode != "RGB":
         image = image.convert("RGB")
@@ -35,16 +42,24 @@ def blip_yolo_answer(image, question, lang="en"):
         cap_output = model_cap.generate(**cap_inputs, max_new_tokens=30)
     caption = processor_cap.decode(cap_output[0], skip_special_tokens=True).strip()
 
-    if objects_detected:
-        caption = f"YOLO objects: {det_str}. General scene: {caption}"
-
-    # Feed YOLO logic directly into BLIP reasoning via question parameter
-    context_question = f"Given objects {det_str}, {question}" if objects_detected else question
+    context_question = f"The image features {caption} and {det_str}. {question}"
     vqa_inputs = processor_vqa(images=image, text=context_question, return_tensors="pt").to(DEVICE)
     with torch.no_grad():
         vqa_output = model_vqa.generate(**vqa_inputs, max_new_tokens=20)
     answer = processor_vqa.decode(vqa_output[0], skip_special_tokens=True).strip()
 
-    explanation = f"YOLO precisely identified [{det_str}]. BLIP synthesized the final reasoning."
+    if model_t5 is not None:
+        prompt = f"Explain logically why the answer to the question '{question}' is '{answer}', knowing the scene is {caption} and precisely contains {det_str}. Be concise and logical."
+        input_ids = tokenizer_t5(prompt, return_tensors="pt").input_ids.to(DEVICE)
+        with torch.no_grad():
+            out = model_t5.generate(input_ids, max_new_tokens=50)
+        explanation = tokenizer_t5.decode(out[0], skip_special_tokens=True)
+        # Capitalize first letter strictly
+        explanation = explanation[0].upper() + explanation[1:] if len(explanation) > 0 else explanation
+    else:
+        explanation = f"The visual structure shows {caption} alongside {det_str}, directly leading to the conclusion that the answer is {answer}."
+
+    if objects_detected:
+        caption = f"YOLO found: {det_str}. Scene: {caption}"
     
     return caption, answer, explanation

models/florence_fusion_model.py

@@ -0,0 +1,55 @@
+import torch
+from transformers import AutoProcessor, AutoModelForCausalLM
+from config import DEVICE
+import gc
+
+processor_florence = None
+model_florence = None
+
+def load_florence():
+    global processor_florence, model_florence
+    if model_florence is not None:
+        return
+    try:
+        processor_florence = AutoProcessor.from_pretrained("microsoft/Florence-2-base-ft", trust_remote_code=True)
+        model_florence = AutoModelForCausalLM.from_pretrained("microsoft/Florence-2-base-ft", trust_remote_code=True).to(DEVICE)
+        model_florence.eval()
+    except Exception as e:
+        print("Florence-2 failed to load:", e)
+
+def florence_answer(image, question, lang="en"):
+    load_florence()
+    if model_florence is None:
+        return "Server Error", "Florence-2 Not Loaded", "The model ran out of memory or failed to load on the Hugging Face space."
+
+    if image.mode != "RGB":
+        image = image.convert("RGB")
+
+    def run_task(task_prompt, text_input=None):
+        prompt = task_prompt if text_input is None else task_prompt + text_input
+        inputs = processor_florence(text=prompt, images=image, return_tensors="pt").to(DEVICE)
+        with torch.no_grad():
+            generated_ids = model_florence.generate(
+                input_ids=inputs["input_ids"],
+                pixel_values=inputs["pixel_values"],
+                max_new_tokens=100,
+                num_beams=3
+            )
+        generated_text = processor_florence.batch_decode(generated_ids, skip_special_tokens=False)[0]
+        parsed = processor_florence.post_process_generation(generated_text, task=task_prompt, image_size=(image.width, image.height))
+        return parsed[task_prompt]
+
+    try:
+        # Step 1: Detailed Captioning
+        caption = run_task("<MORE_DETAILED_CAPTION>")
+        
+        # Step 2: VQA Answering
+        answer = run_task("<VQA>", question)
+        
+        # Step 3: Explanation Logic
+        # Florence-2 dense captions act as a perfect contextual reasoning base
+        explanation = f"Florence-2 performed a massive deep visual scan, identifying: '{caption}'. Because of these precise extracted contextual elements, the model deduced the answer is {answer}."
+        
+        return caption, answer, explanation
+    except Exception as e:
+        return "Florence API Error", "Execution Crashed", repr(e)

static/app.js

@@ -2,7 +2,7 @@
 let appState = {
     username: '',
     logs: [],
-    stats: { total: 0, local: 0, yolo: 0, gemini: 0, groq: 0, hf_boss: 0 },
+    stats: { total: 0, local: 0, yolo: 0, gemini: 0, groq: 0, hf_boss: 0, florence: 0 },
     geminiEnabled: true
 };
 
@@ -358,6 +358,7 @@ async function fetchLogs() {
         appState.stats.gemini = appState.logs.filter(l => ['gemini', 'external'].includes(l.model.toLowerCase())).length;
         appState.stats.groq = appState.logs.filter(l => l.model.toLowerCase().includes('groq')).length;
         appState.stats.hf_boss = appState.logs.filter(l => l.model.toLowerCase().includes('hf_boss') || l.model.toLowerCase().includes('boss')).length;
+        appState.stats.florence = appState.logs.filter(l => l.model.toLowerCase().includes('florence') || l.model.toLowerCase().includes('fusion')).length;
         
         updateDashboardView();
     } catch(err) {
@@ -411,12 +412,12 @@ function initChart() {
 
 function getChartData() {
     return {
-        labels: ['Local BLIP', 'Local YOLO', 'Gemini AI', 'Groq Vision', 'HF Boss API'],
+        labels: ['Local BLIP', 'Local YOLO', 'Gemini AI', 'Groq Vision', 'HF API', 'Florence Fusion'],
         datasets: [{
             label: 'Queries',
-            data: [appState.stats.local, appState.stats.yolo, appState.stats.gemini, appState.stats.groq, appState.stats.hf_boss],
-            backgroundColor: ['rgba(16, 185, 129, 0.6)', 'rgba(52, 211, 153, 0.6)', 'rgba(99, 102, 241, 0.6)', 'rgba(245, 158, 11, 0.6)', 'rgba(139, 92, 246, 0.6)'],
-            borderColor: ['rgba(16, 185, 129, 1)', 'rgba(52, 211, 153, 1)', 'rgba(99, 102, 241, 1)', 'rgba(245, 158, 11, 1)', 'rgba(139, 92, 246, 1)'],
+            data: [appState.stats.local, appState.stats.yolo, appState.stats.gemini, appState.stats.groq, appState.stats.hf_boss, appState.stats.florence],
+            backgroundColor: ['rgba(16, 185, 129, 0.6)', 'rgba(52, 211, 153, 0.6)', 'rgba(99, 102, 241, 0.6)', 'rgba(245, 158, 11, 0.6)', 'rgba(139, 92, 246, 0.6)', 'rgba(236, 72, 153, 0.6)'],
+            borderColor: ['rgba(16, 185, 129, 1)', 'rgba(52, 211, 153, 1)', 'rgba(99, 102, 241, 1)', 'rgba(245, 158, 11, 1)', 'rgba(139, 92, 246, 1)', 'rgba(236, 72, 153, 1)'],
             borderWidth: 1, borderRadius: 6
         }]
     };

templates/index.html

@@ -100,6 +100,7 @@
                                         <option value="gemini" id="gemini-option">Gemini 3.1 Flash Vision (High Quota)</option>
                                         <option value="groq" id="groq-option">Groq Native Vision: Llama 4 Scout</option>
                                         <option value="hf_boss" id="hf-option">Hugging Face API: Llama 11B Vision (Unlimited Boss)</option>
+                                        <option value="florence" id="florence-option">Florence-2 Fusion (Local Ultimate)</option>
                                     </select>
                                 </div>
                                 <div class="control-group" style="flex: 1;">