Update app.py

app.py

@@ -1,49 +1,39 @@
 import gradio as gr
-from transformers import pipeline
-import torch
+from llama_cpp import Llama
 import json
 
-# Using the 1.5B version for speed and stability on free CPU
-model_id = "Qwen/Qwen2.5-1.5B-Instruct"
-
-# Optimized loader for CPU
-pipe = pipeline(
-    "text-generation", 
-    model=model_id, 
-    model_kwargs={"torch_dtype": torch.float32, "low_cpu_mem_usage": True}
+# 1. LOAD THE ENGINE
+# We are using a 4-bit Quantized version of Llama 3.2 3B. 
+# This is the "Owner's Engine" - it runs locally on your Space.
+print("Loading Coretex Engine...")
+llm = Llama.from_pretrained(
+    repo_id="hugging-quants/Llama-3.2-3B-Instruct-Q8_0-GGUF",
+    filename="llama-3.2-3b-instruct-q8_0.gguf",
+    n_ctx=2048,  # Context window
+    n_threads=2  # Matches the 2 vCPUs on Hugging Face Free
 )
 
 def load_knowledge():
-    knowledge = ""
     try:
         with open("knowledge.jsonl", "r") as f:
-            for line in f:
-                data = json.loads(line)
-                knowledge += f"Role: {data['role']} | Info: {data['context']} | Response: {data['response']}\n"
-    except Exception:
-        knowledge = "No extra knowledge found."
-    return knowledge
+            return [json.loads(line) for line in f]
+    except:
+        return []
 
 def coretex_chat(user_input):
-    kb = load_knowledge()
+    knowledge = load_knowledge()
     
-    # Keeping the prompt clean so the 1.5B model stays focused
-    prompt = f"You are Coretex. Use this info:\n{kb}\nUser: {user_input}\nCoretex:"
-
-    response = pipe(prompt, max_new_tokens=128, clean_up_tokenization_spaces=True)
+    # Format your custom knowledge into the prompt
+    context_str = "\n".join([f"Info: {k['context']} -> {k['response']}" for k in knowledge])
     
-    # Extract only the new part of the text
-    full_text = response[0]['generated_text']
-    answer = full_text.split("Coretex:")[-1].strip()
-    return answer
+    prompt = f"<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n" \
+             f"You are Coretex. Use this custom knowledge:\n{context_str}<|eot_id|>" \
+             f"<|start_header_id|>user<|end_header_id|>\n\n{user_input}<|eot_id|>" \
+             f"<|start_header_id|>assistant<|end_header_id|>\n\n"
 
-# The .queue() is the secret to handling "huge limits" 
-# it lines up users so the CPU never crashes
-demo = gr.Interface(
-    fn=coretex_chat, 
-    inputs=gr.Textbox(label="Message Coretex"), 
-    outputs=gr.Textbox(label="Response"),
-    title="Coretex AI System (v1.5B)"
-)
+    # THE ENGINE THINKS HERE
+    output = llm(prompt, max_tokens=150, stop=["<|eot_id|>"], echo=False)
+    return output['choices'][0]['text']
 
+demo = gr.Interface(fn=coretex_chat, inputs="text", outputs="text", title="Coretex Private Engine")
 demo.queue().launch()