Update app.py

app.py

@@ -1,27 +1,20 @@
+import os
+os.system("pip install llama-cpp-python==0.2.75 --quiet")
+
 import gradio as gr
-from transformers import AutoTokenizer, AutoModelForCausalLM
-import torch
+from llama_cpp import Llama
 
-# Model path
-MODEL_ID = "TheBloke/dolphin-2.6-mixtral-8x7b-GGUF"
+MODEL_PATH = "dolphin-2.6-mixtral-8x7b.Q4_K_M.gguf"  # Quantized file
 
-# Load model & tokenizer
-tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
-model = AutoModelForCausalLM.from_pretrained(
-    MODEL_ID,
-    torch_dtype=torch.float16,
-    low_cpu_mem_usage=True,
-    device_map="auto"
-)
+# Download model if not present
+if not os.path.exists(MODEL_PATH):
+    os.system(f"wget https://huggingface.co/TheBloke/dolphin-2.6-mixtral-8x7b-GGUF/resolve/main/{MODEL_PATH}")
+
+llm = Llama(model_path=MODEL_PATH, n_ctx=2048, n_threads=4)
 
 def chat_fn(prompt, max_new_tokens=256, temperature=0.7):
-    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
-    outputs = model.generate(
-        **inputs,
-        max_new_tokens=max_new_tokens,
-        temperature=temperature
-    )
-    return tokenizer.decode(outputs[0], skip_special_tokens=True)
+    output = llm(prompt, max_tokens=max_new_tokens, temperature=temperature)
+    return output["choices"][0]["text"]
 
 iface = gr.Interface(
     fn=chat_fn,