Upload 2 files

app.py

@@ -1,12 +1,135 @@
+import os
+import json
+import faiss
+from pathlib import Path
+from git import Repo
+from huggingface_hub import snapshot_download
+from sentence_transformers import SentenceTransformer
+from openai import OpenAI
 import gradio as gr
-from huggingface_hub import InferenceClient
+from openai import AzureOpenAI
 
-"""
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-"""
-client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
 
 
+# ——— Configuration ———
+REPO_URL        = "https://github.com/dotnet/xharness.git"
+REPO_LOCAL_DIR  = Path("artifacts/repo_code")
+HF_REPO_ID      = "kotlarmilos/repository-learning"
+HF_BASE_DIR     = Path("artifacts/repo_hf")
+HF_INDEX_DIR    = HF_BASE_DIR / "dotnet-xharness" / "index"
+METADATA_PATH   = HF_INDEX_DIR / "metadata.json"
+ROOT_DIR        = REPO_LOCAL_DIR              # where your repo code lives
+AZURE_OPENAI_ENDPOINT = os.getenv("AZURE_OPENAI_ENDPOINT")
+AZURE_OPENAI_API_KEY  = os.getenv("AZURE_OPENAI_API_KEY")
+API_VERSION = "2024-12-01-preview"
+EMBEDDER_MODEL  = "sentence-transformers/all-MiniLM-L6-v2"
+OPENAI_MODEL    = "gpt-4o-mini"
+TOP_K           = 5
+
+
+
+# ——— Step 1: Acquire code and artifacts ———
+# Clone or pull the GitHub repo
+if REPO_LOCAL_DIR.exists() and (REPO_LOCAL_DIR / ".git").exists():
+    Repo(REPO_LOCAL_DIR).remotes.origin.pull()
+else:
+    Repo.clone_from(REPO_URL, REPO_LOCAL_DIR)
+
+# Download Hugging Face snapshots for index & metadata
+snapshot_download(
+    repo_id=HF_REPO_ID,
+    local_dir=str(HF_BASE_DIR),
+    local_dir_use_symlinks=False,
+    token=os.getenv("HUGGINGFACE_HUB_TOKEN"),
+)
+
+# ——— Step 2: Load FAISS index & metadata ———
+index = faiss.read_index(str(HF_INDEX_DIR / "index.faiss"))
+with open(METADATA_PATH, "r", encoding="utf-8") as f:
+    metadata = json.load(f)
+
+# ——— Step 3: Prepare embedder & OpenAI client ———
+embedder = SentenceTransformer(EMBEDDER_MODEL)
+openai   = AzureOpenAI(
+            api_version=API_VERSION,
+            azure_endpoint=AZURE_OPENAI_ENDPOINT,
+            api_key=AZURE_OPENAI_API_KEY,
+        )
+
+# ——— Helper: load code snippets by FAISS ID ———
+def load_snippets_from_metadata(ids, metadata, root_dir):
+    snippets = []
+    for idx in ids:
+        entry    = metadata[idx]
+        file_rel = entry["file"]
+        start, end = entry["start_line"], entry["end_line"]
+        file_path = Path(root_dir) / file_rel
+        try:
+            lines = file_path.read_text(encoding="utf-8").splitlines()
+            code  = "\n".join(lines[start-1 : end]).rstrip()
+        except FileNotFoundError:
+            code = f"# ERROR: {file_rel} not found"
+        snippets.append({
+            "file": file_rel,
+            "lines": (start, end),
+            "code": code,
+            "description": entry.get("llm_description", "")
+        })
+    return snippets
+
+# ——— Core: embed question, retrieve, and call OpenAI ———
+def answer_from_index(question: str, top_k: int = TOP_K) -> str:
+    # 1) Encode question
+    q_emb = embedder.encode([question])
+    # 2) Search FAISS
+    _, indices = index.search(q_emb, top_k)
+    ids = indices[0].tolist()
+    # 3) Load code snippets
+    snippets = load_snippets_from_metadata(ids, metadata, ROOT_DIR)
+    # 4) Build context block
+    context_parts = []
+    for snip in snippets:
+        context_parts.append(
+            f"File: {snip['file']} (lines {snip['lines'][0]}–{snip['lines'][1]})\n"
+            "```python\n"
+            f"{snip['code']}\n"
+            "```\n"
+            f"Description: {snip['description']}"
+        )
+    context_block = "\n\n".join(context_parts)
+    # 5) Prompt OpenAI
+    prompt = (
+        "You are a code assistant. Use the following code snippets to answer the user's question.\n\n"
+        f"{context_block}\n\n"
+        "Question:\n" f"{question}\n\n"
+        "Answer:"
+    )
+    resp = openai.chat.completions.create(
+        model=OPENAI_MODEL,
+        messages=[{"role": "user", "content": prompt}],
+        temperature=0.2,
+        max_tokens=512
+    )
+    return resp.choices[0].message.content.strip()
+
+def rewrite_followup(history: list[tuple[str,str]], followup: str) -> str:
+    # history is list of (user,assistant) pairs
+    convo = "\n".join(
+        f"User: {u}\nAssistant: {a}" for u,a in history[-4:]
+    )
+    prompt = (
+        "Given the conversation below, rewrite the final user query into "
+        "a standalone question.\n\n"
+        f"{convo}\nUser: {followup}\n\nStandalone question:"
+    )
+    resp = openai.chat.completions.create(
+        model=OPENAI_MODEL,
+        messages=[{"role":"user","content":prompt}],
+        temperature=0,
+        max_tokens=128
+    )
+    return resp.choices[0].message.content.strip()
+
 def respond(
     message,
     history: list[tuple[str, str]],
@@ -15,30 +138,15 @@ def respond(
     temperature,
     top_p,
 ):
-    messages = [{"role": "system", "content": system_message}]
-
-    for val in history:
-        if val[0]:
-            messages.append({"role": "user", "content": val[0]})
-        if val[1]:
-            messages.append({"role": "assistant", "content": val[1]})
-
-    messages.append({"role": "user", "content": message})
-
-    response = ""
-
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        token = message.choices[0].delta.content
-
-        response += token
-        yield response
+    # Use the existing answer_from_index function to get the response
+    
+    standalone = rewrite_followup(history, message)
+    response = answer_from_index(standalone)
+    yield response
 
+# ——— Gradio interface ———
+def on_submit(question: str) -> str:
+    return answer_from_index(question)
 
 """
 For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
@@ -59,6 +167,5 @@ demo = gr.ChatInterface(
     ],
 )
 
-
 if __name__ == "__main__":
     demo.launch()

requirements.txt

@@ -1 +1,51 @@
-huggingface_hub==0.25.2
+# Core ML dependencies
+torch>=2.0.0
+transformers>=4.30.0
+sentence-transformers>=2.2.2
+faiss-cpu>=1.7.4
+numpy>=1.21.0
+
+# Data processing
+pandas>=1.5.0
+datasets>=2.12.0
+accelerate>=0.20.0
+peft>=0.4.0
+bitsandbytes>=0.39.0
+huggingface_hub>=0.16.0
+
+# GitHub integration  
+PyGithub
+GitPython>=3.1.0
+requests>=2.28.0
+python-dotenv>=1.0.0
+tenacity>=8.2.0
+
+# Tree-sitter parsers
+tree-sitter>=0.20.0
+tree-sitter-python>=0.20.0
+tree-sitter-c>=0.20.0
+tree-sitter-cpp>=0.20.0
+tree-sitter-java>=0.20.0
+tree-sitter-c-sharp>=0.20.0
+tree-sitter-javascript>=0.20.0
+
+# Web interface
+flask>=2.3.0
+flask-session>=0.5.0
+
+# Utilities
+tqdm>=4.64.0
+schedule>=1.2.0
+openai
+
+# Development
+pytest>=7.0.0
+black>=23.0.0
+flake8>=6.0.0
+
+# Transformers library for NLP tasks
+transformers
+matplotlib
+scikit-learn
+gradio
+openai