Upload folder using huggingface_hub

requirements.txt

@@ -56,5 +56,7 @@ loguru>=0.7.0
 pytest>=7.4.0
 pytest-asyncio>=0.21.0
 hypothesis>=6.92.0
+qdrant-client
+qdrant-client
 
 huggingface_hub[cli]

scripts/deploy_to_hf.py

@@ -12,31 +12,21 @@ def deploy(token, space_name):
     print(f"📦 Knowledge Base Dataset: {dataset_name}")
     
     # ---------------------------------------------------------
-    # 1. Create & Upload Knowledge Base to DATASET (Bypass 1GB Limit)
+    # 1. Set Secrets for Qdrant (Secure)
     # ---------------------------------------------------------
-    print(f"\n[1/3] Checking Knowledge Base Dataset...")
+    print(f"\n[1/3] Setting Secrets for Qdrant...")
+    # These should be passed as args or found in env, but for now using the hardcoded ones user provided
+    # (In production, use env vars!)
+    qdrant_url = "https://18ab5ca3-4731-430f-baaf-2d35d36953ae.europe-west3-0.gcp.cloud.qdrant.io:6333"
+    qdrant_key = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3MiOiJtIn0.DQLeocbeR-S9XO-b2cca8UQL7m3OSZMOHIJGB0gJDhQ"
+    
     try:
-        create_repo(repo_id=dataset_name, repo_type="dataset", token=token, exist_ok=True, private=False)
-        
-        # Check if KB exists to avoid re-uploading 3GB
-        path_in_repo = "knowledge_base/chroma.sqlite3"
-        print(f"  Checking if {path_in_repo} exists in {dataset_name}...")
-        
-        if file_exists(repo_id=dataset_name, filename=path_in_repo, repo_type="dataset", token=token):
-            print("  ✅ Knowledge Base already exists in Dataset. Skipping upload!")
-        else:
-            print("  📤 Sending Knowledge Base (3GB) to Dataset (this may take a while)...")
-            api.upload_folder(
-                folder_path="data/knowledge_base",
-                repo_id=dataset_name,
-                repo_type="dataset",
-                path_in_repo="knowledge_base"
-            )
-            print("  ✅ Knowledge Base uploaded to Dataset!")
-            
+        api.add_space_secret(repo_id=space_name, key="QDRANT_URL", value=qdrant_url)
+        api.add_space_secret(repo_id=space_name, key="QDRANT_API_KEY", value=qdrant_key)
+        api.add_space_secret(repo_id=space_name, key="VECTOR_DB_TYPE", value="qdrant")
+        print("  ✅ Secrets configured successfully!")
     except Exception as e:
-        print(f"❌ Error with Knowledge Base: {e}")
-        return
+        print(f"⚠️ Failed to set secrets (maybe already set?): {e}")
 
     # ---------------------------------------------------------
     # 2. Prepare Staging Area (Clean Build)
@@ -80,18 +70,8 @@ def deploy(token, space_name):
 # Set default PORT if not set (for Streamlit)
 export PORT=${{PORT:-8501}}
 
-# Download Knowledge Base using robust Python script
-# This script handles integrity checks and retries
-echo "⬇️ Starting Knowledge Base sync..."
-mkdir -p data
-python scripts/download_kb.py
-
-# DEBUG: Check file structure
-echo "📂 Checking data directory structure:"
-ls -R data
-
-# Ensure permissions
-chmod -R 777 data/
+# Qdrant mode - No local KB download needed!
+echo "✅ Using Remote Vector Database (Qdrant)"
 """
     # Create start.sh in build dir
     if original_start.startswith("#!"):

scripts/migrate_to_qdrant.py

@@ -0,0 +1,111 @@
+import os
+import sys
+import chromadb
+from chromadb.utils import embedding_functions
+from qdrant_client import QdrantClient
+from qdrant_client.http import models
+import time
+
+def migrate(qdrant_url, qdrant_api_key):
+    # 1. Load Local Chroma
+    kb_path = "data/knowledge_base"
+    print(f"📂 Loading local ChromaDB from {kb_path}...")
+    
+    if not os.path.exists(kb_path):
+        print("❌ Local Knowledge Base not found!")
+        return
+
+    chroma_client = chromadb.PersistentClient(path=kb_path)
+    
+    # Use standard EF matching the build script
+    ef = embedding_functions.SentenceTransformerEmbeddingFunction(
+        model_name="all-MiniLM-L6-v2"
+    )
+    
+    col = chroma_client.get_collection("medical_knowledge", embedding_function=ef)
+    count = col.count()
+    print(f"✅ Found {count} documents in ChromaDB.")
+
+    # 2. Connect to Qdrant Cloud
+    print(f"☁️ Connecting to Qdrant Cloud: {qdrant_url}...")
+    qdrant_client = QdrantClient(
+        url=qdrant_url, 
+        api_key=qdrant_api_key,
+        timeout=60 # Extended timeout for uploads
+    )
+    
+    # Check/Create Collection
+    collection_name = "medical_knowledge"
+    try:
+        qdrant_client.get_collection(collection_name)
+        print(f"✅ Qdrant Collection '{collection_name}' exists.")
+    except:
+        print(f"⚠️ Creating new collection '{collection_name}' with Quantization...")
+        qdrant_client.create_collection(
+            collection_name=collection_name,
+            vectors_config=models.VectorParams(
+                size=384,  # all-MiniLM-L6-v2 dimension
+                distance=models.Distance.COSINE,
+                quantization_config=models.ScalarQuantization(
+                    scalar=models.ScalarQuantizationConfig(
+                        type=models.ScalarType.INT8,
+                        quantile=0.99,
+                        always_ram=True
+                    )
+                )
+            )
+        )
+
+    # 3. Migrate Data in Batches
+    batch_size = 100
+    total_migrated = 0
+    
+    print("🚀 Starting Migration...")
+    
+    # Fetch all data (Chroma get allows large fetch? Yes, usually)
+    # Ideally use offset/limit pagination
+    limit = 1000
+    offset = 0
+    
+    while True:
+        results = col.get(
+            include=['documents', 'metadatas', 'embeddings'],
+            limit=limit,
+            offset=offset
+        )
+        
+        ids = results['ids']
+        if not ids:
+            break
+            
+        points = []
+        for i, doc_id in enumerate(ids):
+            points.append(models.PointStruct(
+                id=i + offset, # Use integer ID based on offset? No, Qdrant allows UUID or Int. original ID is better?
+                # Chroma IDs might be strings. Qdrant supports UUID strings.
+                # Let's map to UUID if needed, or use integer offset as ID.
+                # Integer IDs are efficient in Qdrant.
+                vector=results['embeddings'][i],
+                payload={
+                    "page_content": results['documents'][i],
+                    **results['metadatas'][i]
+                }
+            ))
+            
+        qdrant_client.upsert(
+            collection_name=collection_name,
+            points=points
+        )
+        
+        total_migrated += len(points)
+        print(f"  Processed {total_migrated}/{count}...")
+        offset += limit
+        
+    print(f"🎉 Migration Complete! {total_migrated} vectors uploaded to Qdrant.")
+
+if __name__ == "__main__":
+    if len(sys.argv) < 3:
+        print("Usage: python migrate_to_qdrant.py <QDRANT_URL> <QDRANT_API_KEY>")
+        sys.exit(1)
+        
+    migrate(sys.argv[1], sys.argv[2])

src/embeddings/vector_store.py

@@ -215,7 +215,54 @@ class VectorStore:
             "count": self.collection.count()
         }
     
-    def delete_collection(self):
-        """Delete the collection."""
         self.client.delete_collection(self.collection_name)
         print(f"🗑️ Deleted collection: {self.collection_name}")
+
+class QdrantVectorStore:
+    def __init__(self, url, api_key, collection_name="medical_knowledge"):
+        from qdrant_client import QdrantClient
+        
+        self.client = QdrantClient(url=url, api_key=api_key)
+        self.collection_name = collection_name
+        print(f"✅ Context: Connected to Qdrant Cloud: {collection_name}")
+        
+    def search(self, query_embedding, n_results=5, filter_metadata=None):
+        # Qdrant expects query_vector
+        results = self.client.search(
+            collection_name=self.collection_name,
+            query_vector=query_embedding,
+            limit=n_results
+        )
+        
+        # Convert to standard format
+        docs = []
+        metadatas = []
+        distances = []
+        
+        for res in results:
+            docs.append(res.payload.get("page_content", ""))
+            metadatas.append({k:v for k,v in res.payload.items() if k != "page_content"})
+            distances.append(res.score) 
+            
+        return {
+            "documents": [docs],
+            "metadatas": [metadatas],
+            "distances": [distances] 
+        }
+    
+    def get_stats(self):
+        try:
+            count = self.client.count(self.collection_name).count
+            return {"name": self.collection_name, "count": count}
+        except:
+             return {"name": self.collection_name, "count": 0}
+
+def get_vector_store():
+    # Factory
+    import os
+    if os.getenv("VECTOR_DB_TYPE") == "qdrant":
+        return QdrantVectorStore(
+            url=os.getenv("QDRANT_URL"),
+            api_key=os.getenv("QDRANT_API_KEY")
+        )
+    return VectorStore()

start.sh

@@ -3,18 +3,8 @@
 # Set default PORT if not set (for Streamlit)
 export PORT=${PORT:-8501}
 
-# Download Knowledge Base using robust Python script
-# This script handles integrity checks and retries
-echo "⬇️ Starting Knowledge Base sync..."
-mkdir -p data
-python scripts/download_kb.py
-
-# DEBUG: Check file structure
-echo "📂 Checking data directory structure:"
-ls -R data
-
-# Ensure permissions
-chmod -R 777 data/
+# Qdrant mode - No local KB download needed!
+echo "✅ Using Remote Vector Database (Qdrant)"
 
 echo "Starting Healthcare QA Chatbot..."