Added fashionclip model

.vscode/settings.json

@@ -0,0 +1,4 @@
+{
+    "python-envs.defaultEnvManager": "ms-python.python:conda",
+    "python-envs.defaultPackageManager": "ms-python.python:conda"
+}

Dockerfile

@@ -7,9 +7,12 @@ WORKDIR /app
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
-# 4. Pre-download the model during the build process
+# 4. Pre-download the Fashion-CLIP model during the build process
 # (This prevents the app from timing out when it first starts up)
-RUN python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('clip-ViT-B-32')"
+RUN python -c "from transformers import CLIPModel, CLIPProcessor; \
+    model = CLIPModel.from_pretrained('patrickjohncyh/fashion-clip'); \
+    processor = CLIPProcessor.from_pretrained('patrickjohncyh/fashion-clip'); \
+    print('Fashion-CLIP model preloaded!')"
 
 # 5. Copy the rest of the application code
 COPY . .
@@ -21,4 +24,4 @@ ENV HOME=/home/user \
     PATH=/home/user/.local/bin:$PATH
 
 # 7. Start the application
-CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

app.py

@@ -1,81 +1,78 @@
 from fastapi import FastAPI, HTTPException
 from pydantic import BaseModel
-from sentence_transformers import SentenceTransformer
+from transformers import CLIPProcessor, CLIPModel
 from PIL import Image
 import requests
 from io import BytesIO
-import numpy as np
+import torch
 
 # 1. Initialize the App
 app = FastAPI()
 
-# 2. Load the Model
-# We use 'clip-ViT-B-32'. It is small, fast, and excellent for connecting text/images.
+# 2. Load the Fashion-CLIP Model
+# We use 'CLIPModel' directly to get access to the vector layers.
 # This runs ONCE when the server starts.
-print("Loading Model...")
-model = SentenceTransformer('clip-ViT-B-32')
-tokenizer = model.tokenizer
+print("Loading Fashion-CLIP Model...")
+model_id = "patrickjohncyh/fashion-clip"
+model = CLIPModel.from_pretrained(model_id)
+processor = CLIPProcessor.from_pretrained(model_id)
 print("Model Loaded!")
 
-# 3. Define chunking parameters
-CHUNK_SIZE = 50  # tokens per chunk (CLIP has 77 token limit)
-
-# 4. Define Input Data Structures
+# 3. Define Input Data Structures
 class TextRequest(BaseModel):
     text: str
 
 class ImageRequest(BaseModel):
     image_url: str
 
-# 5. The Home Route (Health Check)
+# 4. The Home Route (Health Check)
 @app.get("/")
 def home():
-    return {"status": "Online", "message": "Send POST requests to /embed-text or /embed-image"}
+    return {"status": "Online", "model": "Fashion-CLIP"}
 
-# 6. Endpoint: Convert Text to Vector
+# 5. Endpoint: Convert Text to Vector
 @app.post("/embed-text")
 def embed_text(req: TextRequest):
     try:
-        # A. Tokenize the input text
-        tokens = tokenizer.encode(req.text)
-        num_tokens = len(tokens)
-        
-        # B. If text fits within chunk size, use direct embedding
-        if num_tokens <= CHUNK_SIZE:
-            embedding = model.encode(req.text).tolist()
-            return {"vector": embedding}
+        # Process text
+        inputs = processor(text=[req.text], return_tensors="pt", padding=True)
         
-        # C. Split tokens into chunks of CHUNK_SIZE
-        chunks = []
-        for i in range(0, num_tokens, CHUNK_SIZE):
-            chunk_tokens = tokens[i:i + CHUNK_SIZE]
-            chunk_text = tokenizer.decode(chunk_tokens)
-            chunks.append(chunk_text)
+        # Calculate features
+        with torch.no_grad():  # Disable gradient calculation for CPU speed
+            text_features = model.get_text_features(**inputs)
         
-        # D. Generate embeddings for each chunk
-        chunk_embeddings = model.encode(chunks)
+        # Normalize the vector (Crucial for Cosine Similarity!)
+        text_features = text_features / text_features.norm(p=2, dim=-1, keepdim=True)
         
-        # E. Average all chunk embeddings
-        avg_embedding = np.mean(chunk_embeddings, axis=0).tolist()
-        
-        return {"vector": avg_embedding}
+        # Convert to standard Python list
+        vector = text_features[0].tolist()
+        return {"vector": vector}
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))
 
-# 7. Endpoint: Convert Image URL to Vector
+# 6. Endpoint: Convert Image URL to Vector
 @app.post("/embed-image")
 def embed_image(req: ImageRequest):
     try:
-        # A. Download the image from Cloudinary/Web
+        # Download image
         response = requests.get(req.image_url)
         if response.status_code != 200:
             raise HTTPException(status_code=400, detail="Could not download image")
         
-        # B. Open image in memory (don't save to disk)
         image = Image.open(BytesIO(response.content))
+
+        # Process image
+        inputs = processor(images=image, return_tensors="pt", padding=True)
         
-        # C. Generate Vector
-        embedding = model.encode(image).tolist()
-        return {"vector": embedding}
+        # Calculate features
+        with torch.no_grad():  # Disable gradient calculation for CPU speed
+            image_features = model.get_image_features(**inputs)
+        
+        # Normalize the vector (Crucial for Cosine Similarity!)
+        image_features = image_features / image_features.norm(p=2, dim=-1, keepdim=True)
+        
+        # Convert to standard Python list
+        vector = image_features[0].tolist()
+        return {"vector": vector}
     except Exception as e:
-        raise HTTPException(status_code=500, detail=str(e))
+        raise HTTPException(status_code=500, detail=str(e))

requirements.txt

@@ -1,6 +1,7 @@
 fastapi
 uvicorn
-sentence-transformers
+transformers
+torch
 requests
 Pillow
-pydantic
+pydantic