Initial commit

app.py

@@ -0,0 +1,132 @@
+import pandas as pd
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+from sklearn.neighbors import NearestNeighbors
+from sentence_transformers import SentenceTransformer
+import numpy as np
+import streamlit as st
+import os
+from rapidfuzz import process
+import re
+
+def load_data(filepath):
+    """Load and deduplicate the dataset."""
+    data = pd.read_csv(filepath)
+    data = data.drop_duplicates(subset=['book_name'], keep='first').reset_index(drop=True)
+    return data
+
+def preprocess_data(data):
+    """Preprocess the dataset by cleaning text fields."""
+    data['book_name'] = data['book_name'].fillna('').str.lower()
+    data['summaries'] = data['summaries'].fillna('').str.lower()
+    data['categories'] = data['categories'].fillna('').str.lower()
+    data['combined_text'] = (
+        data['book_name'] + ' ' +
+        data['summaries'] * 2 + ' ' +
+        data['categories']
+    )
+    return data
+
+def feature_engineering(data):
+    """Generate features using TF-IDF and SentenceTransformer embeddings."""
+    # TF-IDF Vectorization
+    vectorizer = TfidfVectorizer(stop_words='english')
+    tfidf_matrix = vectorizer.fit_transform(data['combined_text'])
+
+    # SentenceTransformer embeddings
+    model = SentenceTransformer('paraphrase-MiniLM-L3-v2')
+    embeddings_file = "semantic_embeddings.npy"
+    if not os.path.exists(embeddings_file):
+        data['semantic_embeddings'] = data['combined_text'].apply(lambda x: model.encode(x))
+        np.save(embeddings_file, np.vstack(data['semantic_embeddings']))
+    else:
+        semantic_embeddings = np.load(embeddings_file)
+        data['semantic_embeddings'] = [embedding for embedding in semantic_embeddings]
+
+    # Fit NearestNeighbors model
+    nn_model = NearestNeighbors(metric='cosine', algorithm='auto')
+    nn_model.fit(np.vstack(data['semantic_embeddings']))
+
+    return vectorizer, tfidf_matrix, model, nn_model
+
+def calculate_similarity(input_title, data, vectorizer, tfidf_matrix, model, nn_model, n_recommendations=5, min_similarity=0.3):
+    """Calculate similarities and generate recommendations."""
+    try:
+        # Compute TF-IDF similarity
+        input_vector = vectorizer.transform([input_title.lower()])
+        cosine_similarities = cosine_similarity(input_vector, tfidf_matrix).flatten()
+        min_similarity = max(0.1, cosine_similarities.mean() - cosine_similarities.std())
+
+        # Compute semantic similarity using NearestNeighbors
+        input_embedding = model.encode(input_title.lower())
+        distances, indices = nn_model.kneighbors([input_embedding], n_neighbors=len(data))
+
+        # Combine similarities
+        recommended_books = []
+        seen_titles = set()
+
+        for idx, dist in zip(indices[0], distances[0]):
+            book_title = data['book_name'].iloc[idx]
+            round_sim_value = round(1 - dist, 2)  # Convert distance to similarity
+            if book_title not in seen_titles and round_sim_value >= min_similarity and str(book_title).lower() != str(input_title).lower():
+                recommended_books.append((book_title, round_sim_value))
+                seen_titles.add(book_title)
+            if len(recommended_books) == n_recommendations:
+                break
+
+        # If no meaningful recommendations, return fallback message
+        if not any(score >= min_similarity for _, score in recommended_books):
+            return [("No meaningful recommendations found.", 0.0)]
+
+        return recommended_books
+    except Exception as e:
+        return [(f"An error occurred: {str(e)}", 0.0)]
+
+def validate_input(input_title, data, threshold=70):
+    """Validate input title using fuzzy matching against dataset titles."""
+    titles = data['book_name'].tolist()
+    closest_match, score, _ = process.extractOne(input_title, titles)
+    if score >= threshold:
+        return closest_match
+    return None
+
+def deploy_ui(data, vectorizer, tfidf_matrix, model, nn_model):
+    """Streamlit-based UI for interacting with the recommendation system."""
+    st.title("Content-Based Book Recommender")
+    st.write("Enter a book title to get up to 5 similar recommendations.")
+    book_title = st.text_input("Enter a book title")
+
+    if st.button("Submit"):
+        # Input validation
+        if len(book_title.strip()) < 3:
+            st.write("Input is too short. Please enter a meaningful book title.")
+            return
+
+        # Fuzzy matching for known titles
+        closest_title = validate_input(book_title, data)
+        if closest_title:
+            st.write(f"Did you mean: {closest_title}?")
+            book_title = closest_title
+
+        # Generate recommendations
+        recommendations = calculate_similarity(book_title, data, vectorizer, tfidf_matrix, model, nn_model, min_similarity=0.3)
+        if "error" in recommendations[0][0]:
+            st.write("Error occured. Please try again later")
+        
+        elif recommendations is None or len(recommendations) == 0 or recommendations[0][0] == "No meaningful recommendations found.":
+            st.write("No meaningful recommendations found for the input. Please try again with a different title.")
+        else:
+            st.write("Recommended Books:")
+            idx = 1
+            for book, score in recommendations:
+                cleaned_book = re.sub(r"[^A-Za-z0-9]+$", "", book)
+                st.write(f"{idx}: {cleaned_book}")
+                idx += 1
+            
+
+if __name__ == "__main__":
+    filepath = "books_summary.csv"
+    data = load_data(filepath)
+    data = preprocess_data(data)
+    vectorizer, tfidf_matrix, model, nn_model = feature_engineering(data)
+    deploy_ui(data, vectorizer, tfidf_matrix, model, nn_model)

requirements.txt

@@ -0,0 +1,6 @@
+streamlit
+pandas
+scikit-learn
+sentence-transformers
+numpy
+rapidfuzz

semantic_embeddings.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fca7d1a53c7299e5278cdbb7fe42495072d0eb5d96676431ece0d6b68ebf4043
+size 1890944