Make the pipelines functional with HF Hub + Local Files

environment.yml

@@ -3,343 +3,36 @@
 
 name: resume-matcher
 channels:
-  - pytorch
-  - nvidia
   - conda-forge
   - defaults
 dependencies:
-  - _openmp_mutex=4.5
-  - _python_abi3_support=1.0
-  - aiohappyeyeballs=2.6.1
-  - aiohttp=3.12.15
-  - aiosignal=1.4.0
-  - anyio=4.9.0
-  - argon2-cffi=25.1.0
-  - argon2-cffi-bindings=21.2.0
-  - arrow=1.3.0
-  - asttokens=3.0.0
-  - async-lru=2.0.5
-  - async-timeout=5.0.1
-  - attrs=25.3.0
-  - aws-c-auth=0.9.0
-  - aws-c-cal=0.9.2
-  - aws-c-common=0.12.4
-  - aws-c-compression=0.3.1
-  - aws-c-event-stream=0.5.5
-  - aws-c-http=0.10.4
-  - aws-c-io=0.21.2
-  - aws-c-mqtt=0.13.3
-  - aws-c-s3=0.8.6
-  - aws-c-sdkutils=0.2.4
-  - aws-checksums=0.2.7
-  - aws-crt-cpp=0.33.1
-  - aws-sdk-cpp=1.11.606
-  - babel=2.17.0
-  - beautifulsoup4=4.13.4
-  - blas=1.0
-  - bleach=6.2.0
-  - bleach-with-css=6.2.0
-  - bottleneck=1.4.2
-  - brotli=1.1.0
-  - brotli-bin=1.1.0
-  - brotli-python=1.1.0
-  - bzip2=1.0.8
-  - c-ares=1.34.5
-  - ca-certificates=2025.8.3
-  - cached-property=1.5.2
-  - cached_property=1.5.2
-  - cairo=1.18.4
-  - certifi=2025.8.3
-  - cffi=1.17.1
-  - click=8.1.8
-  - colorama=0.4.6
-  - comm=0.2.3
-  - contourpy=1.3.2
-  - cpython=3.10.18
-  - cryptography=45.0.6
-  - cuda-cccl=13.0.50
-  - cuda-cccl_win-64=13.0.50
-  - cuda-cudart=12.1.105
-  - cuda-cudart-dev=12.1.105
-  - cuda-cupti=12.1.105
-  - cuda-libraries=12.1.0
-  - cuda-libraries-dev=12.1.0
-  - cuda-nvrtc=12.1.105
-  - cuda-nvrtc-dev=12.1.105
-  - cuda-nvtx=12.1.105
-  - cuda-opencl=13.0.39
-  - cuda-opencl-dev=13.0.39
-  - cuda-profiler-api=13.0.39
-  - cuda-runtime=12.1.0
-  - cuda-version=13.0
-  - cycler=0.12.1
-  - datasets=2.14.4
-  - debugpy=1.8.15
-  - decorator=5.2.1
-  - defusedxml=0.7.1
-  - dill=0.3.7
-  - double-conversion=3.3.1
-  - exceptiongroup=1.3.0
-  - executing=2.2.0
-  - font-ttf-dejavu-sans-mono=2.37
-  - font-ttf-inconsolata=3.000
-  - font-ttf-source-code-pro=2.038
-  - font-ttf-ubuntu=0.83
-  - fontconfig=2.15.0
-  - fonts-conda-ecosystem=1
-  - fonts-conda-forge=1
-  - fonttools=4.59.0
-  - fqdn=1.5.1
-  - freetype=2.13.3
-  - frozenlist=1.7.0
-  - fsspec=2025.7.0
-  - gmp=6.3.0
-  - gmpy2=2.2.1
-  - graphite2=1.3.14
-  - h11=0.16.0
-  - h2=4.2.0
-  - harfbuzz=11.3.2
-  - hf-xet=1.1.7
-  - hpack=4.1.0
-  - httpcore=1.0.9
-  - httpx=0.28.1
-  - huggingface_hub=0.34.4
-  - hyperframe=6.1.0
-  - icc_rt=2022.1.0
-  - icu=75.1
-  - idna=3.10
-  - importlib-metadata=8.7.0
-  - intel-openmp=2023.1.0
-  - ipykernel=6.29.5
-  - ipython=8.37.0
-  - ipywidgets=8.1.7
-  - isoduration=20.11.0
-  - jedi=0.19.2
-  - jinja2=3.1.6
-  - joblib=1.5.1
-  - json5=0.12.0
-  - jsonpointer=3.0.0
-  - jsonschema=4.25.0
-  - jsonschema-specifications=2025.4.1
-  - jsonschema-with-format-nongpl=4.25.0
-  - jupyter=1.1.1
-  - jupyter-lsp=2.2.6
-  - jupyter_client=8.6.3
-  - jupyter_console=6.6.3
-  - jupyter_core=5.8.1
-  - jupyter_events=0.12.0
-  - jupyter_server=2.16.0
-  - jupyter_server_terminals=0.5.3
-  - jupyterlab=4.4.5
-  - jupyterlab_pygments=0.3.0
-  - jupyterlab_server=2.27.3
-  - jupyterlab_widgets=3.0.15
-  - khronos-opencl-icd-loader=2024.05.08
-  - kiwisolver=1.4.8
-  - krb5=1.21.3
-  - lark=1.2.2
-  - lcms2=2.17
-  - lerc=4.0.0
-  - libabseil=20250512.1
-  - libarrow=21.0.0
-  - libarrow-acero=21.0.0
-  - libarrow-compute=21.0.0
-  - libarrow-dataset=21.0.0
-  - libarrow-substrait=21.0.0
-  - libbrotlicommon=1.1.0
-  - libbrotlidec=1.1.0
-  - libbrotlienc=1.1.0
-  - libclang13=20.1.8
-  - libcrc32c=1.1.2
-  - libcublas=12.1.0.26
-  - libcublas-dev=12.1.0.26
-  - libcufft=11.0.2.4
-  - libcufft-dev=11.0.2.4
-  - libcurand=10.4.0.35
-  - libcurand-dev=10.4.0.35
-  - libcurl=8.14.1
-  - libcusolver=11.4.4.55
-  - libcusolver-dev=11.4.4.55
-  - libcusparse=12.0.2.55
-  - libcusparse-dev=12.0.2.55
-  - libdeflate=1.24
-  - libevent=2.1.12
-  - libexpat=2.7.1
-  - libffi=3.4.6
-  - libfreetype=2.13.3
-  - libfreetype6=2.13.3
-  - libgcc=15.1.0
-  - libglib=2.84.2
-  - libgomp=15.1.0
-  - libgoogle-cloud=2.39.0
-  - libgoogle-cloud-storage=2.39.0
-  - libgrpc=1.73.1
-  - libhwloc=2.11.2
-  - libiconv=1.18
-  - libintl=0.22.5
-  - libjpeg-turbo=3.1.0
-  - liblzma=5.8.1
-  - libnpp=12.0.2.50
-  - libnpp-dev=12.0.2.50
-  - libnvjitlink=12.1.105
-  - libnvjitlink-dev=12.1.105
-  - libnvjpeg=12.1.1.14
-  - libnvjpeg-dev=12.1.1.14
-  - libparquet=21.0.0
-  - libpng=1.6.50
-  - libprotobuf=6.31.1
-  - libre2-11=2025.07.22
-  - libsodium=1.0.20
-  - libsqlite=3.50.3
-  - libssh2=1.11.1
-  - libthrift=0.22.0
-  - libtiff=4.7.0
-  - libutf8proc=2.10.0
-  - libuv=1.48.0
-  - libwebp-base=1.6.0
-  - libwinpthread=12.0.0.r4.gg4f2fc60ca
-  - libxcb=1.17.0
-  - libxml2=2.13.8
-  - libxslt=1.1.43
-  - libzlib=1.3.1
-  - lxml=6.0.0
-  - lz4-c=1.10.0
-  - markupsafe=3.0.2
-  - matplotlib=3.10.3
-  - matplotlib-base=3.10.3
-  - matplotlib-inline=0.1.7
-  - mistune=3.1.3
-  - mkl=2023.1.0
-  - mkl-service=2.4.0
-  - mkl_fft=1.3.11
-  - mkl_random=1.2.8
-  - mpc=1.3.1
-  - mpfr=4.2.1
-  - mpmath=1.3.0
-  - multidict=6.6.3
-  - multiprocess=0.70.15
-  - munkres=1.1.4
-  - nbclient=0.10.2
-  - nbconvert-core=7.16.6
-  - nbformat=5.10.4
-  - nest-asyncio=1.6.0
-  - networkx=3.3
-  - nltk=3.7
-  - notebook=7.4.4
-  - notebook-shim=0.2.4
-  - numexpr=2.11.0
-  - openjpeg=2.5.3
-  - openssl=3.5.2
-  - orc=2.2.0
-  - overrides=7.7.0
-  - packaging=25.0
-  - pandas=1.5.3
-  - pandocfilters=1.5.0
-  - parso=0.8.4
-  - patsy=1.0.1
-  - pcre2=10.45
-  - pdfminer.six=20250506
-  - pickleshare=0.7.5
-  - pillow=11.3.0
-  - pip=25.1.1
-  - pixman=0.46.4
-  - platformdirs=4.3.8
-  - prometheus_client=0.22.1
-  - prompt_toolkit=3.0.51
-  - propcache=0.3.1
-  - psutil=7.0.0
-  - pthread-stubs=0.4
-  - pure_eval=0.2.3
-  - pycparser=2.22
-  - pygments=2.19.2
-  - pyparsing=3.2.3
-  - pyside6=6.9.1
-  - pysocks=1.7.1
-  - python=3.10.18
-  - python-dateutil=2.9.0.post0
-  - python-docx=1.2.0
-  - python-fastjsonschema=2.21.1
-  - python-gil=3.10.18
-  - python-json-logger=2.0.7
-  - python-xxhash=3.5.0
-  - python_abi=3.10
-  - pytorch-cuda=12.1
-  - pytorch-mutex=1.0
-  - pytz=2025.2
-  - pywin32=311
-  - pywinpty=2.0.15
-  - pyyaml=6.0.2
-  - pyzmq=27.0.0
-  - qhull=2020.2
-  - qt6-main=6.9.1
-  - re2=2025.07.22
-  - referencing=0.36.2
-  - rfc3339-validator=0.1.4
-  - rfc3986-validator=0.1.1
-  - rfc3987-syntax=1.1.0
-  - rpds-py=0.26.0
-  - safetensors=0.6.2
-  - scikit-learn=1.7.1
-  - scipy=1.15.3
-  - seaborn=0.13.2
-  - seaborn-base=0.13.2
-  - send2trash=1.8.3
-  - sentence-transformers=5.1.0
-  - setuptools=72.1.0
-  - six=1.17.0
-  - snappy=1.2.2
-  - sniffio=1.3.1
-  - soupsieve=2.7
-  - stack_data=0.6.3
-  - statsmodels=0.14.5
-  - tbb=2021.13.0
-  - terminado=0.18.1
-  - threadpoolctl=3.6.0
-  - tinycss2=1.4.0
-  - tk=8.6.13
-  - tomli=2.2.1
-  - tornado=6.5.1
-  - tqdm=4.67.1
-  - traitlets=5.14.3
-  - types-python-dateutil=2.9.0.20250708
-  - typing_extensions=4.14.1
-  - typing_utils=0.1.0
-  - tzdata=2025b
-  - ucrt=10.0.22621.0
-  - unicodedata2=16.0.0
-  - uri-template=1.3.0
-  - urllib3=2.5.0
-  - vc=14.3
-  - vc14_runtime=14.44.35208
-  - vs2015_runtime=14.44.35208
-  - wcwidth=0.2.13
-  - webcolors=24.11.1
-  - webencodings=0.5.1
-  - websocket-client=1.8.0
-  - wheel=0.45.1
-  - widgetsnbextension=4.0.14
-  - win_inet_pton=1.1.0
-  - winpty=0.4.3
-  - wordcloud=1.9.4
-  - xorg-libxau=1.0.12
-  - xorg-libxdmcp=1.1.5
-  - xxhash=0.8.3
-  - yaml=0.2.5
-  - yarl=1.20.1
-  - zeromq=4.3.5
-  - zipp=3.23.0
-  - zstandard=0.23.0
-  - zstd=1.5.7
+  - python=3.10
+  - pip
+  # Core scientific stack
+  - numpy
+  - pandas
+  - scikit-learn
+  - scipy
+  - joblib
+  - tqdm
+  - matplotlib
+  - seaborn
+  - wordcloud
+  # NLP / ML essentials
+  - faiss-cpu
+  - nltk
+  - statsmodels
+  - huggingface_hub
+  # File handling
+  - openpyxl
+  - lxml
+  - pillow
+  - pyyaml
+  - python-docx
+  - pdfminer.six
+  # pip-only packages
   - pip:
-      - charset-normalizer==3.4.3
-      - filelock==3.19.1
-      - numpy==2.2.6
-      - pyarrow==14.0.2
-      - regex==2025.7.34
-      - requests==2.32.5
-      - sympy==1.13.1
-      - tokenizers==0.19.1
-      - torch==2.8.0
-      - torchaudio==2.5.1+cu121
-      - torchvision==0.20.1+cu121
-      - transformers==4.41.2
+      - sentence-transformers
+      - transformers
+      - accelerate
+      - datasets

pipelines/app_pipeline.py

@@ -1,97 +1,138 @@
 import argparse
 import os
 import pandas as pd
-import faiss
-from sentence_transformers import SentenceTransformer
-from src.feature_engg.tfidf_vectorizing_data import load_vectorizer, load_vector_data
-from src.processing.text_cleaning import clean_text
+from src.feature_engg.tfidf_vectorizing_data import load_tfidf_vectorizer, load_tfidf_matrix
+from src.feature_engg.bert_embedding_data import get_bert_model, load_faiss_index
+from src.processing.text_cleaning import clean_text, clean_text_for_bert
 from src.matching.matching_engine import compute_similarity_matrix, top_n_tfidf_matches, top_n_bert_matches
 from src.utils.file_reader import extract_text_from_file
 
 
 def load_job_titles(job_csv_path: str):
     df = pd.read_csv(job_csv_path)
-    if 'title' not in df.columns:
+    if "title" not in df.columns:
         raise ValueError("Job CSV must contain a 'title' column.")
     return df
 
 
+# ------------------------- TF-IDF PIPELINE -------------------------
 def run_tfidf_pipeline(args, raw_resume: str):
 
-    # Step 2: Clean resume text
+    # Step 1: Clean resume
     cleaned_resume = clean_text(raw_resume)
 
-    # Step 3: Load vectorizer and job matrix
-    vectorizer = load_vectorizer(args.vectorizer_path)
-    job_matrix = load_vector_data(args.job_matrix_path)
-
-    # Step 4: Vectorize cleaned resume text
+    # Step 2: Load vectorizer + job matrix (local first, fallback HF)
+    vectorizer = load_tfidf_vectorizer(
+        local_vectorizer_path=args.local_vectorizer_path,
+        repo_id=args.tfidf_repo_id,
+        filename=args.vectorizer_filename
+    )
+    job_matrix = load_tfidf_matrix(
+        local_matrix_path=args.local_matrix_path,
+        repo_id=args.tfidf_repo_id,
+        filename=args.matrix_filename
+    )
+
+    # Step 3: Vectorize resume
     resume_vector = vectorizer.transform([cleaned_resume])
 
-    # Step 5: Compute similarity
+    # Step 4: Compute cosine similarity
     sim_matrix = compute_similarity_matrix(resume_vector, job_matrix)
 
-    # Step 6: Load job titles
-    job_df = load_job_titles(args.job_title_csv)
+    # Step 5: Load job titles
+    job_df = load_job_titles("data/app_data/tfidf_job_titles.csv")
+
+    # Step 6: Get top-N job matches
+    top_k = args.top_k 
+
+    if args.top_k > len(job_df['title'].unique()):
+        print(f"⚠️ Requested top_k={args.top_k} exceeds unique job titles={len(job_df['title'].unique())}. Reducing top_k.")
+        top_k = len(job_df['title'].unique())
 
-    # Step 7: Get top-N job matches
-    matches = top_n_tfidf_matches(sim_matrix, top_n=args.top_k, job_df=job_df)
+    elif args.top_k is None:
+        top_k = len(job_df['title'].unique())
+        print(f"\nℹ️ Showing all {top_k} job titles.\n")
 
-    print(f"\n🎯 Top {args.top_k} Job Matches for the Resume (TF-IDF):")
+    matches = top_n_tfidf_matches(sim_matrix, top_n=top_k, job_df=job_df)
+
+    print(f"\n🎯 Top {top_k} Job Matches for the Resume (TF-IDF):")
     for job_idx, score in matches[0]:
         print(f"🔹 {job_df.iloc[job_idx]['title']} (score: {score:0.4f})")
 
-    # Optional debug
     if args.debug:
         print("\n================ DEBUG MODE ================")
         print("\n📄--- [DEBUG - TFIDF] Cleaned Resume Preview:\n", cleaned_resume[:1000], "---")
-        print(f"\n--- [DEBUG - TFIDF] Raw Similarity Scores (top {args.top_k}) ---")
+        print(f"\n--- [DEBUG - TFIDF] Raw Similarity Scores (top {top_k}) ---")
         for job_idx, score in matches[0]:
             print(f"[{job_idx}] {job_df.iloc[job_idx]['title']} → {score:0.6f}")
         print("==============================================")
 
 
+# ------------------------- BERT PIPELINE -------------------------
 def run_bert_pipeline(args, raw_resume: str):
-    # Step 2: Load SentenceTransformer model
-    model = SentenceTransformer(args.bert_model_path)
 
-    # Step 3: Load FAISS job index
-    job_index = faiss.read_index(args.bert_faiss_index)
+    # Step 1: Load fine-tuned ST model (local or HF Hub)
+    model = get_bert_model(args.local_bert_path or args.bert_repo_id)
+
+    # Step 2: Load FAISS index (local or HF Hub)
+    job_index = load_faiss_index(
+        local_index_path=args.local_index_path,
+        repo_id=args.bert_repo_id,
+        filename=args.index_filename
+    )
+
+    # Step 3: Clean resume text for transformer
+    cleaned_resume = clean_text_for_bert(raw_resume)
 
-    # Step 4: Encode resume into embedding
-    resume_embedding = model.encode([raw_resume], normalize_embeddings=True)
+    # Step 4: Embed
+    resume_embedding = model.encode(
+        [cleaned_resume],
+        normalize_embeddings=True
+    )
 
-    # Step 5: Search deeply in FAISS index in order to eliminate duplicate job titles
-    # Search across all job embeddings in FAISS
+    # Step 5: Search
     n_jobs = job_index.ntotal
     D, I = job_index.search(resume_embedding, n_jobs)
 
     # Step 6: Load job titles
-    job_df = load_job_titles(args.job_title_csv)
+    job_df = load_job_titles("data/app_data/bert_job_titles.csv")
 
-    print(f"\n�� Top {args.top_k} Job Matches for the Resume (BERT):")
-    matches = top_n_bert_matches(I, D, job_df, top_n=args.top_k)
+    # Step 7: Rank top-N
+    top_k = args.top_k 
 
+    if args.top_k > len(job_df['title'].unique()):
+        print(f"⚠️ Requested top_k={args.top_k} exceeds unique job titles={len(job_df['title'].unique())}. Reducing top_k.")
+        top_k = len(job_df['title'].unique())
+
+    elif args.top_k is None:
+        top_k = len(job_df['title'].unique())
+        print(f"\nℹ️ Showing all {top_k} job titles.\n")
+        
+    matches = top_n_bert_matches(I, D, job_df, top_n=top_k)
+
+    print(f"\n🎯 Top {top_k} Job Matches for the Resume (BERT):")
     for idx, score in matches:
         print(f"🔹 {job_df.iloc[idx]['title']} (score: {score:0.4f})")
 
-    # Optional debug
     if args.debug:
         print("\n================ DEBUG MODE ================")
-        print(f"\n--- [DEBUG - BERT/FAISS] Raw Similarity Scores (top {args.top_k}) ---")
+        print(f"\n--- [DEBUG - BERT/FAISS] Raw Similarity Scores (top {top_k}) ---")
         for idx, score in matches:
-            print(f"🔹 {job_df.iloc[idx]['title']} (score: {score})")
+            print(f"[{idx}] {job_df.iloc[idx]['title']} → {score:0.6f}")
         print("==============================================")
 
 
+# ------------------------- MAIN -------------------------
 def main(args):
     try:
-        # Step 1: Load raw resume text
         if not os.path.exists(args.resume_path):
-            raise FileNotFoundError(f"Resume file not found: {args.resume_path}")
+            raise FileNotFoundError(f"⚠️ Resume file not found at: {args.resume_path}")
+        
         raw_resume = extract_text_from_file(args.resume_path)
+        print(f"\n📄 Resume: {args.resume_path}")
 
-        # Run chosen pipeline
+        # Pipeline selector
+        print(f"⚙️ Using model: {args.model.upper()}")
         if args.model == "bert":
             run_bert_pipeline(args, raw_resume)
         else:
@@ -103,28 +144,36 @@ def main(args):
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Match a resume to top relevant job titles")
-    parser.add_argument('--resume_path', type=str, required=True, help="Path to resume file")
-    parser.add_argument('--model', type=str, choices=['tfidf', 'bert'], default='tfidf',
-                        help="Which model pipeline to use: 'tfidf' or 'bert'")
-
 
-    # TF-IDF arguments
-    parser.add_argument('--vectorizer_path', type=str, default='models/tfidf/app_tfidf/job_tfidf_vectorizer.pkl')
-    parser.add_argument('--job_matrix_path', type=str, default='models/tfidf/app_tfidf/job_tfidf_matrix.npz')
-
-    # BERT arguments
-    parser.add_argument('--bert_model_path', type=str, default='models/bert/dapt_minilm_sentence_transformer',
-                        help="Path to fine-tuned SentenceTransformer model")
-    parser.add_argument('--bert_faiss_index', type=str, default='models/bert/app_bert/jobs_bert_embeddings.faiss',
-                        help="Path to FAISS index of job embeddings")
-
-    # Shared arguments
-    parser.add_argument('--job_title_csv', type=str, default='data/app_data/job_titles.csv')
-    parser.add_argument('--top_k', type=int, default=5,
-                        help="Number of top job matches to return")
-    parser.add_argument('--debug', action='store_true',
-                        help="Print raw similarity scores and cleaned resume for tfidf pipeline")
+    # Shared args
+    parser.add_argument("--resume_path", type=str, required=True, help="Path to resume file")
+    parser.add_argument("--model", type=str, choices=["tfidf", "bert"], default="tfidf")
+    parser.add_argument("--top_k", type=int, default=None,
+                        help="Number of top matches to return if not specified, returns all")
+    parser.add_argument("--debug", action="store_true",
+                        help="print raw similarity scores for both and cleaned resume for tfidf pipeline")
+
+    # TF-IDF args
+    parser.add_argument("--local_vectorizer_path", type=str, default=None,
+                        help="Local TF-IDF vectorizer .pkl file")
+    parser.add_argument("--local_matrix_path", type=str, default=None,
+                        help="Local TF-IDF job matrix .npz file") 
+    parser.add_argument("--tfidf_repo_id", type=str, default="Om-Shandilya/resume-matcher-tfidf",
+                        help="Hub repo id for HuggingFace model")
+    parser.add_argument("--vectorizer_filename", type=str, default="applicant/job_vectorizer.pkl",
+                        help="Filename of vectorizer in the HF repo")
+    parser.add_argument("--matrix_filename", type=str, default="applicant/job_matrix.npz",
+                        help="Filename of matrix in the HF repo")
+
+    # BERT args
+    parser.add_argument("--local_bert_path", type=str, default=None,
+                            help="Local fine-tuned ST model path")
+    parser.add_argument("--local_index_path", type=str, default=None,
+                            help="Local FAISS index file path")
+    parser.add_argument("--bert_repo_id", type=str, default="Om-Shandilya/resume-matcher-bert",
+                        help="fine-tuned ST model's HF repo id")
+    parser.add_argument("--index_filename", type=str, default="applicant/jobs.faiss",
+                        help="Filename of FAISS index in the HF repo")
 
     args = parser.parse_args()
     main(args)
-

pipelines/recruiter_pipeline.py

@@ -1,139 +1,167 @@
 import argparse
 import os
-import joblib
+import numpy as np
 from sklearn.metrics.pairwise import cosine_similarity
-import faiss
-from sentence_transformers import SentenceTransformer
+from src.feature_engg.tfidf_vectorizing_data import load_tfidf_vectorizer
+from src.feature_engg.bert_embedding_data import get_bert_model
 from src.utils.bulk_loading import bulk_load_raw_resume_files
 from src.utils.file_reader import extract_text_from_file
-from src.processing.text_cleaning import clean_text
+from src.processing.text_cleaning import clean_text, clean_text_for_bert
 
 
-def rank_with_tfidf(args, raw_job_text, raw_resume_texts):
-    """TF-IDF recruiter pipeline"""
-    # Step 1: Load vectorizer
-    if not os.path.exists(args.vectorizer_path):
-        raise FileNotFoundError(f"⚠️ Vectorizer file not found: {args.vectorizer_path}")
-    vectorizer = joblib.load(args.vectorizer_path)
+# ------------------------- TF-IDF PIPELINE -------------------------
+def run_tfidf_pipeline(args, raw_job_text, raw_resume_texts):
+    # Step 1: Load vectorizer (local or HF Hub)
+    vectorizer = load_tfidf_vectorizer(
+        local_vectorizer_path=args.local_vectorizer_path,
+        repo_id=args.tfidf_repo_id,
+        filename=args.vectorizer_filename
+    )
 
-    # Step 2: Process job description
+    # Step 2: Clean job description
     cleaned_job_text = clean_text(raw_job_text)
     job_vector = vectorizer.transform([cleaned_job_text])
 
-    # Step 3: Process resumes
+    # Step 3: Clean and vectorize resumes
     cleaned_resumes = {fname: clean_text(txt) for fname, txt in raw_resume_texts.items()}
     resume_matrix = vectorizer.transform(cleaned_resumes.values())
 
     # Step 4: Compute similarity
     sims = cosine_similarity(job_vector, resume_matrix)[0]
 
+    # Step 5: Rank resumes
+    ranked = sorted(zip(cleaned_resumes.keys(), sims), key=lambda x: x[1], reverse=True)
+
+    # Step 6: Top-K handling
+    top_k = args.top_k
+    available_resumes = len(ranked)
+
+    if args.top_k is None:
+        top_k = available_resumes
+        print(f"\nℹ️ Showing all {available_resumes} resumes.\n")
+    elif args.top_k > available_resumes:
+        top_k = available_resumes
+        print(f"\n⚠️ Requested top_k={args.top_k} exceeds available resumes={available_resumes}. Reducing top_k.\n")
+
+    print(f"\n🎯 Top {top_k} Candidate Matches for the Job (TF-IDF):")
+    for i, (fname, score) in enumerate(ranked[:top_k], 1):
+        print(f"{i}. {fname} → score: {score:.4f}")
+
     if args.debug:
         print("\n================ DEBUG MODE ================")
-        print("\n[DEBUG - TFIDF] Cleaned job description:")
-        print(cleaned_job_text[:500], "...\n")
-        print("[DEBUG - TFIDF] First 3 cleaned resumes:")
+        print("\n📄--- [DEBUG - TFIDF] Cleaned Job Description Preview:\n", cleaned_job_text[:1000], "---")
+        print("\n--- [DEBUG - TFIDF] First 3 Cleaned Resumes ---")
         for i, (fname, txt) in enumerate(cleaned_resumes.items()):
             if i >= 3: break
             print(f"{fname}: {txt[:300]}...\n")
-        print("[DEBUG - TFIDF] Raw similarity scores:", sims[:10])
+        print(f"\n--- [DEBUG - TFIDF] Raw Similarity Scores (top {top_k}) ---")
+        for fname, score in ranked[:top_k]:
+            print(f"{fname} → {score:0.6f}")
         print("==============================================")
 
-    # Step 5: Rank resumes
-    ranked = sorted(zip(cleaned_resumes.keys(), sims), key=lambda x: x[1], reverse=True)
-    return ranked
 
+# ------------------------- BERT PIPELINE -------------------------
+def run_bert_pipeline(args, raw_job_text, raw_resume_texts):
+    # Step 1: Load fine-tuned ST model (local or HF Hub)
+    model = get_bert_model(args.local_bert_path or args.bert_repo_id)
+
+    # Step 2: Clean job description
+    cleaned_job_text = clean_text_for_bert(raw_job_text)
+    job_embedding = model.encode([cleaned_job_text], normalize_embeddings=True)
 
-def rank_with_bert(args, raw_job_text, raw_resume_texts):
-    """BERT recruiter pipeline using FAISS (on the fly)"""
-    if not os.path.exists(args.bert_model_path):
-        raise FileNotFoundError(f"⚠️ BERT model not found: {args.bert_model_path}")
+    # Step 3: Encode resumes
+    cleaned_resumes = {fname: clean_text_for_bert(txt) for fname, txt in raw_resume_texts.items()}
+    resume_embeddings = model.encode(list(cleaned_resumes.values()), normalize_embeddings=True)
 
-    # Step 1: Load BERT model
-    model = SentenceTransformer(args.bert_model_path)
+    # Step 4: Compute cosine similarity manually
+    # Using dot product as embeddings are normalized and not FAISS since we have small data here.
+    sims = np.dot(resume_embeddings, job_embedding.T).flatten()
 
-    # Step 2: Encode job description
-    job_embedding = model.encode([raw_job_text], convert_to_numpy=True, normalize_embeddings=True)
+    # Step 5: Rank resumes
+    ranked = sorted(zip(cleaned_resumes.keys(), sims), key=lambda x: x[1], reverse=True)
 
-    # Step 3: Encode resumes
-    resume_embeddings = model.encode(list(raw_resume_texts.values()), convert_to_numpy=True, normalize_embeddings=True)
+    # Step 6: Top-K handling
+    top_k = args.top_k
+    available_resumes = len(ranked)
 
-    # Step 4: Create FAISS indices
-    local_index = faiss.IndexFlatIP(resume_embeddings.shape[1])
-    local_index.add(resume_embeddings)
+    if args.top_k is None:
+        top_k = available_resumes
+        print(f"\nℹ️ Showing all {available_resumes} resumes.\n")
+    elif args.top_k > available_resumes:
+        top_k = available_resumes
+        print(f"\n⚠️ Requested top_k={args.top_k} exceeds available resumes={available_resumes}. Reducing top_k.\n")
 
-    scores, indices = local_index.search(job_embedding, len(raw_resume_texts))
+    print(f"\n🎯 Top {top_k} Candidate Matches for the Job (BERT):")
+    for i, (fname, score) in enumerate(ranked[:top_k], 1):
+        print(f"{i}. {fname} → score: {score:.4f}")
 
     if args.debug:
         print("\n================ DEBUG MODE ================")
-        print("\n[DEBUG - BERT/FAISS] Raw job description:")
-        print(raw_job_text[:500], "...\n")
-        print("[DEBUG - BERT/FAISS] First 3 raw resumes:")
-        for i, (fname, txt) in enumerate(raw_resume_texts.items()):
+        print("\n📄--- [DEBUG - BERT] Cleaned Job Description Preview:\n", cleaned_job_text[:1000], "---")
+        print("\n--- [DEBUG - BERT] First 3 Cleaned Resumes ---")
+        for i, (fname, txt) in enumerate(cleaned_resumes.items()):
             if i >= 3: break
             print(f"{fname}: {txt[:300]}...\n")
-        print(f"[DEBUG - BERT/FAISS] all similarity scores:", scores[0][:len(raw_resume_texts)])
+        print(f"\n--- [DEBUG - BERT] Raw Similarity Scores (top {top_k}) ---")
+        for fname, score in ranked[:top_k]:
+            print(f"{fname} → {score:0.6f}")
         print("==============================================")
 
-    # Step 5: Rank resumes
-    ranked = [(list(raw_resume_texts.keys())[i], float(scores[0][j]))
-              for j, i in enumerate(indices[0])]
-    return ranked
-
 
+# ------------------------- MAIN -------------------------
 def main(args):
     try:
-        # Load raw job and resumes 
+        # Load job description and resumes
+        if not os.path.exists(args.job_desc_path):
+            raise FileNotFoundError(f"⚠️ Job description not found: {args.job_desc_path}")
         raw_job_text = extract_text_from_file(args.job_desc_path)
+
+        if not os.path.exists(args.resume_dir):
+            raise FileNotFoundError(f"⚠️ Resume directory not found: {args.resume_dir}")
         raw_resume_texts = bulk_load_raw_resume_files(args.resume_dir)
 
         if not raw_resume_texts:
             raise ValueError("⚠️ No valid resumes found in the given directory.")
 
-        # Limit the number of resumes displayed based on the top_k argument and available resumes
-        available_resumes = len(raw_resume_texts)
-        top_k = min(args.top_k, available_resumes)
-
-        if args.top_k > available_resumes:
-            print(f"\n⚠️ Only {available_resumes} resumes are available. "
-                  f"Showing top {available_resumes} matches instead of {args.top_k}.\n")
+        print(f"\n📄 Job Description: {args.job_desc_path}")
+        print(f"📂 Loaded {len(raw_resume_texts)} resumes from {args.resume_dir}")
 
-        # Choose model
-        if args.model == "tfidf":
-            ranked = rank_with_tfidf(args, raw_job_text, raw_resume_texts)
-        elif args.model == "bert":
-            ranked = rank_with_bert(args, raw_job_text, raw_resume_texts)
+        # Pipeline selector
+        print(f"⚙️ Using model: {args.model.upper()}")
+        if args.model == "bert":
+            run_bert_pipeline(args, raw_job_text, raw_resume_texts)
         else:
-            raise ValueError("❌ Invalid model. Choose 'tfidf' or 'bert'.")
+            run_tfidf_pipeline(args, raw_job_text, raw_resume_texts)
 
-        # Display ranked resumes
-        print(f"\n🎯 Top {top_k} Candidate Matches for the Job ({args.model.upper()}):")
-        for i, (fname, score) in enumerate(ranked[:top_k], 1):
-            print(f"{i}. {fname}  → score: {score:.4f}")
-            
     except Exception as e:
         print(f"❌ Error: {str(e)}")
 
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Recruiter Pipeline: Rank user uploaded resumes for a given job description")
-
-    # Shared arguments
-    parser.add_argument('--job_desc_path', type=str, required=True, help="Path to job description file")
-    parser.add_argument('--resume_dir', type=str, required=True, help="Directory containing applicant resumes")
-    parser.add_argument('--model', type=str, choices=['tfidf', 'bert'], default='tfidf',
-                        help="Model to use: tfidf or bert")
-    parser.add_argument('--top_k', type=int, default=10, help="Number of top resumes to return")
-    parser.add_argument('--debug', action='store_true', help="Print cleaned/raw texts and raw similarity scores")
-
-    # TF-IDF specific
-    parser.add_argument('--vectorizer_path', type=str,
-                        default='models/tfidf/recruiter_tfidf/combined_tfidf_vectorizer.pkl',
-                        help="Path to pre-trained TF-IDF vectorizer")
-
-    # BERT specific
-    parser.add_argument('--bert_model_path', type=str,
-                        default='models/bert/dapt_minilm_sentence_transformer',
-                        help="Path to fine-tuned BERT/SBERT model")
+    parser = argparse.ArgumentParser(description="Recruiter Pipeline: Rank resumes for a given job description")
+
+    # Shared args
+    parser.add_argument("--job_desc_path", type=str, required=True, help="Path to job description file")
+    parser.add_argument("--resume_dir", type=str, required=True, help="Directory containing applicant resumes")
+    parser.add_argument("--model", type=str, choices=["tfidf", "bert"], default="tfidf")
+    parser.add_argument("--top_k", type=int, default=None,
+                        help="Number of top matches to return if not specified, returns all")
+    parser.add_argument("--debug", action="store_true",
+                        help="print raw similarity scores and cleaned texts for debugging")
+
+    # TF-IDF args
+    parser.add_argument("--local_vectorizer_path", type=str, default=None,
+                        help="Local TF-IDF vectorizer .pkl file")
+    parser.add_argument("--tfidf_repo_id", type=str, default="Om-Shandilya/resume-matcher-tfidf",
+                        help="Hub repo id for HuggingFace TF-IDF model")
+    parser.add_argument("--vectorizer_filename", type=str, default="recruiter/combined_vectorizer.pkl",
+                        help="Filename of vectorizer in the HF repo")
+
+    # BERT args
+    parser.add_argument("--local_bert_path", type=str, default=None,
+                        help="Local fine-tuned ST model path")
+    parser.add_argument("--bert_repo_id", type=str, default="Om-Shandilya/resume-matcher-bert",
+                        help="fine-tuned ST model's HF repo id")
 
     args = parser.parse_args()
     main(args)

src/feature_engg/bert_embedding_data.py

@@ -2,9 +2,13 @@ import os
 import numpy as np
 import pandas as pd
 import torch
+import faiss
+import torch
+from faiss import read_index
 from typing import Optional
 from sentence_transformers import SentenceTransformer, models
-import faiss
+from transformers import AutoTokenizer, AutoModel
+from huggingface_hub import hf_hub_download
 
 
 def get_bert_model(model_name: str = "all-MiniLM-L6-v2",
@@ -53,7 +57,7 @@ def save_bert_model(vectorizer: SentenceTransformer,
 
 def bert_embed_text(df: pd.DataFrame,
                    text_column: str,
-                   label: str,
+                   label: str = 'jobs',  # default is 'jobs' as most common use-case in pipeline.
                    model: Optional[SentenceTransformer] = None,
                    save_path: Optional[str] = None,
                    save_model_file: bool = False):
@@ -89,26 +93,41 @@ def bert_embed_text(df: pd.DataFrame,
     )
 
     if save_path and label:
-        save_bert_embeddings(embeddings, os.path.join(save_path, f"{label}_bert_embeddings.faiss"))
+        save_bert_embeddings(embeddings, os.path.join(save_path, f"{label}.faiss"))
         if save_model_file:
-            save_bert_model(model, os.path.join(save_path, f"{label}_bert_model"))
+            save_bert_model(model, save_path)
 
     return embeddings, model
 
 
-def load_bert_embeddings(path: str):
-    """
-    Load a FAISS index file (.faiss) from disk.
-    """
-    if not path.endswith('.faiss'):
-        path += '.faiss'
-    return faiss.read_index(path)
-
-
-def load_bert_model(path: str):
-    """Load a saved SentenceTransformer model."""
-
-    return SentenceTransformer(path)
+def load_faiss_index(local_index_path: str, repo_id: str, filename: str):
+    """Load FAISS index, preferring local then HF Hub."""
+    if local_index_path and os.path.exists(local_index_path):
+        print(f"📂 Loading local FAISS index from {local_index_path}")
+        return read_index(local_index_path)
+    else:
+        print(f"🌐 Downloading FAISS index from Hugging Face Hub ({repo_id})")
+        faiss_path = hf_hub_download(repo_id=repo_id, filename=filename)
+        return read_index(faiss_path)
+
+def load_bert_model(local_model_path: str, repo_id: str):
+    """Load BERT model, preferring local then HF Hub."""
+    if local_model_path and os.path.exists(local_model_path):
+        print(f"📂 Loading local BERT model from {local_model_path}")
+        tokenizer = AutoTokenizer.from_pretrained(local_model_path)
+        model = AutoModel.from_pretrained(local_model_path)
+    else:
+        print(f"🌐 Downloading BERT model from Hugging Face Hub ({repo_id})")
+        tokenizer = AutoTokenizer.from_pretrained(repo_id)
+        model = AutoModel.from_pretrained(repo_id)
+    return tokenizer, model
+
+def mean_pooling(model_output, attention_mask):
+    """Mean pooling for sentence embeddings."""
+    token_embeddings = model_output[0]  # First element is [batch, seq_len, hidden_dim]
+    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(
+        input_mask_expanded.sum(1), min=1e-9)
 
 
 def convert_hf_model_to_st(hf_model_path: str,

src/feature_engg/tfidf_vectorizing_data.py

@@ -4,6 +4,7 @@ import joblib
 import os
 from scipy.sparse import csr_matrix, save_npz, load_npz
 from typing import Optional, Tuple
+from huggingface_hub import hf_hub_download
 
 
 def get_tfidf_vectorizer(max_features: int = 5000, 
@@ -108,19 +109,23 @@ def tfidf_vectorize_text(df: pd.DataFrame,
     return X, vectorizer
 
 
-def load_vectorizer(path: str):
-    """
-    Loads a vectorizer from a .pkl file.
-    """
-    if not path.endswith('.pkl'):
-        path += '.pkl'
-    return joblib.load(path)
+def load_tfidf_vectorizer(local_vectorizer_path: str, repo_id: str, filename: str):
+    """Load TF-IDF vectorizer, preferring local then HF Hub."""
+    if local_vectorizer_path and os.path.exists(local_vectorizer_path):
+        print(f"📂 Loading local TF-IDF vectorizer from {local_vectorizer_path}")
+        return joblib.load(local_vectorizer_path)
+    else:
+        print(f"🌐 Downloading TF-IDF vectorizer from Hugging Face Hub ({repo_id})")
+        vec_path = hf_hub_download(repo_id=repo_id, filename=filename)
+        return joblib.load(vec_path)
 
 
-def load_vector_data(path: str):
-    """
-    Loads a sparse matrix from a .npz file.
-    """
-    if not path.endswith('.npz'):
-        path += '.npz'
-    return load_npz(path)
+def load_tfidf_matrix(local_matrix_path: str, repo_id: str, filename: str):
+    """Load TF-IDF matrix, preferring local then HF Hub."""
+    if local_matrix_path and os.path.exists(local_matrix_path):
+        print(f"📂 Loading local TF-IDF matrix from {local_matrix_path}")
+        return load_npz(local_matrix_path)
+    else:
+        print(f"🌐 Downloading TF-IDF matrix from Hugging Face Hub ({repo_id})")
+        mat_path = hf_hub_download(repo_id=repo_id, filename=filename)
+        return load_npz(mat_path)

src/matching/matching_engine.py

@@ -1,6 +1,6 @@
 import numpy as np
 from sklearn.metrics.pairwise import cosine_similarity
-from src.feature_engg.tfidf_vectorizing_data import load_vector_data
+from src.feature_engg.tfidf_vectorizing_data import load_tfidf_matrix
 
 def compute_similarity_matrix(X_resumes, X_jobs ):
     """
@@ -70,8 +70,8 @@ if __name__ == "__main__":
     job_vec_path = "models/dev_tfidf/jobs_tfidf_matrix.npz"
 
     # Load sparse TF-IDF matrices
-    X_resumes = load_vector_data(resume_vec_path)
-    X_jobs = load_vector_data(job_vec_path)
+    X_resumes = load_tfidf_matrix(resume_vec_path)
+    X_jobs = load_tfidf_matrix(job_vec_path)
 
     print(f"✅ Loaded resumes vector shape: {X_resumes.shape}")
     print(f"✅ Loaded job descriptions vector shape: {X_jobs.shape}")

src/processing/text_cleaning.py

@@ -157,4 +157,24 @@ def clean_column(
         )
     )
     print(f"✅ Cleaned column '{column_name}' and saved cleaned data to '{new_column_name}' column.")
-    return df
+    return df
+
+def clean_text_for_bert(text: str) -> str:
+    """
+    Minimal cleaning for BERT input.
+    Keeps stopwords, punctuation, and numbers because they carry meaning.
+    Only removes obvious noise.
+    """
+    if pd.isnull(text):
+        return ""
+
+    # Normalize unicode to standard form
+    text = unicodedata.normalize("NFKC", text)
+
+    # Remove HTML tags
+    text = re.sub(r"<.*?>", " ", text)
+
+    # Replace multiple spaces/newlines with a single space
+    text = re.sub(r"\s+", " ", text).strip()
+
+    return text