Add BiLSTM/LSTM track: phase-wise notebooks, figures, tables, models, thesis

.gitattributes

@@ -33,3 +33,81 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+FinalThesisSP.pdf filter=lfs diff=lfs merge=lfs -text
+figures/methodology_pipeline.png filter=lfs diff=lfs merge=lfs -text
+phase1/figures/accuracy_plot.png filter=lfs diff=lfs merge=lfs -text
+phase1/figures/loss_plot.png filter=lfs diff=lfs merge=lfs -text
+phase2/figures/roc_curve.png filter=lfs diff=lfs merge=lfs -text
+phase5/figures/english_f1_curve.png filter=lfs diff=lfs merge=lfs -text
+phase5/figures/english_pr_curve.png filter=lfs diff=lfs merge=lfs -text
+phase5/figures/english_roc_curve.png filter=lfs diff=lfs merge=lfs -text
+phase5/figures/english_train_accuracy.png filter=lfs diff=lfs merge=lfs -text
+phase5/figures/english_train_loss.png filter=lfs diff=lfs merge=lfs -text
+phase5/figures/full_dataset_confusion_matrix.png filter=lfs diff=lfs merge=lfs -text
+phase5/figures/full_dataset_f1_curve.png filter=lfs diff=lfs merge=lfs -text
+phase5/figures/full_dataset_pr_curve.png filter=lfs diff=lfs merge=lfs -text
+phase5/figures/full_dataset_roc_curve.png filter=lfs diff=lfs merge=lfs -text
+phase5/figures/full_dataset_train_accuracy.png filter=lfs diff=lfs merge=lfs -text
+phase5/figures/full_dataset_train_loss.png filter=lfs diff=lfs merge=lfs -text
+phase5/figures/hindi_roc_curve.png filter=lfs diff=lfs merge=lfs -text
+phase5/figures/hindi_train_accuracy.png filter=lfs diff=lfs merge=lfs -text
+phase5/figures/hindi_train_loss.png filter=lfs diff=lfs merge=lfs -text
+phase5/figures/hinglish_f1_curve.png filter=lfs diff=lfs merge=lfs -text
+phase5/figures/hinglish_pr_curve.png filter=lfs diff=lfs merge=lfs -text
+phase5/figures/hinglish_roc_curve.png filter=lfs diff=lfs merge=lfs -text
+phase5/figures/hinglish_train_accuracy.png filter=lfs diff=lfs merge=lfs -text
+phase5/figures/hinglish_train_loss.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/sentence_108_FALSE_NEGATIVE.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/sentence_146_FALSE_POSITIVE.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/sentence_150_FALSE_POSITIVE.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/sentence_216_FALSE_POSITIVE.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/sentence_312_FALSE_NEGATIVE.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/sentence_322_FALSE_NEGATIVE.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/sentence_32_FALSE_POSITIVE.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/sentence_341_FALSE_POSITIVE.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/sentence_351_FALSE_NEGATIVE.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/sentence_359_FALSE_NEGATIVE.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/sentence_408_FALSE_NEGATIVE.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/sentence_411_FALSE_NEGATIVE.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/sentence_425_FALSE_POSITIVE.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/sentence_435_FALSE_POSITIVE.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/sentence_447_FALSE_POSITIVE.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/sentence_461_FALSE_NEGATIVE.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/sentence_538_FALSE_POSITIVE.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/sentence_578_FALSE_POSITIVE.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/sentence_60_FALSE_NEGATIVE.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/sentence_89_FALSE_NEGATIVE.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_1.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_10.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_100.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_101.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_102.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_103.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_104.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_105.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_106.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_107.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_108.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_109.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_11.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_110.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_111.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_112.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_113.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_114.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_115.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_116.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_117.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_118.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_119.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_12.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_120.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_121.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_122.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_123.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_124.png filter=lfs diff=lfs merge=lfs -text
+phase5/shap/waterfall_sentence_125.png filter=lfs diff=lfs merge=lfs -text
+phase6/figures/(_english_,__hinglish_,__hindi_)_hinglish_loss.png filter=lfs diff=lfs merge=lfs -text
+phase6/figures/(_hindi_,__english_,__hinglish_)_hinglish_loss.png filter=lfs diff=lfs merge=lfs -text
+phase6/figures/(_hindi_,__hinglish_,__english_)_hindi_acc.png filter=lfs diff=lfs merge=lfs -text
+phase6/figures/(_hindi_,__hinglish_,__english_)_hindi_loss.png filter=lfs diff=lfs merge=lfs -text

FinalThesisSP.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5024b69730ac3e111aafbd14bd346c0324953041985b202166933c501e2588a3
+size 5425457

README.md

@@ -0,0 +1,77 @@
+---
+license: mit
+language:
+  - en
+  - hi
+tags:
+  - text-classification
+  - hate-speech-detection
+  - code-mixed
+  - hinglish
+  - bilstm
+  - lstm
+datasets:
+  - pankajbiswas6/prism-hinglish-hate-speech
+metrics:
+  - f1
+  - accuracy
+  - roc_auc
+---
+
+# Code-Mixed Hinglish Hate-Speech Classification (BiLSTM / LSTM Track)
+
+**Contributor:** Pankaj Biswas (222025043), B.Tech CSE, RSET, The Assam Royal Global University.
+Part of the group project *Developing a Sentiment Analysis Model for Code-Mixed Hindi-English
+(Hinglish) Text* (Guide: Dr. Dillip Rout).
+
+This repository holds the complete artifacts for the **embedding + recurrent-network track**:
+GloVe / Word2Vec / FastText embeddings with **BiLSTM** and **many-to-one LSTM**, trained on the
+**PRISM** Hinglish hate-speech dataset across regular, language-wise, and multi-stage regimes.
+Every phase below includes its notebooks, figures, result tables, and trained models.
+
+**Companion dataset repository:** [PRISM Hinglish Hate-Speech Dataset](https://huggingface.co/datasets/pankajbiswas6/prism-hinglish-hate-speech)
+
+## Pipeline
+
+![Methodology](figures/methodology_pipeline.png)
+
+## Best model
+
+| Model | Regime | Strategy | Accuracy | F1 | AUC-ROC |
+|-------|--------|----------|----------|------|---------|
+| **GloVe+BiLSTM** | multi-stage | combined | **0.8204** | **0.8041** | **0.9139** |
+
+Strongest model in this track; outperforms the transformer baselines in the same thesis table.
+
+## Model comparison (test set)
+
+| Model | Best regime / strategy | Accuracy | F1 | AUC-ROC |
+|-------|------------------------|----------|------|---------|
+| GloVe+BiLSTM | multi-stage / combined | 0.8204 | 0.8041 | 0.9139 |
+| FastText+BiLSTM | langwise / english | 0.7524 | 0.7688 | 0.8251 |
+| Word2Vec+BiLSTM | multi-stage / combined | 0.7305 | 0.6989 | 0.8091 |
+| GloVe+LSTM | regular | 0.6779 | 0.5644 | 0.7532 |
+| Word2Vec+LSTM | regular | 0.6675 | 0.5892 | 0.7294 |
+| FastText+LSTM | regular | 0.6610 | 0.5729 | 0.7208 |
+
+Full tables: [`tables/`](tables/) (model comparison, six-variation sweep, hyperparameters, literature).
+
+## Training phases
+
+Each phase folder contains its own README plus the notebooks, figures, tables, and models for that step.
+
+| Phase | Focus | Contents |
+|-------|-------|----------|
+| [Phase 1](phase1/README.md) | Dataset split and FastText baseline | notebooks, figures |
+| [Phase 2](phase2/README.md) | Hybrid baselines (monolingual) | notebooks, figures, tables |
+| [Phase 3](phase3/README.md) | Scaled BiLSTM and language-wise strategy | figures, tables, models |
+| [Phase 4](phase4/README.md) | Many-to-one LSTM and split-data hybrids | notebooks, figures, tables, models |
+| [Phase 5](phase5/README.md) | GloVe+BiLSTM fine-tune and SHAP explainability | figures, SHAP, tables, models |
+| [Phase 6](phase6/README.md) | Regular / sequential / multi-stage (six variations) | notebooks, figures, tables, models |
+
+## Full thesis
+
+This repository focuses on Pankaj Biswas's contribution. For the complete group thesis
+(all 17 models, including the transformer and LLM tracks built by teammates):
+
+**[Read the full thesis](THESIS.md)** (with downloadable PDF).

THESIS.md

@@ -0,0 +1,49 @@
+# Full Group Thesis
+
+**Developing a Sentiment Analysis Model for Code-Mixed Hindi-English (Hinglish) Text**
+
+B.Tech (Computer Science & Engineering), 8th Semester — Royal School of Engineering and
+Technology, The Assam Royal Global University, Guwahati. Session 2022–2026.
+Guide: Dr. Dillip Rout (Assistant Professor, CSE, RSET).
+
+**Authors:** Pulakala Prithvi Raj (222025042), Pankaj Biswas (222025043), Pritisha Goswami (222025049).
+
+**[Download the full thesis (PDF)](FinalThesisSP.pdf)**
+
+**Model repository (Pankaj Biswas's BiLSTM / LSTM track):** [code, figures, tables, models](README.md)
+**Dataset repository:** [PRISM Hinglish Hate-Speech Dataset](https://huggingface.co/datasets/pankajbiswas6/prism-hinglish-hate-speech)
+
+> The [model card](README.md) covers only Pankaj Biswas's BiLSTM / LSTM track.
+> This page describes the full group work; the transformer (MuRIL, mBART, HingRoBERTa,
+> MPNet) and Sarvam LLM tracks were contributed by the other team members.
+
+## Abstract
+
+Code-mixed languages such as Hinglish (an informal infusion of Hindi and English) pose
+significant challenges for sentiment analysis due to inconsistent grammar, transliteration
+variations, and limited annotated resources. This study presents a comprehensive comparison
+of classical machine learning, hybrid models, and deep learning architectures (including
+transformers) for binary classification of Hinglish text. The analysis uses the **PRISM**
+dataset of **29,550** Hinglish samples labeled non-hate (0) or hate (1), sourced from Kaggle.
+Preprocessing removed URLs, mentions, and hashtags and normalized whitespace. **Seventeen
+models** were implemented — Word2Vec, GloVe, FastText, USE, ELMo with both LSTM and LightGBM
+for regular training, and GloVe+BiLSTM, FastText+BiLSTM, Word2Vec+BiLSTM, MuRIL, mBART,
+HingRoBERTa-mixed, and MPNet for regular, language-wise, and multi-stage training (including
+six multi-stage variations on GloVe+BiLSTM and MuRIL). Metrics: Accuracy, Balanced Accuracy,
+Precision, Recall, Specificity, F1, and AUC-ROC. Multi-stage results show **MPNet** reaching
+the highest F1 (0.720) and AUC (0.820), and **MuRIL** the highest accuracy (0.724) in the
+Hinglish->Hindi->English->Full variation. Classical models performed worse, though GloVe+BiLSTM
+outperformed the Word2Vec and FastText baselines and, in the multi-stage combined strategy,
+reached F1 0.804 / AUC 0.914. The findings emphasize the value of multiple model evaluations
+for robust hate-speech classification of low-resource, code-mixed social media data.
+
+## Contents
+
+- Chapter 1 — Introduction
+- Chapter 2 — Literature Survey
+- Chapter 3 — Methodology (dataset, preprocessing, splitting, models, hyperparameters, metrics)
+- Chapter 4 — Results and Discussion (Tables 4.1–4.11)
+- Chapter 5 — Conclusion and Future Work
+- References
+
+For the complete document with all figures and tables, download the PDF above.

phase1/README.md

@@ -0,0 +1,22 @@
+# Phase 1 - Dataset Split and FastText Baseline
+
+[Back to model card](../README.md) | [Dataset repository](https://huggingface.co/datasets/pankajbiswas6/prism-hinglish-hate-speech)
+
+## Setup
+- Goal: establish a clean, reproducible data foundation and a first embedding baseline.
+- Dataset: PRISM (combined_hate_speech_dataset, Kaggle), 29,550 raw Hinglish samples.
+- Preprocessing: lowercase; remove URLs, mentions, hashtags; normalize elongated words and
+  whitespace; drop duplicates, giving 29,506 clean rows with 5 retained features.
+- Split: stratified 70/30 holdout; the 70 percent pool split 60/10 into train/val, with
+  language-wise class balance preserved across all subsets.
+- Embedding: FastText (subword).
+
+## Findings
+- Near-balanced classes (non-hate 53.5 percent / hate 46.5 percent) but skewed by language
+  (English 50.8, Hindi 33.0, Hinglish 16.2), motivating the later language-wise strategies.
+- Subword FastText embeddings handle Hinglish transliteration noise better than word-level
+  lookups, setting the baseline for the hybrid models.
+
+## Files
+- Notebooks: [`notebooks/`](notebooks/) - Dataset_Split, Fasttext
+- Figures: [`figures/`](figures/) - accuracy/loss curves, confusion matrix, ROC

phase1/notebooks/Dataset_Split.ipynb

@@ -0,0 +1 @@
+{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"gpuType":"T4","authorship_tag":"ABX9TyNBBY9t4rr3PgL3e1owA5LY"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"accelerator":"GPU"},"cells":[{"cell_type":"code","execution_count":1,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"7CyFwFUfV2VP","executionInfo":{"status":"ok","timestamp":1767686830634,"user_tz":-330,"elapsed":8722,"user":{"displayName":"Pankaj Biswas","userId":"08016076892025567798"}},"outputId":"4e6a3e32-58c4-4281-cac3-99f3145c1302"},"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting langdetect\n","  Downloading langdetect-1.0.9.tar.gz (981 kB)\n","\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/981.5 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m981.5/981.5 kB\u001b[0m \u001b[31m34.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n","Requirement already satisfied: six in /usr/local/lib/python3.12/dist-packages (from langdetect) (1.17.0)\n","Building wheels for collected packages: langdetect\n","  Building wheel for langdetect (setup.py) ... \u001b[?25l\u001b[?25hdone\n","  Created wheel for langdetect: filename=langdetect-1.0.9-py3-none-any.whl size=993223 sha256=a10e6373452363592857fae04074212717ca07e354f60345759978a16c2c07e4\n","  Stored in directory: /root/.cache/pip/wheels/c1/67/88/e844b5b022812e15a52e4eaa38a1e709e99f06f6639d7e3ba7\n","Successfully built langdetect\n","Installing collected packages: langdetect\n","Successfully installed langdetect-1.0.9\n"]}],"source":["!pip install langdetect\n"]},{"cell_type":"code","source":["import pandas as pd\n","import re\n","import os\n","from langdetect import detect\n"],"metadata":{"id":"e4Ah-kVLWDHB","executionInfo":{"status":"ok","timestamp":1767686842453,"user_tz":-330,"elapsed":311,"user":{"displayName":"Pankaj Biswas","userId":"08016076892025567798"}}},"execution_count":2,"outputs":[]},{"cell_type":"code","source":["from google.colab import files\n","uploaded = files.upload()\n"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":73},"id":"ADfV0f8JWKVM","executionInfo":{"status":"ok","timestamp":1767686924372,"user_tz":-330,"elapsed":66560,"user":{"displayName":"Pankaj Biswas","userId":"08016076892025567798"}},"outputId":"e4f59fad-e79a-4742-95c7-1de1d5a993c1"},"execution_count":3,"outputs":[{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["\n","     <input type=\"file\" id=\"files-d85913e3-eba1-4cb0-a259-16dbfa718fbc\" name=\"files[]\" multiple disabled\n","        style=\"border:none\" />\n","     <output id=\"result-d85913e3-eba1-4cb0-a259-16dbfa718fbc\">\n","      Upload widget is only available when the cell has been executed in the\n","      current browser session. Please rerun this cell to enable.\n","      </output>\n","      <script>// Copyright 2017 Google LLC\n","//\n","// Licensed under the Apache License, Version 2.0 (the \"License\");\n","// you may not use this file except in compliance with the License.\n","// You may obtain a copy of the License at\n","//\n","//      http://www.apache.org/licenses/LICENSE-2.0\n","//\n","// Unless required by applicable law or agreed to in writing, software\n","// distributed under the License is distributed on an \"AS IS\" BASIS,\n","// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n","// See the License for the specific language governing permissions and\n","// limitations under the License.\n","\n","/**\n"," * @fileoverview Helpers for google.colab Python module.\n"," */\n","(function(scope) {\n","function span(text, styleAttributes = {}) {\n","  const element = document.createElement('span');\n","  element.textContent = text;\n","  for (const key of Object.keys(styleAttributes)) {\n","    element.style[key] = styleAttributes[key];\n","  }\n","  return element;\n","}\n","\n","// Max number of bytes which will be uploaded at a time.\n","const MAX_PAYLOAD_SIZE = 100 * 1024;\n","\n","function _uploadFiles(inputId, outputId) {\n","  const steps = uploadFilesStep(inputId, outputId);\n","  const outputElement = document.getElementById(outputId);\n","  // Cache steps on the outputElement to make it available for the next call\n","  // to uploadFilesContinue from Python.\n","  outputElement.steps = steps;\n","\n","  return _uploadFilesContinue(outputId);\n","}\n","\n","// This is roughly an async generator (not supported in the browser yet),\n","// where there are multiple asynchronous steps and the Python side is going\n","// to poll for completion of each step.\n","// This uses a Promise to block the python side on completion of each step,\n","// then passes the result of the previous step as the input to the next step.\n","function _uploadFilesContinue(outputId) {\n","  const outputElement = document.getElementById(outputId);\n","  const steps = outputElement.steps;\n","\n","  const next = steps.next(outputElement.lastPromiseValue);\n","  return Promise.resolve(next.value.promise).then((value) => {\n","    // Cache the last promise value to make it available to the next\n","    // step of the generator.\n","    outputElement.lastPromiseValue = value;\n","    return next.value.response;\n","  });\n","}\n","\n","/**\n"," * Generator function which is called between each async step of the upload\n"," * process.\n"," * @param {string} inputId Element ID of the input file picker element.\n"," * @param {string} outputId Element ID of the output display.\n"," * @return {!Iterable<!Object>} Iterable of next steps.\n"," */\n","function* uploadFilesStep(inputId, outputId) {\n","  const inputElement = document.getElementById(inputId);\n","  inputElement.disabled = false;\n","\n","  const outputElement = document.getElementById(outputId);\n","  outputElement.innerHTML = '';\n","\n","  const pickedPromise = new Promise((resolve) => {\n","    inputElement.addEventListener('change', (e) => {\n","      resolve(e.target.files);\n","    });\n","  });\n","\n","  const cancel = document.createElement('button');\n","  inputElement.parentElement.appendChild(cancel);\n","  cancel.textContent = 'Cancel upload';\n","  const cancelPromise = new Promise((resolve) => {\n","    cancel.onclick = () => {\n","      resolve(null);\n","    };\n","  });\n","\n","  // Wait for the user to pick the files.\n","  const files = yield {\n","    promise: Promise.race([pickedPromise, cancelPromise]),\n","    response: {\n","      action: 'starting',\n","    }\n","  };\n","\n","  cancel.remove();\n","\n","  // Disable the input element since further picks are not allowed.\n","  inputElement.disabled = true;\n","\n","  if (!files) {\n","    return {\n","      response: {\n","        action: 'complete',\n","      }\n","    };\n","  }\n","\n","  for (const file of files) {\n","    const li = document.createElement('li');\n","    li.append(span(file.name, {fontWeight: 'bold'}));\n","    li.append(span(\n","        `(${file.type || 'n/a'}) - ${file.size} bytes, ` +\n","        `last modified: ${\n","            file.lastModifiedDate ? file.lastModifiedDate.toLocaleDateString() :\n","                                    'n/a'} - `));\n","    const percent = span('0% done');\n","    li.appendChild(percent);\n","\n","    outputElement.appendChild(li);\n","\n","    const fileDataPromise = new Promise((resolve) => {\n","      const reader = new FileReader();\n","      reader.onload = (e) => {\n","        resolve(e.target.result);\n","      };\n","      reader.readAsArrayBuffer(file);\n","    });\n","    // Wait for the data to be ready.\n","    let fileData = yield {\n","      promise: fileDataPromise,\n","      response: {\n","        action: 'continue',\n","      }\n","    };\n","\n","    // Use a chunked sending to avoid message size limits. See b/62115660.\n","    let position = 0;\n","    do {\n","      const length = Math.min(fileData.byteLength - position, MAX_PAYLOAD_SIZE);\n","      const chunk = new Uint8Array(fileData, position, length);\n","      position += length;\n","\n","      const base64 = btoa(String.fromCharCode.apply(null, chunk));\n","      yield {\n","        response: {\n","          action: 'append',\n","          file: file.name,\n","          data: base64,\n","        },\n","      };\n","\n","      let percentDone = fileData.byteLength === 0 ?\n","          100 :\n","          Math.round((position / fileData.byteLength) * 100);\n","      percent.textContent = `${percentDone}% done`;\n","\n","    } while (position < fileData.byteLength);\n","  }\n","\n","  // All done.\n","  yield {\n","    response: {\n","      action: 'complete',\n","    }\n","  };\n","}\n","\n","scope.google = scope.google || {};\n","scope.google.colab = scope.google.colab || {};\n","scope.google.colab._files = {\n","  _uploadFiles,\n","  _uploadFilesContinue,\n","};\n","})(self);\n","</script> "]},"metadata":{}},{"output_type":"stream","name":"stdout","text":["Saving bprism.csv to bprism.csv\n"]}]},{"cell_type":"code","source":["df = pd.read_csv(\"bprism.csv\")\n","\n","# Check first few rows\n","df.head()\n"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":345},"id":"F4x_CaCLWOIw","executionInfo":{"status":"ok","timestamp":1767686951012,"user_tz":-330,"elapsed":450,"user":{"displayName":"Pankaj Biswas","userId":"08016076892025567798"}},"outputId":"e277e517-a5ba-40a6-a76f-3e91facf549c"},"execution_count":4,"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                                text  hate_label  \\\n","0  Knowing ki Vikas kitna samjhata hai Priyanka a...           0   \n","1  I am Muhajir .. Aur mere lye sab se Pehly Paki...           0   \n","2  Doctor  sab sahi me ke PhD (in hate politics) ...           0   \n","3  Poore Desh me Patel OBC me aate Hain sirf gujr...           0   \n","4  Sarkar banne ke bad Hindu hit me ek bhi faisla...           1   \n","\n","            source  profanity_score  language dataset_version combined_date  \\\n","0  hate_speech_tsv                0  hinglish            v1.0    2025-09-11   \n","1  hate_speech_tsv                0  hinglish            v1.0    2025-09-11   \n","2  hate_speech_tsv                0  hinglish            v1.0    2025-09-11   \n","3  hate_speech_tsv                0  hinglish            v1.0    2025-09-11   \n","4  hate_speech_tsv                0  hinglish            v1.0    2025-09-11   \n","\n","   text_length  word_count  \n","0          126          25  \n","1          196          41  \n","2          166          29  \n","3          257          49  \n","4          140          25  "],"text/html":["\n","  <div id=\"df-fe982d51-8e77-4ee9-8b9a-81749b7b7078\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>text</th>\n","      <th>hate_label</th>\n","      <th>source</th>\n","      <th>profanity_score</th>\n","      <th>language</th>\n","      <th>dataset_version</th>\n","      <th>combined_date</th>\n","      <th>text_length</th>\n","      <th>word_count</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Knowing ki Vikas kitna samjhata hai Priyanka a...</td>\n","      <td>0</td>\n","      <td>hate_speech_tsv</td>\n","      <td>0</td>\n","      <td>hinglish</td>\n","      <td>v1.0</td>\n","      <td>2025-09-11</td>\n","      <td>126</td>\n","      <td>25</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>I am Muhajir .. Aur mere lye sab se Pehly Paki...</td>\n","      <td>0</td>\n","      <td>hate_speech_tsv</td>\n","      <td>0</td>\n","      <td>hinglish</td>\n","      <td>v1.0</td>\n","      <td>2025-09-11</td>\n","      <td>196</td>\n","      <td>41</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Doctor  sab sahi me ke PhD (in hate politics) ...</td>\n","      <td>0</td>\n","      <td>hate_speech_tsv</td>\n","      <td>0</td>\n","      <td>hinglish</td>\n","      <td>v1.0</td>\n","      <td>2025-09-11</td>\n","      <td>166</td>\n","      <td>29</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Poore Desh me Patel OBC me aate Hain sirf gujr...</td>\n","      <td>0</td>\n","      <td>hate_speech_tsv</td>\n","      <td>0</td>\n","      <td>hinglish</td>\n","      <td>v1.0</td>\n","      <td>2025-09-11</td>\n","      <td>257</td>\n","      <td>49</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Sarkar banne ke bad Hindu hit me ek bhi faisla...</td>\n","      <td>1</td>\n","      <td>hate_speech_tsv</td>\n","      <td>0</td>\n","      <td>hinglish</td>\n","      <td>v1.0</td>\n","      <td>2025-09-11</td>\n","      <td>140</td>\n","      <td>25</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-fe982d51-8e77-4ee9-8b9a-81749b7b7078')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-fe982d51-8e77-4ee9-8b9a-81749b7b7078 button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-fe982d51-8e77-4ee9-8b9a-81749b7b7078');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","    <div id=\"df-0a93dca7-5ab7-48e8-ae2b-3bb035050ab8\">\n","      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-0a93dca7-5ab7-48e8-ae2b-3bb035050ab8')\"\n","                title=\"Suggest charts\"\n","                style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","      </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","      <script>\n","        async function quickchart(key) {\n","          const quickchartButtonEl =\n","            document.querySelector('#' + key + ' button');\n","          quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","          quickchartButtonEl.classList.add('colab-df-spinner');\n","          try {\n","            const charts = await google.colab.kernel.invokeFunction(\n","                'suggestCharts', [key], {});\n","          } catch (error) {\n","            console.error('Error during call to suggestCharts:', error);\n","          }\n","          quickchartButtonEl.classList.remove('colab-df-spinner');\n","          quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","        }\n","        (() => {\n","          let quickchartButtonEl =\n","            document.querySelector('#df-0a93dca7-5ab7-48e8-ae2b-3bb035050ab8 button');\n","          quickchartButtonEl.style.display =\n","            google.colab.kernel.accessAllowed ? 'block' : 'none';\n","        })();\n","      </script>\n","    </div>\n","\n","    </div>\n","  </div>\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","variable_name":"df","summary":"{\n  \"name\": \"df\",\n  \"rows\": 29550,\n  \"fields\": [\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 29539,\n        \"samples\": [\n          \"silence betrays the innocent activism hunger no fur animal rights cousin eah wildlife vegan lgbt\",\n          \"no other asians being welcome at my engagement party\",\n          \"two aboard gone\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"hate_label\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 0,\n        \"max\": 1,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          1,\n          0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"source\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 4,\n        \"samples\": [\n          \"profanity_list\",\n          \"english_dataset\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"profanity_score\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 0,\n        \"max\": 10,\n        \"num_unique_values\": 11,\n        \"samples\": [\n          4,\n          0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"language\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 3,\n        \"samples\": [\n          \"hinglish\",\n          \"hindi\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"dataset_version\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"v1.0\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"combined_date\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"2025-09-11\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"text_length\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 189,\n        \"min\": 1,\n        \"max\": 1926,\n        \"num_unique_values\": 1191,\n        \"samples\": [\n          752\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"word_count\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 34,\n        \"min\": 1,\n        \"max\": 300,\n        \"num_unique_values\": 291,\n        \"samples\": [\n          128\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"}},"metadata":{},"execution_count":4}]},{"cell_type":"code","source":["# Text column name\n","TEXT_COLUMN = \"text\"\n","\n","# Hindi Unicode pattern (Devanagari)\n","hindi_pattern = re.compile(r'[\\u0900-\\u097F]')\n","\n","# Common Hinglish words\n","hinglish_words = {\n","    \"hai\", \"nahi\", \"kya\", \"kyu\", \"kyon\", \"bahut\", \"bohot\",\n","    \"acha\", \"accha\", \"theek\", \"yaar\", \"bhai\", \"kar\", \"karo\",\n","    \"ho\", \"hoga\", \"tha\", \"thi\", \"raha\", \"rahi\", \"rahe\",\n","    \"mat\", \"tum\", \"mera\", \"meri\", \"apna\", \"apni\"\n","}\n"],"metadata":{"id":"0u1e_fAFWkwU","executionInfo":{"status":"ok","timestamp":1767686967077,"user_tz":-330,"elapsed":18,"user":{"displayName":"Pankaj Biswas","userId":"08016076892025567798"}}},"execution_count":5,"outputs":[]},{"cell_type":"code","source":["def detect_language(text):\n","    text = str(text).strip().lower()\n","\n","    if not text:\n","        return \"Other\"\n","\n","    # Hindi detection\n","    if hindi_pattern.search(text):\n","        return \"Hindi\"\n","\n","    # English / Hinglish detection\n","    try:\n","        if detect(text) == \"en\":\n","            words = set(text.split())\n","            if words & hinglish_words:\n","                return \"Hinglish\"\n","            return \"English\"\n","    except:\n","        return \"Other\"\n","\n","    return \"Other\"\n"],"metadata":{"id":"4x4g4TjvWoyb","executionInfo":{"status":"ok","timestamp":1767686979559,"user_tz":-330,"elapsed":41,"user":{"displayName":"Pankaj Biswas","userId":"08016076892025567798"}}},"execution_count":6,"outputs":[]},{"cell_type":"code","source":["# Apply language detection\n","df[\"language\"] = df[TEXT_COLUMN].apply(detect_language)\n","\n","# Remove 'Other' rows\n","df = df[df[\"language\"].isin([\"English\", \"Hindi\", \"Hinglish\"])]\n","\n","# Check final distribution\n","df[\"language\"].value_counts()\n"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":209},"id":"eTQqIwsbWr1K","executionInfo":{"status":"ok","timestamp":1767687567726,"user_tz":-330,"elapsed":75105,"user":{"displayName":"Pankaj Biswas","userId":"08016076892025567798"}},"outputId":"abae93a0-adc9-4a31-ed59-2846b4f87bf0"},"execution_count":8,"outputs":[{"output_type":"execute_result","data":{"text/plain":["language\n","English     14178\n","Hindi        9767\n","Hinglish      379\n","Name: count, dtype: int64"],"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>count</th>\n","    </tr>\n","    <tr>\n","      <th>language</th>\n","      <th></th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>English</th>\n","      <td>14178</td>\n","    </tr>\n","    <tr>\n","      <th>Hindi</th>\n","      <td>9767</td>\n","    </tr>\n","    <tr>\n","      <th>Hinglish</th>\n","      <td>379</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div><br><label><b>dtype:</b> int64</label>"]},"metadata":{},"execution_count":8}]},{"cell_type":"code","source":["english_df = df[df[\"language\"] == \"English\"]\n","hindi_df = df[df[\"language\"] == \"Hindi\"]\n","hinglish_df = df[df[\"language\"] == \"Hinglish\"]\n"],"metadata":{"id":"O5NmAM5IWub7","executionInfo":{"status":"ok","timestamp":1767687660839,"user_tz":-330,"elapsed":51,"user":{"displayName":"Pankaj Biswas","userId":"08016076892025567798"}}},"execution_count":9,"outputs":[]},{"cell_type":"code","source":["from google.colab import drive\n","drive.mount('/content/drive')\n"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"bY4yPgNNZSMH","executionInfo":{"status":"ok","timestamp":1767687866939,"user_tz":-330,"elapsed":44855,"user":{"displayName":"Pankaj Biswas","userId":"08016076892025567798"}},"outputId":"9e482587-17a5-45d3-ab9e-6bd0333cf6ce"},"execution_count":10,"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/drive\n"]}]},{"cell_type":"code","source":["BASE_DIR = \"/content/drive/MyDrive/project_dataset_split\"\n","\n","english_dir = os.path.join(BASE_DIR, \"english\")\n","hindi_dir = os.path.join(BASE_DIR, \"hindi\")\n","hinglish_dir = os.path.join(BASE_DIR, \"hinglish\")\n","\n","os.makedirs(english_dir, exist_ok=True)\n","os.makedirs(hindi_dir, exist_ok=True)\n","os.makedirs(hinglish_dir, exist_ok=True)\n","\n","print(\"📁 Folder structure created in Google Drive\")\n"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"OMmdAAvkZ5hV","executionInfo":{"status":"ok","timestamp":1767687897963,"user_tz":-330,"elapsed":618,"user":{"displayName":"Pankaj Biswas","userId":"08016076892025567798"}},"outputId":"22c30246-38b9-425b-eebc-79b91a592398"},"execution_count":11,"outputs":[{"output_type":"stream","name":"stdout","text":["📁 Folder structure created in Google Drive\n"]}]},{"cell_type":"code","source":["english_df.to_csv(os.path.join(english_dir, \"english_dataset.csv\"), index=False)\n","hindi_df.to_csv(os.path.join(hindi_dir, \"hindi_dataset.csv\"), index=False)\n","hinglish_df.to_csv(os.path.join(hinglish_dir, \"hinglish_dataset.csv\"), index=False)\n","\n","print(\"✅ Datasets saved successfully in Google Drive\")\n"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"3ASmROpxaL5-","executionInfo":{"status":"ok","timestamp":1767687914862,"user_tz":-330,"elapsed":898,"user":{"displayName":"Pankaj Biswas","userId":"08016076892025567798"}},"outputId":"53c80beb-70b2-4791-bc55-a6548af28c34"},"execution_count":12,"outputs":[{"output_type":"stream","name":"stdout","text":["✅ Datasets saved successfully in Google Drive\n"]}]},{"cell_type":"code","source":["!ls /content/drive/MyDrive/project_dataset_split\n","!ls /content/drive/MyDrive/project_dataset_split/english\n","!ls /content/drive/MyDrive/project_dataset_split/hindi\n","!ls /content/drive/MyDrive/project_dataset_split/hinglish\n"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"cxVZRj7LaP_7","executionInfo":{"status":"ok","timestamp":1767687925087,"user_tz":-330,"elapsed":681,"user":{"displayName":"Pankaj Biswas","userId":"08016076892025567798"}},"outputId":"0e5b3134-7966-4360-aa52-9aa5e3923131"},"execution_count":13,"outputs":[{"output_type":"stream","name":"stdout","text":["english  hindi\thinglish\n","english_dataset.csv\n","hindi_dataset.csv\n","hinglish_dataset.csv\n"]}]},{"cell_type":"code","source":["!zip -r project_dataset_split.zip /content/drive/MyDrive/project_dataset_split\n"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"5KVf7RM9aSim","executionInfo":{"status":"ok","timestamp":1767688037838,"user_tz":-330,"elapsed":931,"user":{"displayName":"Pankaj Biswas","userId":"08016076892025567798"}},"outputId":"ccd0afa8-50fa-4bf9-a60c-83af07febffd"},"execution_count":14,"outputs":[{"output_type":"stream","name":"stdout","text":["  adding: content/drive/MyDrive/project_dataset_split/ (stored 0%)\n","  adding: content/drive/MyDrive/project_dataset_split/english/ (stored 0%)\n","  adding: content/drive/MyDrive/project_dataset_split/english/english_dataset.csv (deflated 67%)\n","  adding: content/drive/MyDrive/project_dataset_split/hindi/ (stored 0%)\n","  adding: content/drive/MyDrive/project_dataset_split/hindi/hindi_dataset.csv (deflated 78%)\n","  adding: content/drive/MyDrive/project_dataset_split/hinglish/ (stored 0%)\n","  adding: content/drive/MyDrive/project_dataset_split/hinglish/hinglish_dataset.csv (deflated 63%)\n"]}]},{"cell_type":"code","source":["from google.colab import files\n","files.download(\"project_dataset_split.zip\")\n"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":17},"id":"4B_w7G_dauAR","executionInfo":{"status":"ok","timestamp":1767688057136,"user_tz":-330,"elapsed":75,"user":{"displayName":"Pankaj Biswas","userId":"08016076892025567798"}},"outputId":"a5e5a767-8acf-477c-af86-87ece5f96ae0"},"execution_count":15,"outputs":[{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.Javascript object>"],"application/javascript":["\n","    async function download(id, filename, size) {\n","      if (!google.colab.kernel.accessAllowed) {\n","        return;\n","      }\n","      const div = document.createElement('div');\n","      const label = document.createElement('label');\n","      label.textContent = `Downloading \"${filename}\": `;\n","      div.appendChild(label);\n","      const progress = document.createElement('progress');\n","      progress.max = size;\n","      div.appendChild(progress);\n","      document.body.appendChild(div);\n","\n","      const buffers = [];\n","      let downloaded = 0;\n","\n","      const channel = await google.colab.kernel.comms.open(id);\n","      // Send a message to notify the kernel that we're ready.\n","      channel.send({})\n","\n","      for await (const message of channel.messages) {\n","        // Send a message to notify the kernel that we're ready.\n","        channel.send({})\n","        if (message.buffers) {\n","          for (const buffer of message.buffers) {\n","            buffers.push(buffer);\n","            downloaded += buffer.byteLength;\n","            progress.value = downloaded;\n","          }\n","        }\n","      }\n","      const blob = new Blob(buffers, {type: 'application/binary'});\n","      const a = document.createElement('a');\n","      a.href = window.URL.createObjectURL(blob);\n","      a.download = filename;\n","      div.appendChild(a);\n","      a.click();\n","      div.remove();\n","    }\n","  "]},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.Javascript object>"],"application/javascript":["download(\"download_46ae44dd-ab82-45a4-ae18-aa89026240e3\", \"project_dataset_split.zip\", 1888453)"]},"metadata":{}}]},{"cell_type":"code","source":[],"metadata":{"id":"0aHfajrMay7c"},"execution_count":null,"outputs":[]}]}

phase2/README.md

@@ -0,0 +1,21 @@
+# Phase 2 - Hybrid Baselines (Monolingual)
+
+[Back to model card](../README.md) | [Dataset repository](https://huggingface.co/datasets/pankajbiswas6/prism-hinglish-hate-speech)
+
+## Setup
+- Goal: build the first hybrid models (pretrained embeddings feeding a BiLSTM) and measure
+  them per language (monolingual).
+- Models: FastText+BiLSTM, GloVe+BiLSTM, Word2Vec+BiLSTM.
+- Config: embedding_dim 300, hidden_dim 256, dropout 0.5, max_seq_len 128, Adam lr 1e-3,
+  10 epochs, batch 32.
+
+## Findings
+- BiLSTM over pretrained embeddings clearly beats plain LSTM baselines on code-mixed data.
+- GloVe and FastText are competitive; Word2Vec trails slightly on the harder Hinglish subset.
+- Monolingual training transfers poorly across languages, motivating the language-wise
+  strategy in Phase 3.
+
+## Files
+- Notebooks: [`notebooks/`](notebooks/) - fasttext_bilstm_monolingual
+- Figures: [`figures/`](figures/) - confusion matrix, ROC, train/val accuracy and loss
+- Tables: [`tables/metrics_table.csv`](tables/metrics_table.csv)

phase2/tables/metrics_table.csv

@@ -0,0 +1,2 @@
+samples,accuracy,balanced_accuracy,precision,recall,specificity,f1,aucroc
+8852,0.6722774514234071,0.6559997291040732,0.7636047017849369,0.426556420233463,0.8854430379728155,0.5473552816352005,0.764029048457535

phase3/README.md

@@ -0,0 +1,27 @@
+# Phase 3 - Scaled BiLSTM and Language-Wise Strategy
+
+[Back to model card](../README.md) | [Dataset repository](https://huggingface.co/datasets/pankajbiswas6/prism-hinglish-hate-speech)
+
+## Setup
+- Goal: evaluate the hybrid BiLSTM models under a language-wise regime (train/evaluate per
+  strategy: English, Hindi, Hinglish, and combined/full).
+- Models: GloVe / Word2Vec / FastText + BiLSTM; larger GloVe variants tested (100d/200d/300d).
+
+## Results (full / combined strategy)
+
+| Model | Accuracy | F1 | AUC-ROC |
+|-------|----------|------|---------|
+| GloVe+BiLSTM | 0.683 | 0.636 | 0.764 |
+| Word2Vec+BiLSTM | 0.670 | 0.611 | 0.736 |
+| FastText+BiLSTM | 0.677 | 0.595 | 0.755 |
+
+## Findings
+- English strategy is easiest (AUC ~0.85 for GloVe+BiLSTM); Hindi is hardest.
+- GloVe+BiLSTM is the most consistent across strategies.
+- Per-language gaps confirm a single combined model underfits the minority languages,
+  motivating the multi-stage curriculum in Phase 6.
+
+## Files
+- Figures: [`figures/`](figures/) - per-language accuracy curves
+- Tables: [`tables/`](tables/) - strategy_results_table, master_metrics, final_results
+- Models: [`models/`](models/) - english_bilstm.h5, hindi_bilstm.h5, hinglish_bilstm.h5

phase3/models/english_bilstm.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:559d43379704aabeff4c4dc007d1c85b053e965b2771640ece2f0bb366a5124a
+size 12403256

phase3/models/hindi_bilstm.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2cc8f175edd8f316a0c271412c8913d47dee852e3684935a6f48ccdd19806fc
+size 2141224

phase3/models/hinglish_bilstm.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fee008d9fd7100aa5f0ce617f9723da4ae4dad8bf5d2396fab1d84f1f5dbd7bc
+size 6755656

phase3/tables/final_results.csv

@@ -0,0 +1,2 @@
+Accuracy,Balanced Accuracy,Precision,Recall,Specificity,F1,ROC AUC
+0.701536375960235,0.6987377686384606,0.6859817813765182,0.6592898832684825,0.7381856540084388,0.6723710317460317,0.7692907913855096

phase3/tables/master_metrics.csv

@@ -0,0 +1,5 @@
+accuracy,balanced_accuracy,precision,recall,specificity,f1,roc_auc,dataset
+0.553224155578301,0.49907663896583565,0.0,0.0,0.9981532770100153,0.0,0.4934352316830507,hindi
+0.7813333333333333,0.7813333333333333,0.7890410958904109,0.768,0.7946666661368889,0.7783783783783784,0.8626357777777778,english
+0.6729362591431557,0.636187064877409,0.6055363321799307,0.4679144385026738,0.8044596898722819,0.5279034690799397,0.6858907916823365,hinglish
+0.6746192893401015,0.662440974110619,0.7193169690501601,0.49107468123861564,0.8338072667191763,0.5836761203723749,0.7499585344026336,combined

phase3/tables/metrics.csv

@@ -0,0 +1,2 @@
+accuracy,balanced_accuracy,precision,recall,specificity,f1,roc_auc
+0.6746192893401015,0.662440974110619,0.7193169690501601,0.49107468123861564,0.8338072667191763,0.5836761203723749,0.7499585344026336

phase3/tables/strategy_results_table.csv

@@ -0,0 +1,5 @@
+Strategy,Best Epoch,Accuracy,Balanced Acc,Precision,Recall,Specificity,F1,ROC-AUC
+English,3,0.7577388963660835,0.7588410211778358,0.7891737891737892,0.7232375979112271,0.7944444444444444,0.7547683923705722,0.843271322889469
+Hindi,19,0.6206543967280164,0.6050973226141123,0.5652173913043478,0.4964200477326969,0.7137745974955277,0.5285895806861499,0.6246045401565189
+Hinglish,21,0.6625514403292181,0.6223758642363293,0.5714285714285714,0.4540540540540541,0.7906976744186046,0.5060240963855421,0.6602496183891533
+Full Dataset,39,0.6911432444645278,0.6865360619941223,0.6845206213176218,0.6215953307392996,0.7514767932489451,0.6515421871017079,0.748346483073109

phase4/README.md

@@ -0,0 +1,28 @@
+# Phase 4 - Many-to-One LSTM and Split-Data Hybrids
+
+[Back to model card](../README.md) | [Dataset repository](https://huggingface.co/datasets/pankajbiswas6/prism-hinglish-hate-speech)
+
+## Setup
+- Goal: add a many-to-one LSTM baseline across embeddings and re-run the BiLSTM hybrids on
+  consistent split data.
+- LSTM config: embedding_dim 100, hidden_dim 128, dropout 0.2, max_seq_len 100, Adam lr 1e-2,
+  30 epochs, batch 16. Embeddings: Word2Vec, GloVe, FastText.
+
+## Results (many-to-one LSTM, regular training)
+
+| Model | Accuracy | F1 | AUC-ROC |
+|-------|----------|------|---------|
+| GloVe+LSTM | 0.6779 | 0.5644 | 0.7532 |
+| Word2Vec+LSTM | 0.6675 | 0.5892 | 0.7294 |
+| FastText+LSTM | 0.6610 | 0.5729 | 0.7208 |
+
+## Findings
+- The plain many-to-one LSTM underperforms the BiLSTM hybrids by ~0.07 to 0.10 F1, confirming
+  the value of bidirectional context for code-mixed text.
+- High specificity but low recall: the LSTM variants lean toward the majority (non-hate) class.
+
+## Files
+- Notebooks: [`notebooks/`](notebooks/) - many_to_one_lstm, ManytooneLSTM
+- Figures: [`figures/`](figures/) - per-language accuracy curves
+- Tables: [`tables/`](tables/) - master_metrics, metrics
+- Models: [`models/`](models/) - combined/english/hindi/hinglish BiLSTM

phase4/models/combined_bilstm.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1308144cf0acc3095d5010fee15db0f3cf6780daeed76f391b88f914adcddd1c
+size 13053656

phase4/models/english_bilstm.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:709ac0504887cd2d6271aed48757d50c0f696797897bd8670ace9a5ded4d46df
+size 12403656

phase4/models/hindi_bilstm.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37f6a48143bc1c6a10b1c6334aa62bb4be675947de0bf1b0fb97e854befe0044
+size 2141624

phase4/models/hinglish_bilstm.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f528aa6670b1139a7ca71fdc1d5758a8c2c264f1bcce9ba62e8f134b897f56a
+size 6756056

phase4/tables/master_metrics.csv

@@ -0,0 +1,5 @@
+accuracy,balanced_accuracy,precision,recall,specificity,f1,roc_auc,dataset
+0.6187308085977482,0.5973896763783892,0.6101398601398601,0.4006888633754305,0.7940904886481158,0.4837144837144837,0.6283938288527531,hindi
+0.7663333333333333,0.7663333333333333,0.7310584152689416,0.8426666666666667,0.6899999995399999,0.7829049241251161,0.8487511111111111,english
+0.7021943573667712,0.6295277973968318,0.8345864661654135,0.2967914438502674,0.9622641492928574,0.4378698224852071,0.7216453710752974,hinglish
+0.6974619289340102,0.6962163002097739,0.6728060671722643,0.6786885245901639,0.7137440756038723,0.675734494015234,0.7694864423898689,combined

phase4/tables/metrics.csv

@@ -0,0 +1,2 @@
+accuracy,balanced_accuracy,precision,recall,specificity,f1,roc_auc
+0.6974619289340102,0.6962163002097739,0.6728060671722643,0.6786885245901639,0.7137440756038723,0.675734494015234,0.7694864423898689