{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 🛡️ Malicious URL HyperModel\n",
    "\n",
    "**Dataset:** [Malicious URLs](https://www.kaggle.com/datasets/sid321axn/malicious-urls-dataset) — 651K URLs · 4 classes (benign / phishing / defacement / malware)\n",
    "\n",
    "**Objective:** Build a GPU-accelerated stacking HyperModel that outperforms literature baselines:\n",
    "\n",
    "| Baseline | Accuracy | Source |\n",
    "|---|---|---|\n",
    "| RF on URL features | 97.98% | Sahingoz 2019 |\n",
    "| URLNet (Char+Word CNN) | ~98.5% | Le 2018 |\n",
    "| Stacking CNN+LSTM+GRU | 97.5% | Nguyen 2021 |\n",
    "| CharCNN | 96.75% | Aljofey 2022 |\n",
    "\n",
    "**Architecture:**\n",
    "```\n",
    "Raw URLs (651K)\n",
    "    ├── Feature Engineering (54 lexical features)\n",
    "    │       ├── XGBoost  (GPU)\n",
    "    │       └── LightGBM (CPU)\n",
    "    ├── TF-IDF char n-grams → Logistic Regression\n",
    "    └── Character-level Deep Learning (GPU)\n",
    "            ├── CharCNN\n",
    "            ├── CharBiLSTM\n",
    "            └── CNN-BiLSTM Hybrid\n",
    "    │\n",
    "    ▼\n",
    "Stacking Meta-Learner (XGBoost GPU)  →  Final Prediction\n",
    "```\n",
    "\n",
    "See [LITERATURE_REVIEW.md](LITERATURE_REVIEW.md) for a full survey of 15+ research papers."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 0. Install & Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ─── Install Dependencies ─────────────────────────────────────────────────────────────\n",
    "import subprocess, sys\n",
    "\n",
    "def pip_install(pkg):\n",
    "    subprocess.check_call([sys.executable, '-m', 'pip', 'install', pkg, '-q'])\n",
    "\n",
    "for pkg in [\n",
    "    'xgboost>=2.0', 'lightgbm>=4.0',\n",
    "    'optuna', 'kagglehub',\n",
    "    'tldextract', 'imbalanced-learn',\n",
    "    'seaborn', 'plotly', 'pyarrow',\n",
    "]:\n",
    "    try:\n",
    "        pip_install(pkg)\n",
    "    except Exception as e:\n",
    "        print(f'Warning: {pkg}: {e}')\n",
    "\n",
    "# Verify PyTorch CUDA\n",
    "import torch\n",
    "print(f'PyTorch {torch.__version__} | CUDA available: {torch.cuda.is_available()}')\n",
    "if torch.cuda.is_available():\n",
    "    print(f'  GPU: {torch.cuda.get_device_name(0)}')\n",
    "    print(f'  GPUs: {torch.cuda.device_count()}')\n",
    "print('\\n\\u2705 Packages ready')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys, os, pathlib, warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "PROJECT_ROOT = pathlib.Path('.').resolve()\n",
    "sys.path.insert(0, str(PROJECT_ROOT))\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "sns.set_theme(style='whitegrid', palette='muted')\n",
    "pd.set_option('display.max_columns', 30)\n",
    "pd.set_option('display.float_format', '{:.4f}'.format)\n",
    "\n",
    "SEED = 42\n",
    "np.random.seed(SEED)\n",
    "\n",
    "DATA_DIR   = PROJECT_ROOT / 'data'\n",
    "OUTPUT_DIR = PROJECT_ROOT / 'outputs'\n",
    "DATA_DIR.mkdir(parents=True, exist_ok=True)\n",
    "OUTPUT_DIR.mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "print(f'Project root : {PROJECT_ROOT}')\n",
    "print(f'Data dir     : {DATA_DIR}')\n",
    "print(f'Output dir   : {OUTPUT_DIR}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Download Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from download_data import download_malicious_urls\n",
    "\n",
    "csv_path = download_malicious_urls(DATA_DIR)\n",
    "print(f'\\nDataset path: {csv_path}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Exploratory Data Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_raw = pd.read_csv(csv_path)\n",
    "\n",
    "print(f'Shape   : {df_raw.shape}')\n",
    "print(f'Columns : {list(df_raw.columns)}')\n",
    "print(f'\\nClass distribution:')\n",
    "print(df_raw['type'].value_counts())\n",
    "df_raw.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ─── Class distribution plot ─────────────────────────────────────────────────\n",
    "fig, axes = plt.subplots(1, 2, figsize=(14, 4))\n",
    "\n",
    "# 4-class\n",
    "vc = df_raw['type'].value_counts()\n",
    "colors4 = ['#4CAF50', '#F44336', '#FF9800', '#9C27B0']\n",
    "axes[0].bar(vc.index.astype(str), vc.values, color=colors4)\n",
    "axes[0].set_title('4-Class Distribution')\n",
    "axes[0].set_ylabel('Count')\n",
    "for i, v in enumerate(vc.values):\n",
    "    axes[0].text(i, v + vc.max()*0.01, f'{v:,}', ha='center', fontsize=9)\n",
    "\n",
    "# Binary (benign vs malicious)\n",
    "binary_counts = pd.Series({\n",
    "    'benign': (df_raw['type'] == 'benign').sum(),\n",
    "    'malicious': (df_raw['type'] != 'benign').sum(),\n",
    "})\n",
    "axes[1].bar(binary_counts.index, binary_counts.values, color=['#4CAF50', '#F44336'])\n",
    "axes[1].set_title('Binary Distribution (benign vs malicious)')\n",
    "axes[1].set_ylabel('Count')\n",
    "for i, v in enumerate(binary_counts.values):\n",
    "    axes[1].text(i, v + binary_counts.max()*0.01, f'{v:,}', ha='center', fontsize=9)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig(OUTPUT_DIR / 'eda_class_distribution.png', dpi=150)\n",
    "plt.show()\n",
    "\n",
    "print(f'4-class imbalance ratio: {vc.min()/vc.max():.2%}')\n",
    "print(f'Binary  imbalance ratio: {binary_counts.min()/binary_counts.max():.2%}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ─── URL length distribution by class ───────────────────────────────────────\n",
    "df_raw['url_len'] = df_raw['url'].astype(str).str.len()\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(10, 4))\n",
    "for cls, color in zip(['benign','phishing','defacement','malware'], colors4):\n",
    "    data = df_raw.loc[df_raw['type']==cls, 'url_len'].clip(0, 500)\n",
    "    ax.hist(data, bins=60, alpha=0.5, color=color, label=cls, density=True)\n",
    "ax.set_xlabel('URL Length (chars)')\n",
    "ax.set_title('URL Length Distribution by Class')\n",
    "ax.legend()\n",
    "plt.tight_layout()\n",
    "plt.savefig(OUTPUT_DIR / 'eda_url_length_by_class.png', dpi=150)\n",
    "plt.show()\n",
    "\n",
    "print(df_raw.groupby('type')['url_len'].describe()[['mean','50%','max']].round(1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Feature Engineering\n",
    "\n",
    "We extract **54 hand-crafted features** across 8 categories:\n",
    "- Length & structural (7)\n",
    "- Character counts (14)\n",
    "- Character ratios (5)\n",
    "- Entropy / randomness (2)\n",
    "- Boolean indicators (10)\n",
    "- Domain / TLD signals (6)\n",
    "- Suspicious patterns (5)\n",
    "- Path & query analysis (5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from feature_engineering import URLFeatureExtractor, featurize_dataset\n",
    "\n",
    "print('Extracting features from 651K URLs …')\n",
    "df = featurize_dataset(str(csv_path), url_col='url', label_col='type')\n",
    "print(f'\\n\\u2705 {df.shape[0]:,} rows \\u00d7 {df.shape[1]} cols')\n",
    "df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feat_names = URLFeatureExtractor.feature_names()\n",
    "print(f'Total features: {len(feat_names)}')\n",
    "print(feat_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ─── Top features correlated with malicious label (binary) ──────────────\n",
    "df['is_malicious'] = (df['label'] != 'benign').astype(int)\n",
    "\n",
    "feat_corr = df[feat_names].corrwith(df['is_malicious']).abs().sort_values(ascending=False)\n",
    "print('Top 15 features correlated with maliciousness:')\n",
    "print(feat_corr.head(15).to_string())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ─── Feature distributions: benign vs malicious ─────────────────────────\n",
    "top_feats = feat_corr.head(8).index.tolist()\n",
    "fig, axes = plt.subplots(2, 4, figsize=(16, 7))\n",
    "axes = axes.flatten()\n",
    "\n",
    "for ax, feat in zip(axes, top_feats):\n",
    "    for lbl, color, name in [(0, '#4CAF50', 'Benign'), (1, '#F44336', 'Malicious')]:\n",
    "        data = df.loc[df['is_malicious']==lbl, feat].clip(\n",
    "            df[feat].quantile(0.01), df[feat].quantile(0.99)\n",
    "        )\n",
    "        ax.hist(data, bins=40, alpha=0.6, color=color, label=name, density=True)\n",
    "    ax.set_title(feat, fontsize=9)\n",
    "    ax.legend(fontsize=7)\n",
    "\n",
    "plt.suptitle('Feature Distributions: Benign vs Malicious', y=1.02)\n",
    "plt.tight_layout()\n",
    "plt.savefig(OUTPUT_DIR / 'eda_feature_distributions.png', dpi=150, bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Data Preparation\n",
    "\n",
    "We prepare two tasks:\n",
    "- **Binary**: benign (0) vs malicious (1)\n",
    "- **Multi-class**: benign (0) / phishing (1) / defacement (2) / malware (3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler, LabelEncoder\n",
    "\n",
    "# ─── Encode labels ─────────────────────────────────────────────────────\n",
    "label_map_4 = {'benign': 0, 'phishing': 1, 'defacement': 2, 'malware': 3}\n",
    "df['label_4'] = df['label'].map(label_map_4)\n",
    "df['label_2'] = (df['label'] != 'benign').astype(int)  # binary\n",
    "\n",
    "CLASS_NAMES_4 = ['benign', 'phishing', 'defacement', 'malware']\n",
    "CLASS_NAMES_2 = ['Benign', 'Malicious']\n",
    "\n",
    "X = df[feat_names].values.astype(np.float32)\n",
    "y_4 = df['label_4'].values\n",
    "y_2 = df['label_2'].values\n",
    "urls = df['url'].values\n",
    "\n",
    "# ─── Binary split ──────────────────────────────────────────────────────\n",
    "X_train, X_test, y2_train, y2_test, y4_train, y4_test, u_train, u_test = \\\n",
    "    train_test_split(X, y_2, y_4, urls, test_size=0.20, stratify=y_4, random_state=SEED)\n",
    "\n",
    "print(f'Train: {len(y2_train):,} | Test: {len(y2_test):,}')\n",
    "print(f'Binary train dist  : {dict(zip(*np.unique(y2_train, return_counts=True)))}')\n",
    "print(f'4-class train dist : {dict(zip(*np.unique(y4_train, return_counts=True)))}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ─── Feature scaling ───────────────────────────────────────────────────────\n",
    "scaler = StandardScaler()\n",
    "X_train_sc = scaler.fit_transform(X_train).astype(np.float32)\n",
    "X_test_sc  = scaler.transform(X_test).astype(np.float32)\n",
    "\n",
    "print(f'\\u2705 Scaled (dtype={X_train_sc.dtype}, shape={X_train_sc.shape})')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Classical ML Models (GPU where possible)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from models import build_xgb, build_lgbm\n",
    "from evaluation import evaluate_model, print_results_banner\n",
    "\n",
    "all_results_bin = []   # binary results\n",
    "all_results_mc  = []   # 4-class results\n",
    "\n",
    "# ─── Binary Classification ──────────────────────────────────────────────\n",
    "print('\\n' + '='*55)\n",
    "print('  BINARY: Benign vs Malicious')\n",
    "print('='*55)\n",
    "\n",
    "clf_models_bin = {\n",
    "    'XGBoost (GPU)':  build_xgb(num_classes=2),\n",
    "    'LightGBM (CPU)': build_lgbm(num_classes=2),\n",
    "}\n",
    "\n",
    "for name, model in clf_models_bin.items():\n",
    "    print(f'\\nTraining {name} …')\n",
    "    model.fit(X_train_sc, y2_train)\n",
    "    preds = model.predict(X_test_sc)\n",
    "    proba = model.predict_proba(X_test_sc)\n",
    "    result = evaluate_model(name, y2_test, preds, proba,\n",
    "                            class_names=CLASS_NAMES_2, task='binary')\n",
    "    all_results_bin.append(result)\n",
    "    print_results_banner(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ─── 4-Class Classification ─────────────────────────────────────────────\n",
    "print('\\n' + '='*55)\n",
    "print('  MULTI-CLASS: benign / phishing / defacement / malware')\n",
    "print('='*55)\n",
    "\n",
    "clf_models_mc = {\n",
    "    'XGBoost (GPU, 4c)':  build_xgb(num_classes=4),\n",
    "    'LightGBM (CPU, 4c)': build_lgbm(num_classes=4),\n",
    "}\n",
    "\n",
    "for name, model in clf_models_mc.items():\n",
    "    print(f'\\nTraining {name} …')\n",
    "    model.fit(X_train_sc, y4_train)\n",
    "    preds = model.predict(X_test_sc)\n",
    "    proba = model.predict_proba(X_test_sc)\n",
    "    result = evaluate_model(name, y4_test, preds, proba,\n",
    "                            class_names=CLASS_NAMES_4, task='multiclass')\n",
    "    all_results_mc.append(result)\n",
    "    print_results_banner(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. TF-IDF String Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from models import build_tfidf_lr_pipeline\n",
    "\n",
    "# Binary\n",
    "print('Training TF-IDF + LR (binary) …')\n",
    "tfidf_bin = build_tfidf_lr_pipeline(num_classes=2)\n",
    "tfidf_bin.fit(u_train, y2_train)\n",
    "preds = tfidf_bin.predict(u_test)\n",
    "proba = tfidf_bin.predict_proba(u_test)\n",
    "result = evaluate_model('TF-IDF + LR', y2_test, preds, proba,\n",
    "                        class_names=CLASS_NAMES_2, task='binary')\n",
    "all_results_bin.append(result)\n",
    "print_results_banner(result)\n",
    "\n",
    "# 4-class\n",
    "print('\\nTraining TF-IDF + LR (4-class) …')\n",
    "tfidf_mc = build_tfidf_lr_pipeline(num_classes=4)\n",
    "tfidf_mc.fit(u_train, y4_train)\n",
    "preds = tfidf_mc.predict(u_test)\n",
    "proba = tfidf_mc.predict_proba(u_test)\n",
    "result = evaluate_model('TF-IDF + LR (4c)', y4_test, preds, proba,\n",
    "                        class_names=CLASS_NAMES_4, task='multiclass')\n",
    "all_results_mc.append(result)\n",
    "print_results_banner(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7. Deep Learning Models (CharCNN · BiLSTM · Hybrid) — GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from models import DeepURLClassifier\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "print(f'DL device: {device}')\n",
    "\n",
    "dl_configs = [\n",
    "    dict(name='CharCNN',    arch='cnn',       epochs=20, batch_size=2048, lr=1e-3),\n",
    "    dict(name='CharBiLSTM', arch='bilstm',    epochs=15, batch_size=2048, lr=5e-4),\n",
    "    dict(name='CNN-BiLSTM', arch='cnn_bilstm',epochs=15, batch_size=2048, lr=7e-4),\n",
    "]\n",
    "\n",
    "# ─── Binary ───────────────────────────────────────────────────────────────\n",
    "print('\\n\\u2500\\u2500 Binary: Deep Learning Models \\u2500\\u2500')\n",
    "for cfg in dl_configs:\n",
    "    print(f\"\\nTraining {cfg['name']} (binary) …\")\n",
    "    model = DeepURLClassifier(\n",
    "        arch=cfg['arch'], num_classes=2,\n",
    "        epochs=cfg['epochs'], batch_size=cfg['batch_size'],\n",
    "        lr=cfg['lr'], device='auto',\n",
    "    )\n",
    "    model.fit(list(u_train), y2_train, class_weight='balanced')\n",
    "    preds = model.predict(list(u_test))\n",
    "    proba = model.predict_proba(list(u_test))\n",
    "    result = evaluate_model(cfg['name'], y2_test, preds, proba,\n",
    "                            class_names=CLASS_NAMES_2, task='binary')\n",
    "    all_results_bin.append(result)\n",
    "    print_results_banner(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ─── 4-Class ──────────────────────────────────────────────────────────────\n",
    "print('\\n\\u2500\\u2500 4-Class: Deep Learning Models \\u2500\\u2500')\n",
    "for cfg in dl_configs:\n",
    "    print(f\"\\nTraining {cfg['name']} (4-class) …\")\n",
    "    model = DeepURLClassifier(\n",
    "        arch=cfg['arch'], num_classes=4,\n",
    "        epochs=cfg['epochs'], batch_size=cfg['batch_size'],\n",
    "        lr=cfg['lr'], device='auto',\n",
    "    )\n",
    "    model.fit(list(u_train), y4_train, class_weight='balanced')\n",
    "    preds = model.predict(list(u_test))\n",
    "    proba = model.predict_proba(list(u_test))\n",
    "    result = evaluate_model(cfg['name']+' (4c)', y4_test, preds, proba,\n",
    "                            class_names=CLASS_NAMES_4, task='multiclass')\n",
    "    all_results_mc.append(result)\n",
    "    print_results_banner(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 8. Optuna Hyperparameter Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from hypertuning import tune_xgb, tune_lgbm\n",
    "\n",
    "print('Tuning XGBoost (GPU, binary) …')\n",
    "best_xgb_params = tune_xgb(\n",
    "    X_train_sc, y2_train,\n",
    "    num_classes=2, n_trials=60, timeout=300,\n",
    ")\n",
    "\n",
    "print('\\nTuning LightGBM (CPU, binary) …')\n",
    "best_lgbm_params = tune_lgbm(\n",
    "    X_train_sc, y2_train,\n",
    "    num_classes=2, n_trials=60, timeout=300,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xgb_lib\n",
    "import lightgbm as lgb_lib\n",
    "\n",
    "# ── Tuned XGBoost (GPU) ─────────────────────────────────────────────────\n",
    "_xgb_dev = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "\n",
    "tuned_xgb = xgb_lib.XGBClassifier(\n",
    "    **best_xgb_params,\n",
    "    objective='binary:logistic', tree_method='hist',\n",
    "    device=_xgb_dev, eval_metric='logloss',\n",
    "    n_jobs=-1, random_state=42,\n",
    ")\n",
    "tuned_xgb.fit(X_train_sc, y2_train)\n",
    "preds = tuned_xgb.predict(X_test_sc)\n",
    "proba = tuned_xgb.predict_proba(X_test_sc)\n",
    "result = evaluate_model('XGBoost (Optuna-tuned)', y2_test, preds, proba,\n",
    "                        class_names=CLASS_NAMES_2, task='binary')\n",
    "all_results_bin.append(result)\n",
    "print_results_banner(result)\n",
    "\n",
    "# ── Tuned LightGBM (CPU) ────────────────────────────────────────────────\n",
    "tuned_lgbm = lgb_lib.LGBMClassifier(\n",
    "    **best_lgbm_params,\n",
    "    objective='binary', class_weight='balanced',\n",
    "    device_type='cpu', n_jobs=-1, random_state=42, verbose=-1,\n",
    ")\n",
    "tuned_lgbm.fit(X_train_sc, y2_train)\n",
    "preds = tuned_lgbm.predict(X_test_sc)\n",
    "proba = tuned_lgbm.predict_proba(X_test_sc)\n",
    "result = evaluate_model('LightGBM (Optuna-tuned)', y2_test, preds, proba,\n",
    "                        class_names=CLASS_NAMES_2, task='binary')\n",
    "all_results_bin.append(result)\n",
    "print_results_banner(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 9. HyperModel — Stacking Ensemble\n",
    "\n",
    "Combines feature-based models (XGBoost GPU, LightGBM CPU) and character-level DL\n",
    "(CharCNN, BiLSTM) via out-of-fold (OOF) stacking with an XGBoost GPU meta-learner."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from models import build_stacking_hypermodel, ManualStackingEnsemble\n",
    "\n",
    "# ── Binary HyperModel ───────────────────────────────────────────────────\n",
    "arch = build_stacking_hypermodel(num_classes=2, use_deep=True, cv=5)\n",
    "\n",
    "hypermodel = ManualStackingEnsemble(\n",
    "    level0      = arch['level0'],\n",
    "    meta        = arch['meta'],\n",
    "    cv          = 5,\n",
    "    num_classes = 2,\n",
    ")\n",
    "\n",
    "print('Training HyperModel (binary) …')\n",
    "hypermodel.fit(\n",
    "    X_feat = X_train_sc,\n",
    "    X_url  = u_train,\n",
    "    y      = y2_train,\n",
    ")\n",
    "print('\\n\\u2705 HyperModel trained')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hyper_preds = hypermodel.predict(X_test_sc, u_test)\n",
    "hyper_proba = hypermodel.predict_proba(X_test_sc, u_test)\n",
    "\n",
    "result_hyper = evaluate_model(\n",
    "    'HyperModel (Stacking Ensemble)',\n",
    "    y2_test, hyper_preds, hyper_proba,\n",
    "    class_names=CLASS_NAMES_2, task='binary',\n",
    ")\n",
    "all_results_bin.append(result_hyper)\n",
    "print_results_banner(result_hyper)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 10. Results Comparison vs Literature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from evaluation import build_comparison_table, plot_accuracy_bar\n",
    "\n",
    "print('\\n\\ud83d\\udcca Binary Classification \\u2014 Final Comparison')\n",
    "comparison_bin = build_comparison_table(all_results_bin, task='binary')\n",
    "print(comparison_bin[['Method','Accuracy','F1 (macro)','AUC-ROC','Source']].to_string(index=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('\\n\\ud83d\\udcca 4-Class Classification \\u2014 Final Comparison')\n",
    "comparison_mc = build_comparison_table(all_results_mc, task='multiclass')\n",
    "print(comparison_mc[['Method','Accuracy','F1 (macro)','AUC-ROC','Source']].to_string(index=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ─── Accuracy bar chart ───────────────────────────────────────────────────\n",
    "fig_bar = plot_accuracy_bar(\n",
    "    comparison_bin,\n",
    "    save_path=str(OUTPUT_DIR / 'results_accuracy_bar.png'),\n",
    ")\n",
    "plt.show()\n",
    "\n",
    "# ─── ROC curves (binary) ─────────────────────────────────────────────────\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "\n",
    "fig_roc, ax_roc = plt.subplots(figsize=(9, 7))\n",
    "ax_roc.plot([0,1],[0,1], 'k--', lw=1, label='Random')\n",
    "\n",
    "roc_models = {\n",
    "    'XGBoost (GPU)':        ('feat', clf_models_bin['XGBoost (GPU)']),\n",
    "    'LightGBM (CPU)':       ('feat', clf_models_bin['LightGBM (CPU)']),\n",
    "    'TF-IDF + LR':          ('url',  tfidf_bin),\n",
    "    'XGBoost (tuned)':      ('feat', tuned_xgb),\n",
    "    'LightGBM (tuned)':     ('feat', tuned_lgbm),\n",
    "}\n",
    "\n",
    "for name, (src, mdl) in roc_models.items():\n",
    "    try:\n",
    "        X_in = u_test if src == 'url' else X_test_sc\n",
    "        prob = mdl.predict_proba(X_in)[:, 1]\n",
    "        fpr, tpr, _ = roc_curve(y2_test, prob)\n",
    "        roc_auc = auc(fpr, tpr)\n",
    "        ax_roc.plot(fpr, tpr, lw=1.5, label=f'{name} (AUC={roc_auc:.3f})')\n",
    "    except Exception as e:\n",
    "        print(f'  Skip {name}: {e}')\n",
    "\n",
    "# HyperModel\n",
    "try:\n",
    "    fpr, tpr, _ = roc_curve(y2_test, hyper_proba[:, 1])\n",
    "    roc_auc = auc(fpr, tpr)\n",
    "    ax_roc.plot(fpr, tpr, lw=2.5, color='crimson', linestyle='--',\n",
    "               label=f'HyperModel (AUC={roc_auc:.3f})')\n",
    "except Exception as e:\n",
    "    print(f'  HyperModel ROC skip: {e}')\n",
    "\n",
    "ax_roc.set_xlabel('False Positive Rate')\n",
    "ax_roc.set_ylabel('True Positive Rate')\n",
    "ax_roc.set_title('ROC Curves \\u2014 Binary Classification (Benign vs Malicious)')\n",
    "ax_roc.legend(loc='lower right', fontsize=8)\n",
    "plt.tight_layout()\n",
    "plt.savefig(OUTPUT_DIR / 'results_roc_curves.png', dpi=150)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ─── Confusion matrix for HyperModel ──────────────────────────────────────\n",
    "from evaluation import plot_confusion_matrix\n",
    "\n",
    "fig_cm = plot_confusion_matrix(\n",
    "    result_hyper['confusion_matrix'],\n",
    "    class_names=CLASS_NAMES_2,\n",
    "    title='HyperModel Confusion Matrix (Binary)',\n",
    "    save_path=str(OUTPUT_DIR / 'results_confusion_matrix_hypermodel.png'),\n",
    ")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 11. Feature Importance Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from evaluation import plot_feature_importance\n",
    "\n",
    "xgb_model = clf_models_bin['XGBoost (GPU)']\n",
    "importances = xgb_model.feature_importances_\n",
    "\n",
    "fig_fi = plot_feature_importance(\n",
    "    feat_names, importances, top_n=25,\n",
    "    title='XGBoost Feature Importance (Binary)',\n",
    "    save_path=str(OUTPUT_DIR / 'feature_importance_xgb.png'),\n",
    ")\n",
    "plt.show()\n",
    "\n",
    "feat_imp_df = pd.DataFrame({'feature': feat_names, 'importance': importances})\n",
    "print(feat_imp_df.sort_values('importance', ascending=False).head(15).to_string(index=False))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 12. Case Study — Adversarial URL Analysis\n",
    "\n",
    "Testing the HyperModel on hand-crafted adversarial examples: typosquatting,\n",
    "brand impersonation, IP-based phishing, and obfuscated URLs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "CASE_STUDY_URLS = [\n",
    "    # Clearly benign\n",
    "    ('https://www.google.com', 0, 'Legitimate Google'),\n",
    "    ('https://www.amazon.com/dp/B08L5TNJHG', 0, 'Legitimate Amazon'),\n",
    "    ('https://github.com/openai/gpt-4', 0, 'Legitimate GitHub'),\n",
    "    ('https://en.wikipedia.org/wiki/Machine_learning', 0, 'Legitimate Wikipedia'),\n",
    "    # Typosquatting\n",
    "    ('http://googlе.com.login-verify.xyz/account', 1, 'Typosquatting Google'),\n",
    "    ('http://amaz0n-security-alert.com/verify', 1, 'Typosquatting Amazon'),\n",
    "    ('http://paypa1.com-secure.update-account.ru/login', 1, 'Typosquatting PayPal'),\n",
    "    # Brand in subdomain\n",
    "    ('http://apple-id.apple.com-signin.net/verify', 1, 'Brand subdomain phishing'),\n",
    "    ('http://microsoft.com.update-account.xyz/login.php', 1, 'Brand path phishing'),\n",
    "    # IP-based\n",
    "    ('http://192.168.1.234/paypal/login', 1, 'IP-based phishing'),\n",
    "    ('http://45.77.123.45/amazon-account/verify.html', 1, 'IP-based phishing 2'),\n",
    "    # Obfuscated\n",
    "    ('http://xn--pypal-4ve.com/account-login', 1, 'Punycode phishing'),\n",
    "    ('http://bit.ly/3xK7pQr', 1, 'URL shortener (suspicious)'),\n",
    "    ('http://secure-update%2Ebank%2Ecom.evil.tk/login', 1, 'Hex-encoded phishing'),\n",
    "    # Long suspicious\n",
    "    ('http://login-account-update-security-bank-verify.info/account/login/confirm/id=12345&token=abc',\n",
    "     1, 'Long suspicious URL'),\n",
    "    # Legit with suspicious paths\n",
    "    ('https://security.microsoft.com/informationprotection/label', 0, 'Legit MS Security'),\n",
    "    ('https://accounts.google.com/o/oauth2/auth?client_id=abc123', 0, 'Legit Google OAuth'),\n",
    "]\n",
    "\n",
    "case_urls   = [u for u, _, _ in CASE_STUDY_URLS]\n",
    "case_labels = [l for _, l, _ in CASE_STUDY_URLS]\n",
    "case_descs  = [d for _, _, d in CASE_STUDY_URLS]\n",
    "\n",
    "extractor = URLFeatureExtractor()\n",
    "case_feat = extractor.transform(pd.Series(case_urls), verbose=False)\n",
    "case_feat = case_feat[feat_names]\n",
    "X_case = scaler.transform(case_feat.values.astype(np.float32)).astype(np.float32)\n",
    "u_case = np.array(case_urls)\n",
    "\n",
    "pred_xgb   = clf_models_bin['XGBoost (GPU)'].predict_proba(X_case)[:, 1]\n",
    "pred_lgb   = clf_models_bin['LightGBM (CPU)'].predict_proba(X_case)[:, 1]\n",
    "pred_tfidf = tfidf_bin.predict_proba(u_case)[:, 1]\n",
    "pred_hyper = hypermodel.predict_proba(X_case, u_case)[:, 1]\n",
    "\n",
    "case_results = pd.DataFrame({\n",
    "    'Description': case_descs,\n",
    "    'True Label':  ['Malicious' if l else 'Benign' for l in case_labels],\n",
    "    'XGB Score':   pred_xgb.round(3),\n",
    "    'LGB Score':   pred_lgb.round(3),\n",
    "    'TF-IDF Score':pred_tfidf.round(3),\n",
    "    'HyperModel':  pred_hyper.round(3),\n",
    "    'Prediction':  ['Malicious' if p > 0.5 else 'Benign' for p in pred_hyper],\n",
    "    'Correct':     ['\\u2705' if (p>0.5)==bool(l) else '\\u274c'\n",
    "                    for p, l in zip(pred_hyper, case_labels)],\n",
    "})\n",
    "\n",
    "print('\\n\\ud83d\\udccb Case Study Results:')\n",
    "print(case_results.to_string(index=False))\n",
    "accuracy_case = (case_results['Correct'] == '\\u2705').sum() / len(case_results)\n",
    "print(f'\\nCase study accuracy: {accuracy_case:.1%}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ─── Visualise case study ─────────────────────────────────────────────────\n",
    "fig, ax = plt.subplots(figsize=(14, 7))\n",
    "cs_colors = ['#F44336' if l else '#4CAF50' for l in case_labels]\n",
    "x = range(len(case_results))\n",
    "\n",
    "ax.bar(x, pred_hyper, color=cs_colors, alpha=0.8, edgecolor='white', width=0.6)\n",
    "ax.axhline(0.5, color='black', linestyle='--', lw=1.5, label='Decision threshold')\n",
    "ax.set_xticks(x)\n",
    "ax.set_xticklabels(case_descs, rotation=45, ha='right', fontsize=8)\n",
    "ax.set_ylabel('Malicious Probability (HyperModel)')\n",
    "ax.set_title('Case Study: HyperModel Scores on Adversarial URLs')\n",
    "ax.set_ylim(0, 1.1)\n",
    "\n",
    "from matplotlib.patches import Patch\n",
    "legend_els = [\n",
    "    Patch(facecolor='#F44336', alpha=0.8, label='True: Malicious'),\n",
    "    Patch(facecolor='#4CAF50', alpha=0.8, label='True: Benign'),\n",
    "]\n",
    "ax.legend(handles=legend_els + [\n",
    "    plt.Line2D([0],[0], color='black', linestyle='--', label='Threshold')],\n",
    "    loc='upper left')\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig(OUTPUT_DIR / 'case_study_scores.png', dpi=150, bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 13. Save Models & Final Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import joblib\n",
    "\n",
    "MODELS_DIR = OUTPUT_DIR / 'models'\n",
    "MODELS_DIR.mkdir(exist_ok=True)\n",
    "\n",
    "# Classical\n",
    "joblib.dump(clf_models_bin['XGBoost (GPU)'],  MODELS_DIR / 'xgb_binary.joblib')\n",
    "joblib.dump(clf_models_bin['LightGBM (CPU)'], MODELS_DIR / 'lgbm_binary.joblib')\n",
    "joblib.dump(tuned_xgb,  MODELS_DIR / 'xgb_tuned_binary.joblib')\n",
    "joblib.dump(tuned_lgbm, MODELS_DIR / 'lgbm_tuned_binary.joblib')\n",
    "joblib.dump(tfidf_bin,  MODELS_DIR / 'tfidf_lr_binary.joblib')\n",
    "joblib.dump(scaler,     MODELS_DIR / 'scaler.joblib')\n",
    "\n",
    "# DL models from stacking\n",
    "import torch\n",
    "for name, mdl in hypermodel.fitted_l0_.items():\n",
    "    if hasattr(mdl, 'net_'):\n",
    "        state = mdl.net_.state_dict() if not hasattr(mdl.net_, 'module') \\\n",
    "                else mdl.net_.module.state_dict()\n",
    "        torch.save(state, MODELS_DIR / f'{name}_net.pt')\n",
    "\n",
    "# Meta-learner\n",
    "joblib.dump(hypermodel.meta, MODELS_DIR / 'hypermodel_meta.joblib')\n",
    "\n",
    "# Results tables\n",
    "comparison_bin.to_csv(OUTPUT_DIR / 'results_binary.csv', index=False)\n",
    "comparison_mc.to_csv(OUTPUT_DIR / 'results_multiclass.csv', index=False)\n",
    "case_results.to_csv(OUTPUT_DIR / 'case_study_results.csv', index=False)\n",
    "\n",
    "print(f'\\n\\u2705 All models and results saved to: {OUTPUT_DIR.resolve()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ─── Final Summary ───────────────────────────────────────────────────────\n",
    "print('\\n' + '='*60)\n",
    "print('  FINAL RESULTS SUMMARY')\n",
    "print('='*60)\n",
    "\n",
    "print('\\n\\ud83d\\udccc Binary (Benign vs Malicious):')\n",
    "ours_bin = comparison_bin[comparison_bin['Source']=='Ours']\n",
    "print(ours_bin[['Method','Accuracy','F1 (macro)','AUC-ROC']].to_string(index=False))\n",
    "\n",
    "print('\\n\\ud83d\\udccc 4-Class (benign / phishing / defacement / malware):')\n",
    "ours_mc = comparison_mc[comparison_mc['Source']=='Ours']\n",
    "print(ours_mc[['Method','Accuracy','F1 (macro)','AUC-ROC']].to_string(index=False))\n",
    "\n",
    "print('\\n\\ud83d\\udccc Key Literature Baselines:')\n",
    "print('   Sahingoz 2019 (RF):       97.98% acc')\n",
    "print('   Le 2018 (URLNet):         ~98.5% acc')\n",
    "print('   Nguyen 2021 (Stacking):   97.5% acc')\n",
    "print('   Aljofey 2022 (CharCNN):   96.75% acc')\n",
    "\n",
    "best_ours = ours_bin['Accuracy'].max()\n",
    "best_lit  = 0.985  # URLNet\n",
    "delta     = best_ours - best_lit\n",
    "print(f'\\n  Our best binary : {best_ours:.4f}')\n",
    "print(f'  \\u0394 vs literature  : {delta:+.4f} ({\"\\u2705 BEAT\" if delta > 0 else \"\\u274c BELOW\"} URLNet baseline)')\n",
    "print('='*60)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 14. Inference on New URLs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_url(url: str, threshold: float = 0.5) -> dict:\n",
    "    \"\"\"Predict whether a URL is malicious.\"\"\"\n",
    "    ext = URLFeatureExtractor()\n",
    "    feat_df = ext.transform(pd.Series([url]), verbose=False)\n",
    "    feat_df = feat_df[feat_names]\n",
    "    X_sc = scaler.transform(feat_df.values.astype(np.float32)).astype(np.float32)\n",
    "\n",
    "    proba = hypermodel.predict_proba(X_sc, np.array([url]))\n",
    "    score = proba[0, 1]\n",
    "    label = 'Malicious' if score > threshold else 'Benign'\n",
    "\n",
    "    top = feat_imp_df.sort_values('importance', ascending=False).head(5)\n",
    "    top_signals = {row['feature']: round(float(feat_df.iloc[0].get(row['feature'], 0)), 3)\n",
    "                   for _, row in top.iterrows()}\n",
    "\n",
    "    return {\n",
    "        'url': url,\n",
    "        'prediction': label,\n",
    "        'malicious_score': round(float(score), 4),\n",
    "        'confidence': f\"{abs(score - 0.5) * 200:.1f}%\",\n",
    "        'top_signals': top_signals,\n",
    "    }\n",
    "\n",
    "\n",
    "test_urls = [\n",
    "    'https://www.google.com/search?q=machine+learning',\n",
    "    'http://secure-paypal-login.xyz/verify-account?user=victim@email.com',\n",
    "    'https://github.com/anthropics/anthropic-sdk-python',\n",
    "    'http://192.168.10.45/banking/login.php?redirect=true',\n",
    "    'http://free-iphone-giveaway.tk/claim?id=abc123',\n",
    "]\n",
    "\n",
    "for url in test_urls:\n",
    "    r = predict_url(url)\n",
    "    icon = '\\ud83d\\udd34' if r['prediction'] == 'Malicious' else '\\ud83d\\udfe2'\n",
    "    print(f\"{icon} [{r['prediction']:9s}] score={r['malicious_score']:.4f} | {url[:70]}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}