urlmodel_info/download_data.py

"""
Download the Malicious URLs dataset (~651K rows, 4 classes).

Dataset: Manu Siddhartha — Malicious URLs Dataset
Source:  https://www.kaggle.com/datasets/sid321axn/malicious-urls-dataset
Classes: benign (428,103) · defacement (96,457) · phishing (94,111) · malware (32,520)
"""

import os
import sys
import pathlib
import shutil
import subprocess
import glob

KAGGLE_SLUG = "sid321axn/malicious-urls-dataset"

CSV_FILENAME = "malicious_phishing.csv"


def download_malicious_urls(data_dir) -> pathlib.Path:
    """Download and return the path to the Malicious URLs CSV.

    Tries kagglehub first, then the kaggle CLI, then prints manual
    download instructions.

    Parameters
    ----------
    data_dir : str or pathlib.Path
        Directory to store the downloaded CSV.

    Returns
    -------
    pathlib.Path
        Path to the CSV file.
    """
    data_dir = pathlib.Path(data_dir)
    data_dir.mkdir(parents=True, exist_ok=True)
    csv_path = data_dir / CSV_FILENAME

    if csv_path.exists():
        print(f"✅ Dataset already exists: {csv_path}")
        return csv_path

    # ── Attempt 1: kagglehub ──────────────────────────────────────────────
    try:
        import kagglehub
    except ImportError:
        subprocess.check_call(
            [sys.executable, "-m", "pip", "install", "kagglehub", "-q"]
        )
        import kagglehub

    try:
        print("Downloading via kagglehub …")
        dl_path = kagglehub.dataset_download(KAGGLE_SLUG)
        src = _find_csv(dl_path)
        if src:
            shutil.copy2(src, csv_path)
            print(f"✅ Dataset saved to {csv_path}")
            return csv_path
    except Exception as e:
        print(f"  kagglehub failed: {e}")

    # ── Attempt 2: kaggle CLI ────────────────────────────────────────────
    if shutil.which("kaggle"):
        try:
            print("Downloading via kaggle CLI …")
            subprocess.check_call(
                [
                    "kaggle", "datasets", "download",
                    "-d", KAGGLE_SLUG,
                    "-p", str(data_dir),
                    "--unzip",
                ],
                timeout=300,
            )
            src = _find_csv(str(data_dir))
            if src and pathlib.Path(src) != csv_path:
                shutil.move(src, csv_path)
            if csv_path.exists():
                print(f"✅ Dataset saved to {csv_path}")
                return csv_path
        except Exception as e:
            print(f"  kaggle CLI failed: {e}")

    # ── Manual instructions ──────────────────────────────────────────────
    print("\n" + "=" * 60)
    print("  MANUAL DOWNLOAD REQUIRED")
    print("=" * 60)
    print(f"  1. Visit: https://www.kaggle.com/datasets/{KAGGLE_SLUG}")
    print(f"  2. Click 'Download' and extract the ZIP")
    print(f"  3. Place '{CSV_FILENAME}' in: {data_dir.resolve()}")
    print("=" * 60)
    raise FileNotFoundError(
        f"Could not auto-download dataset. Place '{CSV_FILENAME}' in {data_dir}"
    )


def _find_csv(root_dir: str):
    """Walk a directory tree and return the first CSV file path found."""
    for dirpath, _, filenames in os.walk(root_dir):
        for fn in filenames:
            if fn.endswith(".csv"):
                return os.path.join(dirpath, fn)
    return None


if __name__ == "__main__":
    path = download_malicious_urls("data")
    print(f"Dataset at: {path}")