refactor: Relocate ML models to backend package, update path loaders, simplify Dockerfile

backend/app/models/url.py

@@ -51,6 +51,12 @@ class URLAnalyzeResponse(BaseModel):
     malicious_score: float = Field(..., ge=0.0, le=1.0)
     confidence: str
     confidence_level: URLConfidenceLevel
+    raw_score: float = Field(
+        ...,
+        ge=0.0,
+        le=1.0,
+        description="Meta-learner score before isotonic calibration.",
+    )
     threshold_used: float
     base_model_scores: Dict[str, float]
     top_signals: Dict[str, float]
@@ -62,7 +68,7 @@ class URLModelInfoResponse(BaseModel):
     model_type: str
     base_models: List[str]
     meta_learner: str
+    calibration: str
     scaler_features: int
     meta_features: int
-    dl_available: bool
     feature_categories: List[str]

backend/app/routers/url.py

@@ -39,13 +39,15 @@ async def analyze_url(
 ):
     """
     Analyze a URL for malicious indicators using the stacking HyperModel.
-    
-    The HyperModel combines 5 base models:
-    - XGBoost & LightGBM on 54 lexical features
+
+    The HyperModel combines 3 base models:
+    - XGBoost & LightGBM on 66 lexical features
     - TF-IDF + Logistic Regression on raw URL text
-    - CharCNN & CharBiLSTM deep learning on character-level URL encoding
-    
-    Their probability outputs are stacked and fed to an XGBoost meta-learner.
+
+    Their P(malicious) outputs are stacked into a logistic-regression
+    meta-learner, whose score is then passed through isotonic calibration.
+    The response carries both `raw_score` (pre-calibration) and
+    `malicious_score` (calibrated, and what the verdict uses).
     """
     try:
         url_service = get_url_service()
@@ -89,9 +91,9 @@ async def url_model_info():
             "model_type": "Stacking HyperModel",
             "base_models": [],
             "meta_learner": "hidden",
+            "calibration": "hidden",
             "scaler_features": 0,
             "meta_features": 0,
-            "dl_available": False,
             "feature_categories": [],
         }
     try:
@@ -111,7 +113,8 @@ async def url_health():
         return {
             "status": "healthy" if url_service.model_loaded else "unhealthy",
             "model_loaded": url_service.model_loaded,
-            "dl_available": url_service._dl_available,
+            "base_models": list(url_service.BASE_MODEL_NAMES),
+            "calibrated": url_service.calibrator is not None,
         }
     except Exception as e:
         return {

backend/app/services/feedback_service.py

@@ -16,7 +16,8 @@ import json
 import logging
 import sqlite3
 import time
-from datetime import datetime, timezone
+import warnings
+from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from typing import Optional
 
@@ -244,6 +245,49 @@ class FeedbackService:
                 return None
         return None
 
+    def get_daily_counts(self, days: int = 30) -> list[dict]:
+        """
+        Real per-day feedback counts for the dashboard time series.
+
+        The frontend used to synthesise this from the totals with a modulo seed,
+        which drew a plausible-looking 30-day chart out of a single number. Both
+        feedback tables store created_at, so the actual series is available.
+
+        Returns one entry per day, oldest first, with zero-filled gaps so the
+        chart has a continuous x-axis.
+        """
+        conn = self._get_conn()
+
+        def counts_for(table: str) -> dict[str, int]:
+            # Table name is interpolated, so restrict it to a known set rather
+            # than trusting callers.
+            if table not in {"malware_feedback", "url_feedback"}:
+                raise ValueError(f"unexpected table {table!r}")
+            rows = conn.execute(
+                "SELECT substr(created_at, 1, 10) AS day, COUNT(*) AS n "
+                f"FROM {table} WHERE created_at >= ? GROUP BY day",
+                (self._cutoff_iso(days),),
+            ).fetchall()
+            return {r["day"]: r["n"] for r in rows}
+
+        malware = counts_for("malware_feedback")
+        url = counts_for("url_feedback")
+
+        today = datetime.now(timezone.utc).date()
+        series = []
+        for offset in range(days - 1, -1, -1):
+            day = today - timedelta(days=offset)
+            key = day.isoformat()
+            m = malware.get(key, 0)
+            u = url.get(key, 0)
+            series.append({"date": key, "malware": m, "url": u, "total": m + u})
+        return series
+
+    @staticmethod
+    def _cutoff_iso(days: int) -> str:
+        cutoff = datetime.now(timezone.utc) - timedelta(days=days - 1)
+        return cutoff.date().isoformat()
+
     def get_stats(self) -> dict:
         conn = self._get_conn()
 
@@ -301,6 +345,7 @@ class FeedbackService:
             "recent_malware": recent_m,
             "recent_url": recent_u,
             "model_updates": updates,
+            "daily_counts": self.get_daily_counts(30),
         }
 
     # ── Incremental model updates ─────────────────────────────────────────
@@ -381,11 +426,18 @@ class FeedbackService:
         self, min_samples: int = 10, lr: float = 0.01, rounds: int = 5
     ) -> dict:
         """
-        Incrementally update the URL meta-learner with accumulated
-        user feedback.  Reconstructs 10 meta-features from stored
-        base model scores.
+        Incrementally update the URL meta-learner with accumulated user feedback.
+        Reconstructs the 3 meta-features (one P(malicious) per base model) from
+        stored base model scores.
+
+        The meta-learner is a LogisticRegression, so 'incremental' here means
+        refitting on the original decision surface plus the feedback rows rather
+        than appending boosting rounds. `lr` and `rounds` are accepted for API
+        compatibility and ignored; `rounds` is reported back so callers can see
+        it had no effect.
         """
-        import xgboost as xgb
+        from sklearn.linear_model import LogisticRegression
+
         from .url_service import get_url_service
 
         conn = self._get_conn()
@@ -401,45 +453,69 @@ class FeedbackService:
                 "message": f"Need ≥ {min_samples} samples, have {len(rows)}",
             }
 
-        BASE_MODEL_ORDER = [
-            "XGBoost", "LightGBM", "TF-IDF + LR", "CharCNN", "CharBiLSTM"
-        ]
+        # One column per base model, P(malicious) only - matches the 3-column
+        # meta-feature layout the calibrated ensemble was trained on.
+        BASE_MODEL_ORDER = ["XGBoost", "LightGBM", "TF-IDF + LR"]
 
         meta_features = []
         labels = []
         for r in rows:
             scores = json.loads(r["base_model_scores"])
-            mf = []
-            for name in BASE_MODEL_ORDER:
-                mal = scores.get(name, 0.5)
-                mf.extend([1.0 - mal, mal])
+            mf = [float(scores.get(name, 0.5)) for name in BASE_MODEL_ORDER]
             meta_features.append(mf)
             labels.append(1 if r["correct_label"] == "Malicious" else 0)
 
         X = np.array(meta_features, dtype=np.float32)
-        y = np.array(labels, dtype=np.float32)
-
-        url_service = get_url_service()
-        booster = url_service.meta_learner.get_booster()
-        dtrain = xgb.DMatrix(X, label=y)
+        y = np.array(labels, dtype=np.int32)
 
-        params = {
-            "objective": "binary:logistic",
-            "eval_metric": "logloss",
-            "learning_rate": lr,
-            "max_depth": 3,
-        }
+        # A refit needs both classes present, otherwise LogisticRegression raises.
+        if len(np.unique(y)) < 2:
+            only = "Malicious" if y[0] == 1 else "Benign"
+            return {
+                "success": False,
+                "model_type": "url",
+                "samples_used": len(rows),
+                "message": (
+                    f"All {len(rows)} feedback samples are labelled {only}. "
+                    "Need both classes to refit the meta-learner."
+                ),
+            }
 
-        updated = xgb.train(
-            params, dtrain, num_boost_round=rounds, xgb_model=booster
+        url_service = get_url_service()
+        old_meta = url_service.meta_learner
+
+        # Anchor the refit to the existing decision surface: sample the current
+        # meta-learner across the unit cube and mix those pseudo-rows in, so a
+        # handful of feedback samples nudges the boundary instead of replacing it.
+        # The grid reaches 0.0/1.0 so the anchor carries the model's confident
+        # regions, not just its middle.
+        levels = (0.0, 0.15, 0.4, 0.6, 0.85, 1.0)
+        grid = np.array(
+            [[a, b, c] for a in levels for b in levels for c in levels],
+            dtype=np.float32,
         )
-
-        # Patch sklearn model
-        url_service.meta_learner._Booster = updated
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", message="X does not have valid feature names")
+            grid_labels = (old_meta.predict_proba(grid)[:, 1] > 0.5).astype(np.int32)
+
+        X_fit = np.vstack([grid, X])
+        y_fit = np.concatenate([grid_labels, y])
+        # Feedback rows outweigh the anchor per-sample so real corrections move the
+        # boundary, but the anchor keeps total mass so a handful of rows can't
+        # flatten the surface. Capped at 25% of the grid's mass in aggregate.
+        per_row = min(8.0, (0.25 * len(grid)) / max(len(X), 1))
+        weights = np.concatenate([
+            np.ones(len(grid), dtype=np.float32),
+            np.full(len(X), per_row, dtype=np.float32),
+        ])
+
+        updated = LogisticRegression(C=1.0, max_iter=500, solver="lbfgs", random_state=42)
+        updated.fit(X_fit, y_fit, sample_weight=weights)
+        url_service.meta_learner = updated
 
         # Persist
         meta_path = url_service.models_dir / "hypermodel_meta.joblib"
-        joblib.dump(url_service.meta_learner, meta_path)
+        joblib.dump(updated, meta_path)
 
         # Log update
         now = datetime.now(timezone.utc).isoformat()
@@ -450,14 +526,15 @@ class FeedbackService:
         )
         conn.commit()
 
-        logger.info(
-            f"URL meta-learner updated: {len(rows)} samples, lr={lr}, rounds={rounds}"
-        )
+        logger.info(f"URL meta-learner refit on {len(rows)} feedback samples")
         return {
             "success": True,
             "model_type": "url",
             "samples_used": len(rows),
-            "message": f"URL meta-learner updated with {len(rows)} feedback samples ({rounds} rounds, lr={lr})",
+            "message": (
+                f"URL meta-learner refit with {len(rows)} feedback samples "
+                f"(logistic meta-learner: lr/rounds not applicable)"
+            ),
         }
 
 

backend/app/services/url_feature_engineering.py

@@ -1,5 +1,5 @@
 """
-URL Feature Engineering — 54 lexical features for malicious URL detection.
+URL Feature Engineering — 66 lexical features for malicious URL detection.
 
 Adapted from the URL model notebook for backend integration.
 Features are grouped into 8 categories:

backend/app/services/url_service.py

@@ -1,399 +1,297 @@
-"""
-URL Malware Detection Service — Stacking HyperModel.
-
-Full inference pipeline:
-  1. Extract 54 lexical features from URL
-  2. Scale features with StandardScaler
-  3. Run base models (XGBoost, LightGBM, and optionally CharCNN + CharBiLSTM)
-  4. Stack their probability outputs → meta-features
-  5. Meta-learner (XGBoost) produces final prediction
-"""
-
-from __future__ import annotations
-
-import logging
-import time
-import warnings
-from pathlib import Path
-
-
-import numpy as np
-import joblib
-
-from .url_feature_engineering import extract_url_features, extract_url_features_dict, FEATURE_NAMES
-
-logger = logging.getLogger(__name__)
-
-# ── DL architecture definitions (inline, no external dependency) ──────────────
-# These mirror the exact architectures from the training notebook.
-
-MAX_URL_LEN = 200
-
-# Character lookup table
-_CHAR_LUT = np.zeros(256, dtype=np.int64)
-_PRINTABLE = (
-    "abcdefghijklmnopqrstuvwxyz"
-    "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
-    "0123456789"
-    "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~ "
-)
-for _i, _ch in enumerate(_PRINTABLE, start=1):
-    _CHAR_LUT[ord(_ch)] = _i
-_UNK_ID = len(_PRINTABLE) + 1  # 96
-VOCAB_SIZE = _UNK_ID + 1       # 97
-
-try:
-    import torch
-    import torch.nn as nn
-    TORCH_AVAILABLE = True
-except ImportError:
-    TORCH_AVAILABLE = False
-    logger.warning("PyTorch not available — DL base models will be skipped")
-
-
-def _url_to_tensor(urls: list[str], max_len: int = MAX_URL_LEN):
-    """Vectorised char→int encoding for a batch of URL strings."""
-    arr = np.zeros((len(urls), max_len), dtype=np.int64)
-    for i, url in enumerate(urls):
-        raw = str(url)[:max_len]
-        chars = np.frombuffer(raw.encode("ascii", "ignore"), dtype=np.uint8)
-        if len(chars) == 0:
-            continue
-        ids = _CHAR_LUT[chars]
-        ids = np.where(ids > 0, ids, _UNK_ID)
-        arr[i, : len(ids)] = ids[: max_len]
-    import torch
-    return torch.from_numpy(arr)
-
-
-if TORCH_AVAILABLE:
-    class CharCNNNet(nn.Module):
-        """1-D CNN on character-level URL embeddings."""
-        def __init__(self, num_classes: int = 2, embed_dim: int = 64):
-            super().__init__()
-            self.embed = nn.Embedding(VOCAB_SIZE, embed_dim, padding_idx=0)
-            self.convs = nn.Sequential(
-                nn.Conv1d(embed_dim, 128, kernel_size=3, padding=1),
-                nn.BatchNorm1d(128), nn.ReLU(), nn.MaxPool1d(2),
-                nn.Conv1d(128, 128, kernel_size=3, padding=1),
-                nn.BatchNorm1d(128), nn.ReLU(), nn.MaxPool1d(2),
-                nn.Conv1d(128, 256, kernel_size=3, padding=1),
-                nn.BatchNorm1d(256), nn.ReLU(), nn.AdaptiveMaxPool1d(1),
-            )
-            self.head = nn.Sequential(
-                nn.Dropout(0.3), nn.Linear(256, 128), nn.ReLU(),
-                nn.Dropout(0.2), nn.Linear(128, num_classes),
-            )
-
-        def forward(self, x):
-            x = self.embed(x).permute(0, 2, 1)
-            x = self.convs(x).squeeze(-1)
-            return self.head(x)
-
-    class CharBiLSTMNet(nn.Module):
-        """Bidirectional LSTM on character embeddings."""
-        def __init__(self, num_classes: int = 2, embed_dim: int = 64,
-                     hidden: int = 128, n_layers: int = 2):
-            super().__init__()
-            self.embed = nn.Embedding(VOCAB_SIZE, embed_dim, padding_idx=0)
-            self.lstm = nn.LSTM(
-                embed_dim, hidden, num_layers=n_layers,
-                batch_first=True, bidirectional=True, dropout=0.3,
-            )
-            self.head = nn.Sequential(
-                nn.Dropout(0.3), nn.Linear(hidden * 2, 128), nn.ReLU(),
-                nn.Dropout(0.2), nn.Linear(128, num_classes),
-            )
-
-        def forward(self, x):
-            x = self.embed(x)
-            _, (h, _) = self.lstm(x)
-            h = torch.cat([h[-2], h[-1]], dim=-1)
-            return self.head(h)
-
-
-# Highly trusted popular domains to prevent false positives on official/major websites
-TRUSTED_DOMAINS = {
-    "google.com", "wikipedia.org", "wikimedia.org", "github.com", "github.io",
-    "apple.com", "microsoft.com", "microsoftonline.com", "live.com", "outlook.com",
-    "office.com", "bing.com", "msn.com", "yahoo.com", "amazon.com",
-    "paypal.com", "wellsfargo.com", "facebook.com", "instagram.com", "whatsapp.com",
-    "twitter.com", "x.com", "linkedin.com", "netflix.com", "dropbox.com", "zoom.us",
-    "salesforce.com", "adobe.com", "ebay.com", "chase.com", "bankofamerica.com",
-    "citibank.com", "hsbc.com", "dhl.com", "fedex.com", "usps.com", "spotify.com",
-    "reddit.com", "google.co.uk", "google.ca", "google.de", "google.fr",
-    "amazon.co.uk", "amazon.de", "amazon.co.jp"
-}
-
-
-def _get_registered_domain(url: str) -> str:
-    """Helper to extract the registered domain (e.g. google.com) from a URL."""
-    try:
-        from urllib.parse import urlparse
-        try:
-            import tldextract
-        except ImportError:
-            tldextract = None
-    except Exception:
-        return ""
-        
-    url = str(url).strip()
-    if "://" not in url:
-        url = f"http://{url}"
-        
-    try:
-        parsed = urlparse(url)
-        hostname = parsed.hostname or ""
-    except Exception:
-        hostname = ""
-        
-    if not hostname:
-        return ""
-        
-    hostname = hostname.lower()
-    if ":" in hostname:
-        hostname = hostname.split(":")[0]
-        
-    if tldextract is not None:
-        try:
-            ext = tldextract.extract(url)
-            if ext.domain and ext.suffix:
-                return f"{ext.domain}.{ext.suffix}".lower()
-        except Exception:
-            pass
-            
-    # Fallback to last two parts of the hostname
-    parts = hostname.split(".")
-    if len(parts) >= 2:
-        return f"{parts[-2]}.{parts[-1]}".lower()
-    return hostname
-
-
-class URLModelService:
-    """
-    Stacking HyperModel service for URL malware detection.
-    
-    Loads all base models + meta-learner at initialization time.
-    Thread-safe for concurrent FastAPI requests.
-    """
-
-    def __init__(self, models_dir: str | Path | None = None):
-        if models_dir is None:
-            # Default to urlmodel_info/ in project root
-            models_dir = Path(__file__).parent.parent.parent.parent / "urlmodel_info"
-        self.models_dir = Path(models_dir)
-        
-        self.scaler = None
-        self.meta_learner = None
-        self.xgb_model = None
-        self.lgbm_model = None
-        self.cnn_net = None
-        self.bilstm_net = None
-        self.model_loaded = False
-        self._dl_available = False
-        
-        self._load_models()
-
-    def _load_models(self):
-        """Load all model components."""
-        logger.info("Loading URL HyperModel components...")
-        
-        with warnings.catch_warnings():
-            warnings.simplefilter("ignore")
-            
-            # 1. Scaler
-            scaler_path = self.models_dir / "scaler.joblib"
-            self.scaler = joblib.load(scaler_path)
-            logger.info(f"  Scaler loaded ({self.scaler.n_features_in_} features)")
-            
-            # 2. Meta-learner
-            meta_path = self.models_dir / "hypermodel_meta.joblib"
-            self.meta_learner = joblib.load(meta_path)
-            logger.info(f"  Meta-learner loaded ({self.meta_learner.n_features_in_} meta-features)")
-            
-            # 3. XGBoost base model
-            xgb_path = self.models_dir / "xgb_binary.joblib"
-            self.xgb_model = joblib.load(xgb_path)
-            logger.info("  XGBoost base model loaded")
-            
-            # 4. LightGBM base model
-            lgbm_path = self.models_dir / "lgbm_binary.joblib"
-            self.lgbm_model = joblib.load(lgbm_path)
-            logger.info("  LightGBM base model loaded")
-            
-            # 5. Deep Learning models (optional — requires PyTorch)
-            if TORCH_AVAILABLE:
-                try:
-                    cnn_path = self.models_dir / "char_cnn_net.pt"
-                    bilstm_path = self.models_dir / "char_bilstm_net.pt"
-                    
-                    self.cnn_net = CharCNNNet(num_classes=2)
-                    self.cnn_net.load_state_dict(
-                        torch.load(cnn_path, map_location="cpu", weights_only=True)
-                    )
-                    self.cnn_net.eval()
-                    logger.info("  CharCNN loaded")
-                    
-                    self.bilstm_net = CharBiLSTMNet(num_classes=2)
-                    self.bilstm_net.load_state_dict(
-                        torch.load(bilstm_path, map_location="cpu", weights_only=True)
-                    )
-                    self.bilstm_net.eval()
-                    logger.info("  CharBiLSTM loaded")
-                    
-                    self._dl_available = True
-                except Exception as e:
-                    logger.warning(f"  DL models failed to load: {e}")
-                    self._dl_available = False
-            
-        self.model_loaded = True
-        n_base = 2 + (2 if self._dl_available else 0)
-        logger.info(f"URL HyperModel ready ({n_base} base models + meta-learner)")
-
-    def predict(self, url: str, threshold: float = 0.5) -> dict:
-        """
-        Run the full stacking pipeline on a single URL.
-        
-        Returns dict with:
-          - url: input URL
-          - prediction: 'Malicious' or 'Benign'
-          - malicious_score: float [0, 1]
-          - confidence: str percentage
-          - confidence_level: 'High', 'Medium', or 'Low'
-          - base_model_scores: dict of individual model scores
-          - top_signals: dict of most important feature values
-          - inference_time_ms: float
-        """
-        start = time.perf_counter()
-        
-        # 1. Extract features
-        features = extract_url_features(url)
-        features_dict = extract_url_features_dict(url)
-        
-        # Ensure backward compatibility with existing model/scaler
-        if self.scaler is not None:
-            features = features[:self.scaler.n_features_in_]
-        X_feat = features.reshape(1, -1)
-        
-        # 2. Scale features
-        X_scaled = self.scaler.transform(X_feat).astype(np.float32)
-        
-        # 3. Get base model predictions
-        p_xgb = self.xgb_model.predict_proba(X_scaled)
-        p_lgbm = self.lgbm_model.predict_proba(X_scaled)
-        
-        base_scores = {
-            "XGBoost": float(p_xgb[0, 1]),
-            "LightGBM": float(p_lgbm[0, 1]),
-        }
-        
-        # Determine how many features the meta-learner expects (e.g. 4 or 8)
-        meta_feats_expected = getattr(self.meta_learner, "n_features_in_", 8)
-        
-        # 4. DL models (if available and expected by the meta-learner)
-        if meta_feats_expected == 4:
-            meta_features = np.hstack([p_xgb, p_lgbm])
-        else:
-            if self._dl_available:
-                xt = _url_to_tensor([url])
-                with torch.no_grad():
-                    p_cnn = torch.softmax(self.cnn_net(xt).float(), dim=-1).numpy()
-                    p_bilstm = torch.softmax(self.bilstm_net(xt).float(), dim=-1).numpy()
-                base_scores["CharCNN"] = float(p_cnn[0, 1])
-                base_scores["CharBiLSTM"] = float(p_bilstm[0, 1])
-                meta_features = np.hstack([p_xgb, p_lgbm, p_cnn, p_bilstm])
-            else:
-                # Fallback: pad with zeros for missing DL models
-                zeros = np.zeros((1, 2), dtype=np.float32)
-                meta_features = np.hstack([p_xgb, p_lgbm, zeros, zeros])
-        
-        meta_features = meta_features.astype(np.float32)
-        
-        # 5. Meta-learner prediction
-        final_proba = self.meta_learner.predict_proba(meta_features)
-        malicious_score = float(final_proba[0, 1])
-        
-        # Check domain-level trust resolver to prevent high-entropy administrative false positives
-        reg_domain = _get_registered_domain(url)
-        if reg_domain and reg_domain in TRUSTED_DOMAINS:
-            malicious_score = malicious_score * 0.15  # Dampen but don't hard-cap for trusted domains
-            
-        # Determine verdict
-        prediction = "Malicious" if malicious_score > threshold else "Benign"
-        confidence_pct = abs(malicious_score - 0.5) * 200
-        
-        if confidence_pct >= 80:
-            confidence_level = "High"
-        elif confidence_pct >= 50:
-            confidence_level = "Medium"
-        else:
-            confidence_level = "Low"
-        
-        # Top feature signals
-        important_features = [
-            "has_ip", "has_suspicious_tld", "phishing_keyword_count",
-            "brand_in_subdomain", "has_punycode", "is_shortened",
-            "url_length", "num_subdomains", "has_https", "domain_entropy",
-            "has_suspicious_ext", "has_hex_encoding", "brand_count",
-            "path_depth", "url_entropy",
-        ]
-        top_signals = {}
-        for feat in important_features:
-            if feat in features_dict:
-                val = features_dict[feat]
-                if val != 0:
-                    top_signals[feat] = round(float(val), 4)
-        
-        elapsed = (time.perf_counter() - start) * 1000
-        
-        return {
-            "url": url,
-            "prediction": prediction,
-            "malicious_score": round(malicious_score, 6),
-            "confidence": f"{confidence_pct:.1f}%",
-            "confidence_level": confidence_level,
-            "threshold_used": threshold,
-            "base_model_scores": base_scores,
-            "top_signals": top_signals,
-            "inference_time_ms": round(elapsed, 2),
-        }
-
-    def get_model_info(self) -> dict:
-        """Return info about loaded models."""
-        return {
-            "model_type": "Stacking HyperModel",
-            "base_models": list(self._get_base_model_names()),
-            "meta_learner": "XGBoost (GPU-trained)",
-            "scaler_features": self.scaler.n_features_in_,
-            "meta_features": self.meta_learner.n_features_in_,
-            "dl_available": self._dl_available,
-            "feature_categories": [
-                "Length/structural (7)",
-                "Character counts (14)",
-                "Character ratios (5)",
-                "Entropy/randomness (2)",
-                "Boolean indicators (10)",
-                "Domain/TLD signals (6)",
-                "Suspicious patterns (5)",
-                "Path & query (5)",
-            ],
-        }
-
-    def _get_base_model_names(self):
-        meta_feats_expected = getattr(self.meta_learner, "n_features_in_", 8)
-        names = ["XGBoost", "LightGBM"]
-        if meta_feats_expected == 8 and self._dl_available:
-            names.extend(["CharCNN", "CharBiLSTM"])
-        return names
-
-
-# Singleton accessor
-_url_service: URLModelService | None = None
-
-
-def get_url_service() -> URLModelService:
-    """Get or create the singleton URL model service."""
-    global _url_service
-    if _url_service is None:
-        _url_service = URLModelService()
-    return _url_service
+"""
+URL Malware Detection Service - Stacking HyperModel.
+
+Full inference pipeline:
+  1. Extract 66 lexical features from the URL
+  2. Scale features with StandardScaler
+  3. Run three base models: XGBoost, LightGBM (both on scaled features)
+     and TF-IDF + LogisticRegression (on the raw URL text)
+  4. Stack their P(malicious) outputs into a 3-column meta-feature vector
+  5. LogisticRegression meta-learner produces a raw score
+  6. IsotonicRegression calibrates that score into a usable probability
+
+Step 6 is what stops scores saturating at 1.0, and together with the cleaned
+training corpus it is what removed the benign false positives - the old model
+called github.com and google.com malicious because its labels said so.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+import warnings
+from pathlib import Path
+
+import numpy as np
+import joblib
+
+from .url_feature_engineering import extract_url_features, extract_url_features_dict
+
+logger = logging.getLogger(__name__)
+
+
+# Highly trusted popular domains to prevent false positives on official/major websites.
+# Retained deliberately: the retrained model still has one residual false positive on
+# the OAuth consent-URL pattern, so this stays as the final safety net.
+TRUSTED_DOMAINS = {
+    "google.com", "wikipedia.org", "wikimedia.org", "github.com", "github.io",
+    "apple.com", "microsoft.com", "microsoftonline.com", "live.com", "outlook.com",
+    "office.com", "bing.com", "msn.com", "yahoo.com", "amazon.com",
+    "paypal.com", "wellsfargo.com", "facebook.com", "instagram.com", "whatsapp.com",
+    "twitter.com", "x.com", "linkedin.com", "netflix.com", "dropbox.com", "zoom.us",
+    "salesforce.com", "adobe.com", "ebay.com", "chase.com", "bankofamerica.com",
+    "citibank.com", "hsbc.com", "dhl.com", "fedex.com", "usps.com", "spotify.com",
+    "reddit.com", "google.co.uk", "google.ca", "google.de", "google.fr",
+    "amazon.co.uk", "amazon.de", "amazon.co.jp"
+}
+
+
+def _get_registered_domain(url: str) -> str:
+    """Helper to extract the registered domain (e.g. google.com) from a URL."""
+    try:
+        from urllib.parse import urlparse
+        try:
+            import tldextract
+        except ImportError:
+            tldextract = None
+    except Exception:
+        return ""
+
+    url = str(url).strip()
+    if "://" not in url:
+        url = f"http://{url}"
+
+    try:
+        parsed = urlparse(url)
+        hostname = parsed.hostname or ""
+    except Exception:
+        hostname = ""
+
+    if not hostname:
+        return ""
+
+    hostname = hostname.lower()
+    if ":" in hostname:
+        hostname = hostname.split(":")[0]
+
+    if tldextract is not None:
+        try:
+            ext = tldextract.extract(url)
+            if ext.domain and ext.suffix:
+                return f"{ext.domain}.{ext.suffix}".lower()
+        except Exception:
+            pass
+
+    # Fallback to last two parts of the hostname
+    parts = hostname.split(".")
+    if len(parts) >= 2:
+        return f"{parts[-2]}.{parts[-1]}".lower()
+    return hostname
+
+
+class URLModelService:
+    """
+    Stacking HyperModel service for URL malware detection.
+
+    Loads all base models, the meta-learner and the calibrator at init.
+    Thread-safe for concurrent FastAPI requests.
+    """
+
+    BASE_MODEL_NAMES = ("XGBoost", "LightGBM", "TF-IDF + LR")
+
+    def __init__(self, models_dir: str | Path | None = None):
+        if models_dir is None:
+            # Default to urlmodel_info/ in project root
+            models_dir = Path(__file__).parent.parent.parent.parent / "urlmodel_info"
+        self.models_dir = Path(models_dir)
+
+        self.scaler = None
+        self.meta_learner = None
+        self.calibrator = None
+        self.xgb_model = None
+        self.lgbm_model = None
+        self.tfidf_lr = None
+        self.model_loaded = False
+
+        self._load_models()
+
+    def _load_models(self):
+        """Load all model components."""
+        logger.info("Loading URL HyperModel components...")
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+
+            self.scaler = joblib.load(self.models_dir / "scaler.joblib")
+            logger.info(f"  Scaler loaded ({self.scaler.n_features_in_} features)")
+
+            self.meta_learner = joblib.load(self.models_dir / "hypermodel_meta.joblib")
+            n_meta = getattr(self.meta_learner, "n_features_in_", None)
+            logger.info(f"  Meta-learner loaded ({n_meta} meta-features)")
+
+            self.calibrator = joblib.load(self.models_dir / "hypermodel_calibrator.joblib")
+            logger.info("  Isotonic calibrator loaded")
+
+            self.xgb_model = joblib.load(self.models_dir / "xgb_binary.joblib")
+            logger.info("  XGBoost base model loaded")
+
+            self.lgbm_model = joblib.load(self.models_dir / "lgbm_binary.joblib")
+            logger.info("  LightGBM base model loaded")
+
+            self.tfidf_lr = joblib.load(self.models_dir / "tfidf_lr_binary.joblib")
+            logger.info("  TF-IDF + LogisticRegression base model loaded")
+
+        if n_meta is not None and n_meta != len(self.BASE_MODEL_NAMES):
+            # Loud rather than silently wrong: a mismatch here means the artifacts on
+            # disk are not the 3-column ensemble this code stacks for.
+            raise ValueError(
+                f"Meta-learner expects {n_meta} features but this service stacks "
+                f"{len(self.BASE_MODEL_NAMES)} base models "
+                f"({', '.join(self.BASE_MODEL_NAMES)}). Model artifacts in "
+                f"{self.models_dir} do not match the expected ensemble topology."
+            )
+
+        self.model_loaded = True
+        logger.info(
+            f"URL HyperModel ready ({len(self.BASE_MODEL_NAMES)} base models "
+            f"+ meta-learner + isotonic calibration)"
+        )
+
+    def predict(self, url: str, threshold: float = 0.5) -> dict:
+        """
+        Run the full stacking pipeline on a single URL.
+
+        Returns dict with:
+          - url: input URL
+          - prediction: 'Malicious' or 'Benign'
+          - malicious_score: calibrated float [0, 1]
+          - raw_score: meta-learner score before isotonic calibration
+          - confidence: str percentage
+          - confidence_level: 'High', 'Medium', or 'Low'
+          - base_model_scores: dict of individual model scores
+          - top_signals: dict of most important feature values
+          - inference_time_ms: float
+        """
+        start = time.perf_counter()
+
+        # 1. Extract features
+        features = extract_url_features(url)
+        features_dict = extract_url_features_dict(url)
+
+        # Guard against a feature-extractor/scaler drift
+        if self.scaler is not None:
+            features = features[:self.scaler.n_features_in_]
+        X_feat = features.reshape(1, -1)
+
+        # 2. Scale features
+        X_scaled = self.scaler.transform(X_feat).astype(np.float32)
+
+        # 3. Base model P(malicious). TF-IDF/LR consumes the raw URL string,
+        #    not the engineered features.
+        #
+        # LightGBM was fitted on a bare numpy array, so it carries placeholder
+        # feature names (Column_0..65) and warns on every call about the array we
+        # pass. Column ORDER is what matters and is verified identical to
+        # FEATURE_NAMES, so the warning is noise - suppressed here rather than
+        # globally so real warnings still surface.
+        with warnings.catch_warnings():
+            warnings.filterwarnings(
+                "ignore", message="X does not have valid feature names"
+            )
+            p_xgb = float(self.xgb_model.predict_proba(X_scaled)[0, 1])
+            p_lgbm = float(self.lgbm_model.predict_proba(X_scaled)[0, 1])
+        p_tfidf = float(self.tfidf_lr.predict_proba([str(url)])[0, 1])
+
+        base_scores = {
+            "XGBoost": p_xgb,
+            "LightGBM": p_lgbm,
+            "TF-IDF + LR": p_tfidf,
+        }
+
+        # 4. Stack -> 5. meta-learner -> 6. isotonic calibration
+        meta_features = np.array([[p_xgb, p_lgbm, p_tfidf]], dtype=np.float32)
+        raw_score = float(self.meta_learner.predict_proba(meta_features)[0, 1])
+        malicious_score = float(self.calibrator.predict([raw_score])[0])
+
+        # Domain-level trust resolver, for residual edge cases like OAuth consent URLs
+        reg_domain = _get_registered_domain(url)
+        if reg_domain and reg_domain in TRUSTED_DOMAINS:
+            malicious_score = malicious_score * 0.15  # Dampen but don't hard-cap
+
+        # Determine verdict
+        prediction = "Malicious" if malicious_score > threshold else "Benign"
+        # Isotonic calibration can map to exactly 0.0/1.0, which would render as a
+        # flat 100.0% - a certainty no classifier has. Cap the reported confidence
+        # at 99.9 so the string never claims more than the model can support. The
+        # verdict and malicious_score are unaffected.
+        confidence_pct = min(99.9, abs(malicious_score - 0.5) * 200)
+
+        if confidence_pct >= 80:
+            confidence_level = "High"
+        elif confidence_pct >= 50:
+            confidence_level = "Medium"
+        else:
+            confidence_level = "Low"
+
+        # Top feature signals
+        important_features = [
+            "has_ip", "has_suspicious_tld", "phishing_keyword_count",
+            "brand_in_subdomain", "has_punycode", "is_shortened",
+            "url_length", "num_subdomains", "has_https", "domain_entropy",
+            "has_suspicious_ext", "has_hex_encoding", "brand_count",
+            "path_depth", "url_entropy",
+        ]
+        top_signals = {}
+        for feat in important_features:
+            if feat in features_dict:
+                val = features_dict[feat]
+                if val != 0:
+                    top_signals[feat] = round(float(val), 4)
+
+        elapsed = (time.perf_counter() - start) * 1000
+
+        return {
+            "url": url,
+            "prediction": prediction,
+            "malicious_score": round(malicious_score, 6),
+            "raw_score": round(raw_score, 6),
+            "confidence": f"{confidence_pct:.1f}%",
+            "confidence_level": confidence_level,
+            "threshold_used": threshold,
+            "base_model_scores": base_scores,
+            "top_signals": top_signals,
+            "inference_time_ms": round(elapsed, 2),
+        }
+
+    def get_model_info(self) -> dict:
+        """Return info about loaded models."""
+        return {
+            "model_type": "Stacking HyperModel (calibrated)",
+            "base_models": list(self.BASE_MODEL_NAMES),
+            "meta_learner": "LogisticRegression",
+            "calibration": "IsotonicRegression on out-of-fold meta scores",
+            "scaler_features": self.scaler.n_features_in_,
+            "meta_features": getattr(self.meta_learner, "n_features_in_", None),
+            "feature_categories": [
+                "Length/structural (7)",
+                "Character counts (14)",
+                "Character ratios (5)",
+                "Entropy/randomness (2)",
+                "Boolean indicators (10)",
+                "Domain/TLD signals (6)",
+                "Suspicious patterns (5)",
+                "Path & query (5)",
+            ],
+        }
+
+
+# Singleton accessor
+_url_service: URLModelService | None = None
+
+
+def get_url_service() -> URLModelService:
+    """Get or create the singleton URL model service."""
+    global _url_service
+    if _url_service is None:
+        _url_service = URLModelService()
+    return _url_service

urlmodel_info/hypermodel_calibrator.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a7cb24e5c2e73af5e40020fb00a989953c51cb7d6178d5f829b0216220268d6
+size 5735

urlmodel_info/hypermodel_meta.joblib

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:270f07caa538357f439a25dcc433d45f2c376cd47ca673173a58383f5ef28d39
-size 118639
+oid sha256:2e15aa8454216faee298ab6ec31dfd5e5fb51ba9a5046dd9b97fdb2badd38369
+size 879

urlmodel_info/lgbm_binary.joblib

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9c4c1f87ca80b2dd69abb25c568568dee29fcb529ebb4a73e6e58a16581e1c95
-size 366262
+oid sha256:4c18f6fc8832fb18c4b3def87bcd991bac582006e325802e95a7eb0aa0387d36
+size 2113830

urlmodel_info/scaler.joblib

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0340c41cb7fccee8a44f0099b9a5e61b81be5e74c09206ea8b54d9320c3f8aec
+oid sha256:81771f11487019d710177ee4720b2ace069a74c5ff12d2b1b86b01f6909fb153
 size 2167

urlmodel_info/tfidf_lr_binary.joblib

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9cd31b7d37e5b721f95a22483d9e45a28a7b7dc60ca1694cc9c265e797ba517b
-size 824640
+oid sha256:5684ee91ca2f39ae05af312a740a68353f1bd1b6629ac2588291c4d3a4bac0db
+size 1355680

urlmodel_info/xgb_binary.joblib

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9cd7da8abb1fe42878e4f8fdf677eddd8b11a7bf257f704f14582312d947ca6d
-size 470189
+oid sha256:9bcf0af9b9013d0d507d0929ebadd8be67eae58d54a547dbd58490b6200a449f
+size 2017153