feat(phiusiil): vendor the scoring path from the full application

The model was fitted on standardised inputs, so reproducing its
predictions needs the same preprocessing code and not merely the same
recorded numbers.

Vendoring rather than depending on the full application keeps the page
fetcher and its SSRF guard out of a service that must never open a socket,
and leaves this repository installable on its own.

phiusiil/__init__.py

@@ -0,0 +1,19 @@
+"""phiusiil -- the serving subset of the PhiUSIIL phishing URL classifier.
+
+One trained model, the fitted preprocessing state it needs, and nothing else. The module
+boundary rules that matter here:
+
+1. ``features/`` is pure: no network, no fitted state. It turns raw values into raw
+   values, emitting ``NaN`` for anything it cannot determine.
+2. ``preprocess/`` owns all fitted state. ``features/`` produces ``NaN``s; ``preprocess/``
+   decides what they become, reading every number from persisted state and computing none
+   of them at serving time.
+
+There is no fetch layer and no feature extractor for live URLs. This package scores a
+feature row that someone else produced.
+"""
+
+__version__ = "1.0.0"
+ARTIFACT_SCHEMA_VERSION = "v1"
+
+__all__ = ["ARTIFACT_SCHEMA_VERSION", "__version__"]

phiusiil/features/url_features.py

@@ -0,0 +1,751 @@
+"""URL-derived features, ported from the original notebook.
+
+PORT CONTRACT
+=============
+
+The 26 ``fill_*`` functions below are ported **verbatim**. Every regex, every threshold,
+every edge case and every quirk is preserved exactly. Only three changes are permitted:
+
+  1. Dropping the trailing demonstration statements each notebook cell appended.
+  2. Dropping ``print()`` calls.
+  3. Threading fitted statistics in as explicit keyword parameters instead of computing
+     them inline from the input frame.
+
+No improvements. No bug fixes. No renaming. Where a function looks wrong it is still
+ported as-is and the observation is recorded below rather than repaired.
+
+The severity is deliberate: these functions define what the training data *means*.
+Changing one silently changes the feature distribution the models were fitted on, and no
+metric would reveal it -- the numbers would simply be describing something else. The
+fidelity tests run these implementations and a frozen copy of the originals side by side
+over a 2,000-row sample and assert exact frame equality, with no tolerance.
+
+Change 3 is the one that matters for correctness. ``fill_tld_legitimate_prob`` originally
+computed its own skew, median/mean and per-TLD group means from whatever frame it was
+handed. At training time that frame is 112,323 rows; at serving time it would be a single
+row, so the same URL would get a different feature value depending on what it was batched
+with. Passing the statistics in makes the function a pure row-wise mapping, which is what
+makes single-row inference equal batch inference.
+
+PRESERVED DEFECTS
+=================
+
+``detect_advanced_obfuscation`` rule 4 tests whether the *reversed* URL matches
+``^[a-zA-Z0-9.\\-]+$``. A string matches that pattern exactly when its reversal does, so
+the rule fires for any URL consisting solely of letters, digits, dots and hyphens --
+including every bare domain with no scheme. This is almost certainly not what "reversed
+strings" was meant to detect. Preserved.
+
+``fill_has_obfuscation`` ends with ``.apply(lambda x: 1 if x == True else 0)``, which
+rewrites the whole column rather than only the filled cells. Preserved.
+
+The skew threshold here is ``abs(skew) > 1``; the numeric imputer uses ``abs(skew) > 3``.
+The two are genuinely different and both are preserved.
+"""
+
+from __future__ import annotations
+
+import base64
+import re
+from typing import Any
+from urllib.parse import urlparse
+
+import numpy as np
+import pandas as pd
+
+
+def scalar_float(value: Any) -> float:
+    """Coerce a pandas reduction result to a plain float.
+
+    Reductions such as ``.skew()`` and ``.median()`` are typed as a wide union covering
+    every dtype a Series might hold. On the numeric columns here the value is always a
+    float; this narrows it so arithmetic and comparisons type-check, and it is an identity
+    on the values that actually occur.
+    """
+    return float(value)
+
+
+# ---------------------------------------------------------------------------
+# Fit-time statistic
+# ---------------------------------------------------------------------------
+
+
+def calculate_char_prob(df: pd.DataFrame, url_col: str) -> dict[str, float]:
+    """Corpus-wide alphanumeric character frequency table.
+
+    This is a *fit-time* statistic. It is computed once over the training split and
+    thereafter passed into :func:`fill_url_char_prob` as data. It must never be recomputed
+    at serving time -- a single-row corpus would give every character in that one URL a
+    probability of roughly 1/len(url), which has nothing to do with the training
+    distribution the models learned against.
+    """
+    char_count: dict[str, int] = {}
+    total_chars = 0
+    for url in df[url_col].dropna():
+        for char in url.lower():
+            if char.isalnum():  # Only consider alphanumeric characters
+                char_count[char] = char_count.get(char, 0) + 1
+                total_chars += 1
+    return {char: count / total_chars for char, count in char_count.items()}
+
+
+# ---------------------------------------------------------------------------
+# The 26 fill_* functions
+# ---------------------------------------------------------------------------
+
+
+def fill_url_length(
+    data: pd.DataFrame, url_col: str = "URL", url_length_col: str = "URLLength"
+) -> pd.DataFrame:
+    data[url_length_col] = data.apply(
+        lambda row: len(str(row[url_col]))
+        if pd.isnull(row[url_length_col]) and pd.notnull(row[url_col])
+        else row[url_length_col],
+        axis=1,
+    )
+    return data
+
+
+def fill_domain(
+    data: pd.DataFrame, url_col: str = "URL", domain_col: str = "Domain"
+) -> pd.DataFrame:
+    def get_domain(url: Any) -> Any:
+        return urlparse(url).netloc if pd.notnull(url) else None
+
+    data[domain_col] = data.apply(
+        lambda row: get_domain(row[url_col])
+        if pd.isnull(row[domain_col]) and pd.notnull(row[url_col])
+        else row[domain_col],
+        axis=1,
+    )
+    return data
+
+
+def fill_domain_length(
+    data: pd.DataFrame, domain_col: str = "Domain", domain_length_col: str = "DomainLength"
+) -> pd.DataFrame:
+    data[domain_length_col] = data.apply(
+        lambda row: len(str(row[domain_col]))
+        if pd.isnull(row[domain_length_col]) and pd.notnull(row[domain_col])
+        else row[domain_length_col],
+        axis=1,
+    )
+    return data
+
+
+def fill_is_domain_ip(
+    data: pd.DataFrame, domain_col: str = "Domain", is_domain_ip_col: str = "IsDomainIP"
+) -> pd.DataFrame:
+    def is_ipaddress(domain: Any) -> bool:
+        if pd.notnull(domain):
+            ip_pattern = r"^(\d{1,3}\.){3}\d{1,3}$"
+            if re.match(ip_pattern, domain):
+                parts = domain.split(".")
+                return all(0 <= int(part) <= 255 for part in parts)
+        return False
+
+    data[is_domain_ip_col] = data.apply(
+        lambda row: (1 if is_ipaddress(row[domain_col]) else 0)
+        if pd.isnull(row[is_domain_ip_col]) and pd.notnull(row[domain_col])
+        else row[is_domain_ip_col],
+        axis=1,
+    )
+    return data
+
+
+def fill_tld(data: pd.DataFrame, domain_col: str = "Domain", tld_col: str = "TLD") -> pd.DataFrame:
+    def generate_tld(domain: Any) -> Any:
+        if pd.notnull(domain):
+            parts = domain.split(".")
+            if len(parts) > 1:
+                return parts[-1]
+        return np.nan
+
+    data[tld_col] = data.apply(
+        lambda row: generate_tld(row[domain_col]) if pd.isnull(row[tld_col]) else row[tld_col],
+        axis=1,
+    )
+    return data
+
+
+def fill_char_continuation_rate(
+    data: pd.DataFrame, url_col: str = "URL", char_rate_col: str = "CharContinuationRate"
+) -> pd.DataFrame:
+    def generate_char_continuation_rate(url: Any) -> Any:
+        if pd.notnull(url):
+            sequences = re.findall(r"[a-zA-Z0-9_]+", url)
+            total_sequence_length = sum(len(seq) for seq in sequences)
+            total_url_length = len(url)
+            return total_sequence_length / total_url_length if total_url_length > 0 else np.nan
+        return np.nan
+
+    data[char_rate_col] = data.apply(
+        lambda row: generate_char_continuation_rate(row[url_col])
+        if pd.isnull(row[char_rate_col])
+        else row[char_rate_col],
+        axis=1,
+    )
+    return data
+
+
+def fill_tld_legitimate_prob(
+    data: pd.DataFrame,
+    *,
+    tld_prob_mean: pd.Series,
+    global_fill_value: float,
+    tld_col: str = "TLD",
+    tld_prob_col: str = "TLDLegitimateProb",
+    is_domain_ip_col: str = "IsDomainIP",
+) -> pd.DataFrame:
+    """Fill ``TLDLegitimateProb``.
+
+    ``tld_prob_mean`` (per-TLD group means) and ``global_fill_value`` (the median if
+    ``abs(skew) > 1``, else the mean) are computed at fit time by
+    :func:`compute_tld_prob_statistics` and passed in. The original computed both inside
+    this function from its argument frame; the four-branch row rule below is otherwise
+    byte-identical to it.
+    """
+
+    def fill_tld_legit_prob(row: Any) -> Any:
+        if pd.isnull(row[tld_prob_col]):
+            if row[is_domain_ip_col] == 1:
+                return 0
+            if pd.notnull(row[tld_col]) and row[tld_col] in tld_prob_mean.index:
+                return tld_prob_mean[row[tld_col]]
+            if pd.isnull(row[tld_col]) and pd.isnull(row[is_domain_ip_col]):
+                return global_fill_value
+        return row[tld_prob_col]
+
+    data[tld_prob_col] = data.apply(fill_tld_legit_prob, axis=1)
+    data[tld_prob_col] = data[tld_prob_col].fillna(global_fill_value)
+
+    return data
+
+
+def compute_tld_prob_statistics(
+    data: pd.DataFrame,
+    tld_col: str = "TLD",
+    tld_prob_col: str = "TLDLegitimateProb",
+) -> tuple[pd.Series, float]:
+    """Fit-time half of :func:`fill_tld_legitimate_prob`, lifted out of it verbatim.
+
+    Note the threshold: ``abs(skew) > 1`` here, against ``abs(skew) > 3`` in the numeric
+    imputer. The two are genuinely different in the original and both are preserved.
+    """
+    skewness = scalar_float(data[tld_prob_col].skew())
+    if skewness > 1 or skewness < -1:
+        global_fill_value = scalar_float(data[tld_prob_col].median())
+    else:
+        global_fill_value = scalar_float(data[tld_prob_col].mean())
+
+    tld_prob_mean = data.groupby(tld_col)[tld_prob_col].mean()
+    return tld_prob_mean, global_fill_value
+
+
+def fill_url_char_prob(
+    data: pd.DataFrame,
+    url_col: str = "URL",
+    char_prob_col: str = "URLCharProb",
+    char_prob: dict[str, float] | None = None,
+) -> pd.DataFrame:
+    if char_prob is None:
+        raise ValueError("Character probabilities (`char_prob`) must be provided.")
+
+    def calculate_url_char_prob(url: Any) -> Any:
+        if pd.notnull(url):
+            total_prob = sum(char_prob.get(char, 0) for char in url.lower() if char.isalnum())
+            n = len(url)
+            return total_prob / n if n > 0 else np.nan
+        return np.nan
+
+    data[char_prob_col] = data.apply(
+        lambda row: calculate_url_char_prob(row[url_col])
+        if pd.isnull(row[char_prob_col])
+        else row[char_prob_col],
+        axis=1,
+    )
+    return data
+
+
+def fill_tld_length(
+    data: pd.DataFrame, tld_col: str = "TLD", tld_length_col: str = "TLDLength"
+) -> pd.DataFrame:
+    def calculate_tld_length(tld: Any) -> Any:
+        if pd.notnull(tld):
+            return len(str(tld))
+        return np.nan
+
+    data[tld_length_col] = data.apply(
+        lambda row: calculate_tld_length(row[tld_col])
+        if pd.isnull(row[tld_length_col])
+        else row[tld_length_col],
+        axis=1,
+    )
+    return data
+
+
+def fill_no_of_subdomains(
+    data: pd.DataFrame, domain_col: str = "Domain", subdomain_col: str = "NoOfSubDomain"
+) -> pd.DataFrame:
+    def calculate_no_of_subdomains(domain: Any) -> Any:
+        if pd.notnull(domain):
+            parts = domain.split(".")
+            return len(parts) - 2 if len(parts) > 2 else 0
+        return np.nan
+
+    data[subdomain_col] = data.apply(
+        lambda row: calculate_no_of_subdomains(row[domain_col])
+        if pd.isnull(row[subdomain_col])
+        else row[subdomain_col],
+        axis=1,
+    )
+    return data
+
+
+def fill_has_obfuscation(
+    data: pd.DataFrame, url_col: str = "URL", obfuscation_col: str = "HasObfuscation"
+) -> pd.DataFrame:
+    data[obfuscation_col] = data.apply(
+        lambda row: detect_advanced_obfuscation(row[url_col])
+        if pd.isnull(row[obfuscation_col]) and pd.notnull(row[url_col])
+        else row[obfuscation_col],
+        axis=1,
+    )
+    data[obfuscation_col] = data[obfuscation_col].apply(lambda x: 1 if x == True else 0)  # noqa: E712
+    return data
+
+
+def detect_advanced_obfuscation(url: Any) -> int:
+    """The five obfuscation rules, first match wins.
+
+    Rule 4 is preserved despite being near-certainly wrong: a string matches
+    ``^[a-zA-Z0-9.\\-]+$`` exactly when its reversal does, so reversing accomplishes
+    nothing and the rule fires for any scheme-less bare domain.
+    """
+    if pd.notnull(url):
+        if len(re.findall(r"[-_]", url)) > 3:  # Rule 1: Too many special characters
+            return 1
+        if re.search(r"[a-zA-Z]+\d+|\d+[a-zA-Z]+", url):  # Rule 2: Mixed alphanumeric patterns
+            return 1
+        if len(url) % 4 == 0 and re.match(r"^[A-Za-z0-9+/]*={0,2}$", url):  # Rule 3: Base64
+            try:
+                base64.b64decode(url, validate=True)
+                return 1
+            except Exception:
+                pass
+        reversed_url = url[::-1]  # Rule 4: Reversed strings
+        if re.match(r"^[a-zA-Z0-9.\-]+$", reversed_url):
+            return 1
+        if not re.search(r"[a-zA-Z]{3,}", url):  # Rule 5: Randomized strings
+            return 1
+        return 0  # No obfuscation detected
+    return 0  # Missing values treated as no obfuscation
+
+
+def fill_no_of_obfuscated_characters(
+    data: pd.DataFrame, url_col: str = "URL", obf_char_col: str = "NoOfObfuscatedChar"
+) -> pd.DataFrame:
+    def count_obfuscated_characters(url: Any) -> Any:
+        if pd.notnull(url):
+            hex_count = len(re.findall(r"%[0-9a-fA-F]{2}", url))
+            at_count = url.count("@")
+            return hex_count + at_count
+        return np.nan
+
+    data[obf_char_col] = data.apply(
+        lambda row: count_obfuscated_characters(row[url_col])
+        if pd.isnull(row[obf_char_col])
+        else row[obf_char_col],
+        axis=1,
+    )
+    return data
+
+
+def fill_obfuscation_ratio(
+    data: pd.DataFrame,
+    obf_char_col: str = "NoOfObfuscatedChar",
+    url_length_col: str = "URLLength",
+    obf_ratio_col: str = "ObfuscationRatio",
+) -> pd.DataFrame:
+    def calculate_obfuscation_ratio(no_of_obfchar: Any, url_length: Any) -> Any:
+        if pd.notnull(no_of_obfchar) and pd.notnull(url_length) and url_length > 0:
+            return no_of_obfchar / url_length
+        return np.nan
+
+    data[obf_ratio_col] = data.apply(
+        lambda row: calculate_obfuscation_ratio(row[obf_char_col], row[url_length_col])
+        if pd.isnull(row[obf_ratio_col])
+        else row[obf_ratio_col],
+        axis=1,
+    )
+    return data
+
+
+def fill_no_of_letters_in_url(
+    data: pd.DataFrame, url_col: str = "URL", letters_col: str = "NoOfLettersInURL"
+) -> pd.DataFrame:
+    def calculate_no_of_letters(url: Any) -> Any:
+        if pd.notnull(url):
+            return sum(c.isalpha() for c in url)
+        return np.nan
+
+    data[letters_col] = data.apply(
+        lambda row: calculate_no_of_letters(row[url_col])
+        if pd.isnull(row[letters_col])
+        else row[letters_col],
+        axis=1,
+    )
+    return data
+
+
+def fill_letter_ratio_in_url(
+    data: pd.DataFrame,
+    letters_col: str = "NoOfLettersInURL",
+    url_length_col: str = "URLLength",
+    ratio_col: str = "LetterRatioInURL",
+) -> pd.DataFrame:
+    def calculate_letter_ratio(no_of_letters: Any, url_length: Any) -> Any:
+        if pd.notnull(no_of_letters) and pd.notnull(url_length) and url_length > 0:
+            return no_of_letters / url_length
+        return np.nan
+
+    data[ratio_col] = data.apply(
+        lambda row: calculate_letter_ratio(row[letters_col], row[url_length_col])
+        if pd.isnull(row[ratio_col])
+        else row[ratio_col],
+        axis=1,
+    )
+    return data
+
+
+def fill_no_of_digits_in_url(
+    data: pd.DataFrame, url_col: str = "URL", digits_col: str = "NoOfDegitsInURL"
+) -> pd.DataFrame:
+    def calculate_no_of_digits(url: Any) -> Any:
+        if pd.notnull(url):
+            return sum(c.isdigit() for c in url)
+        return np.nan
+
+    data[digits_col] = data.apply(
+        lambda row: calculate_no_of_digits(row[url_col])
+        if pd.isnull(row[digits_col])
+        else row[digits_col],
+        axis=1,
+    )
+    return data
+
+
+def fill_digits_ratio_in_url(
+    data: pd.DataFrame, url_col: str = "URL", ratio_col: str = "DegitRatioInURL"
+) -> pd.DataFrame:
+    def calculate_digits_ratio(url: Any) -> Any:
+        if pd.notnull(url) and len(url) > 0:
+            return sum(c.isdigit() for c in url) / len(url)
+        return np.nan
+
+    data[ratio_col] = data.apply(
+        lambda row: calculate_digits_ratio(row[url_col])
+        if pd.isnull(row[ratio_col])
+        else row[ratio_col],
+        axis=1,
+    )
+    return data
+
+
+def fill_no_of_equals_in_url(
+    data: pd.DataFrame, url_col: str = "URL", equals_col: str = "NoOfEqualsInURL"
+) -> pd.DataFrame:
+    def calculate_no_of_equals(url: Any) -> Any:
+        if pd.notnull(url):
+            return sum(c == "=" for c in url)
+        return np.nan
+
+    data[equals_col] = data.apply(
+        lambda row: calculate_no_of_equals(row[url_col])
+        if pd.isnull(row[equals_col])
+        else row[equals_col],
+        axis=1,
+    )
+    return data
+
+
+def fill_no_of_qmark_in_url(
+    data: pd.DataFrame, url_col: str = "URL", qmark_col: str = "NoOfQMarkInURL"
+) -> pd.DataFrame:
+    def calculate_no_of_qmark(url: Any) -> Any:
+        if pd.notnull(url):
+            return sum(c == "?" for c in url)
+        return np.nan
+
+    data[qmark_col] = data.apply(
+        lambda row: calculate_no_of_qmark(row[url_col])
+        if pd.isnull(row[qmark_col])
+        else row[qmark_col],
+        axis=1,
+    )
+    return data
+
+
+def fill_no_of_ampersand_in_url(
+    data: pd.DataFrame, url_col: str = "URL", ampersand_col: str = "NoOfAmpersandInURL"
+) -> pd.DataFrame:
+    def calculate_no_of_ampersand(url: Any) -> Any:
+        if pd.notnull(url):
+            return sum(c == "&" for c in url)
+        return np.nan
+
+    data[ampersand_col] = data.apply(
+        lambda row: calculate_no_of_ampersand(row[url_col])
+        if pd.isnull(row[ampersand_col])
+        else row[ampersand_col],
+        axis=1,
+    )
+    return data
+
+
+def fill_no_of_special_chars_in_url(
+    data: pd.DataFrame,
+    url_col: str = "URL",
+    special_chars_col: str = "NoOfOtherSpecialCharsInURL",
+) -> pd.DataFrame:
+    def calculate_no_of_specials(url: Any) -> Any:
+        if pd.notnull(url):
+            return sum(c.lower() not in "0123456789abcdefghijklmnopqrstuvwxyz" for c in url)
+        return np.nan
+
+    data[special_chars_col] = data.apply(
+        lambda row: calculate_no_of_specials(row[url_col])
+        if pd.isnull(row[special_chars_col])
+        else row[special_chars_col],
+        axis=1,
+    )
+    return data
+
+
+def fill_specials_ratio_in_url(
+    data: pd.DataFrame, url_col: str = "URL", ratio_col: str = "SpacialCharRatioInURL"
+) -> pd.DataFrame:
+    def calculate_specials_ratio(url: Any) -> Any:
+        if pd.notnull(url) and len(url) > 0:
+            return (
+                sum(c.lower() not in "0123456789abcdefghijklmnopqrstuvwxyz" for c in url)
+                / len(url)
+            )
+        return np.nan
+
+    data[ratio_col] = data.apply(
+        lambda row: calculate_specials_ratio(row[url_col])
+        if pd.isnull(row[ratio_col])
+        else row[ratio_col],
+        axis=1,
+    )
+    return data
+
+
+def fill_is_https(
+    data: pd.DataFrame, url_col: str = "URL", https_col: str = "IsHTTPS"
+) -> pd.DataFrame:
+    def calculate_ishttps(url: Any) -> Any:
+        if pd.notnull(url):
+            return 1 if "https://" in url else 0
+        return np.nan
+
+    data[https_col] = data.apply(
+        lambda row: calculate_ishttps(row[url_col])
+        if pd.isnull(row[https_col])
+        else row[https_col],
+        axis=1,
+    )
+    return data
+
+
+def fill_has_title(
+    data: pd.DataFrame, title_col: str = "Title", has_title_col: str = "HasTitle"
+) -> pd.DataFrame:
+    data[has_title_col] = data.apply(
+        lambda row: row[has_title_col]
+        if not pd.isnull(row[has_title_col])
+        else (1 if not pd.isnull(row[title_col]) else row[has_title_col]),
+        axis=1,
+    )
+    return data
+
+
+def fill_domain_title_match_score(
+    data: pd.DataFrame,
+    title_col: str = "Title",
+    domain_col: str = "Domain",
+    score_col: str = "DomainTitleMatchScore",
+) -> pd.DataFrame:
+    data[score_col] = data.apply(
+        lambda row: calculate_domain_title_match_score(row[title_col], row[domain_col])
+        if pd.isnull(row[score_col])
+        and pd.notnull(row[title_col])
+        and pd.notnull(row[domain_col])
+        else row[score_col],
+        axis=1,
+    )
+    return data
+
+
+def calculate_domain_title_match_score(title: Any, domain: Any) -> float:
+    tSet = title.split(" ") if pd.notnull(title) else []
+    txtDomain = domain.split(".")[:-1] if pd.notnull(domain) else []
+    txtDomain = [i for i in txtDomain if i != "www"]
+    txtDomain = ".".join(txtDomain)
+
+    score = 0.0
+    baseScore = 100 / len(txtDomain) if len(txtDomain) > 0 else 0
+
+    for element in tSet:
+        if element in txtDomain:
+            n = len(element)
+            score += baseScore * n
+            txtDomain = txtDomain.replace(element, "")
+            if score > 99.9:
+                score = 100
+    return score
+
+
+def fill_url_title_match_score(
+    data: pd.DataFrame,
+    title_col: str = "Title",
+    url_col: str = "URL",
+    score_col: str = "URLTitleMatchScore",
+) -> pd.DataFrame:
+    data[score_col] = data.apply(
+        lambda row: calculate_url_title_match_score(row[title_col], row[url_col])
+        if pd.isnull(row[score_col])
+        else row[score_col],
+        axis=1,
+    )
+    return data
+
+
+def calculate_url_title_match_score(title: Any, url: Any) -> Any:
+    if pd.notnull(title) and pd.notnull(url):
+        tSet = title.split(" ")
+        txtURL = urlparse(url).netloc.split(".")[:-1] + [
+            i for i in urlparse(url).path.split("/") if i != ""
+        ]
+        txtURL = [i for i in txtURL if i != "www"]
+        txtURL = ".".join(txtURL)
+
+        score = 0.0
+        baseScore = 100 / len(txtURL) if len(txtURL) > 0 else 0
+
+        for element in tSet:
+            if element in txtURL:
+                n = len(element)
+                score += baseScore * n
+                txtURL = txtURL.replace(element, "")
+                if score > 99.9:
+                    score = 100
+        return score
+    return np.nan
+
+
+# ---------------------------------------------------------------------------
+# Row-level twins
+# ---------------------------------------------------------------------------
+#
+# These are a convenience for single-URL inference, which does not need a DataFrame apply
+# to compute one value. They are NOT a second implementation: the DataFrame functions above
+# remain the authority, and a property test asserts each twin agrees with its DataFrame
+# counterpart on a one-row frame across a 5,000-URL sample. If the two ever disagree, the
+# twin is what is wrong.
+
+
+def url_length(url: str | None) -> float | None:
+    return len(str(url)) if url is not None and pd.notnull(url) else None
+
+
+def domain_of(url: str | None) -> str | None:
+    return urlparse(url).netloc if url is not None and pd.notnull(url) else None
+
+
+def domain_length(domain: str | None) -> float | None:
+    return len(str(domain)) if domain is not None and pd.notnull(domain) else None
+
+
+def is_domain_ip(domain: str | None) -> int | None:
+    if domain is None or not pd.notnull(domain):
+        return None
+    if re.match(r"^(\d{1,3}\.){3}\d{1,3}$", domain):
+        return 1 if all(0 <= int(p) <= 255 for p in domain.split(".")) else 0
+    return 0
+
+
+def tld_of(domain: str | None) -> str | None:
+    if domain is not None and pd.notnull(domain):
+        parts = domain.split(".")
+        if len(parts) > 1:
+            return parts[-1]
+    return None
+
+
+def char_continuation_rate(url: str | None) -> float | None:
+    if url is None or not pd.notnull(url):
+        return None
+    total = sum(len(s) for s in re.findall(r"[a-zA-Z0-9_]+", url))
+    return total / len(url) if len(url) > 0 else None
+
+
+def url_char_prob(url: str | None, char_prob: dict[str, float]) -> float | None:
+    if url is None or not pd.notnull(url):
+        return None
+    total = sum(char_prob.get(c, 0) for c in url.lower() if c.isalnum())
+    return total / len(url) if len(url) > 0 else None
+
+
+def tld_length(tld: str | None) -> float | None:
+    return len(str(tld)) if tld is not None and pd.notnull(tld) else None
+
+
+def no_of_subdomains(domain: str | None) -> float | None:
+    if domain is None or not pd.notnull(domain):
+        return None
+    parts = domain.split(".")
+    return len(parts) - 2 if len(parts) > 2 else 0
+
+
+def has_obfuscation(url: str | None) -> int:
+    return detect_advanced_obfuscation(url)
+
+
+def no_of_obfuscated_chars(url: str | None) -> float | None:
+    if url is None or not pd.notnull(url):
+        return None
+    return len(re.findall(r"%[0-9a-fA-F]{2}", url)) + url.count("@")
+
+
+def no_of_letters(url: str | None) -> float | None:
+    return sum(c.isalpha() for c in url) if url is not None and pd.notnull(url) else None
+
+
+def no_of_digits(url: str | None) -> float | None:
+    return sum(c.isdigit() for c in url) if url is not None and pd.notnull(url) else None
+
+
+def no_of_equals(url: str | None) -> float | None:
+    return sum(c == "=" for c in url) if url is not None and pd.notnull(url) else None
+
+
+def no_of_qmark(url: str | None) -> float | None:
+    return sum(c == "?" for c in url) if url is not None and pd.notnull(url) else None
+
+
+def no_of_ampersand(url: str | None) -> float | None:
+    return sum(c == "&" for c in url) if url is not None and pd.notnull(url) else None
+
+
+def no_of_special_chars(url: str | None) -> float | None:
+    if url is None or not pd.notnull(url):
+        return None
+    return sum(c.lower() not in "0123456789abcdefghijklmnopqrstuvwxyz" for c in url)
+
+
+def is_https(url: str | None) -> int | None:
+    if url is None or not pd.notnull(url):
+        return None
+    return 1 if "https://" in url else 0

phiusiil/models/base.py

@@ -0,0 +1,56 @@
+"""The interface all four models present.
+
+The application treats a from-scratch implementation and its scikit-learn counterpart
+identically, so neither gets special-cased in the interface layer.
+
+``score`` is always P(class 0) -- the probability of *phishing*. Class 0 is the positive
+class throughout this codebase, and a model that returned P(class 1) here would invert
+every threshold, every ranking and every confidence display without failing a single type
+check.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+import numpy as np
+
+from phiusiil.schema import PHISHING_LABEL
+
+
+class Classifier(ABC):
+    """Fit on a float32 matrix; predict labels and phishing scores."""
+
+    #: Shown in the interface. The from-scratch pair is labelled as an educational
+    #: reimplementation rather than presented as an independent fourth opinion.
+    name: str = "classifier"
+    family: str = "unknown"
+    is_scratch: bool = False
+
+    @abstractmethod
+    def fit(self, X: np.ndarray, y: np.ndarray) -> Classifier: ...
+
+    @abstractmethod
+    def predict(self, X: np.ndarray) -> np.ndarray: ...
+
+    @abstractmethod
+    def score_phishing(self, X: np.ndarray) -> np.ndarray:
+        """P(class 0) per row, in [0, 1]."""
+
+    def predict_at_threshold(self, X: np.ndarray, threshold: float) -> np.ndarray:
+        """Label as phishing when the phishing score reaches the threshold."""
+        scores = self.score_phishing(X)
+        return np.where(scores >= threshold, PHISHING_LABEL, 1 - PHISHING_LABEL).astype(np.int64)
+
+
+def as_float32(X: np.ndarray) -> np.ndarray:
+    """Pin the dtype at every model boundary.
+
+    The transform emits float32 and the distance kernel expects it. Letting a float64
+    matrix through would silently change the arithmetic between training and serving,
+    which is the same class of skew the preprocessing split exists to prevent.
+    """
+    array = np.asarray(X, dtype=np.float32)
+    if array.ndim == 1:
+        array = array.reshape(1, -1)
+    return array

phiusiil/models/knn_scratch.py

@@ -0,0 +1,183 @@
+"""From-scratch k-nearest neighbours, vectorised.
+
+The original iterated ``X_test.iterrows()`` in pure Python and took 5m43s to classify the
+validation split. Same algorithm, same answers, expressed as matrix arithmetic.
+
+THE TIE-BREAKING HAZARD
+=======================
+
+This is the one subtlety, and it is easy to vectorise straight past.
+
+The original selected neighbours with ``np.argsort(distances)[:k]`` and voted with
+``Counter(...).most_common(1)``. When two classes tie on count, ``most_common`` returns
+whichever was inserted into the counter first -- which, given argsort's output, is the
+class of the nearest neighbour among the k.
+
+``np.argpartition`` is the fast way to take the k smallest, and it does *not* order the k
+it returns. Feeding its output straight into the vote silently changes which class wins
+every tie, and with an even k ties are common rather than exotic. So the k selected
+indices are re-sorted by distance before voting. Without that line the vectorised model
+disagrees with the original on a subset of rows and no test that only checks accuracy
+would notice.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+
+import numpy as np
+
+from phiusiil.models.base import Classifier, as_float32
+from phiusiil.schema import PHISHING_LABEL
+
+#: Query rows per chunk. Bounds the distance matrix at chunk x n_reference floats, which
+#: at 512 x 10,000 float32 is about 20 MB -- large enough to keep BLAS busy, small enough
+#: that it never dominates the container's memory budget.
+CHUNK_SIZE = 512
+
+
+class ScratchKNN(Classifier):
+    name = "KNN (from scratch)"
+    family = "knn"
+    is_scratch = True
+
+    def __init__(self, k: int = 20, metric: str = "euclidean", p: int = 1) -> None:
+        if not isinstance(k, int) or k < 1:
+            raise ValueError("Invalid neighbor count. k must be an integer greater than 0.")
+        if metric not in {"manhattan", "euclidean", "minkowski"}:
+            raise ValueError(
+                "Invalid distance metric. Valid metric: 'euclidean', 'manhattan', or 'minkowski'."
+            )
+        if not isinstance(p, int) or p < 1:
+            raise ValueError("Invalid minkowski distance variable. p must be an integer > 0.")
+
+        self.k = k
+        self.metric = metric
+        self.p = p
+        self.X_train: np.ndarray | None = None
+        self.y_train: np.ndarray | None = None
+
+    def fit(self, X: np.ndarray, y: np.ndarray) -> ScratchKNN:
+        self.X_train = as_float32(X)
+        self.y_train = np.asarray(y, dtype=np.int64)
+        if len(self.X_train) != len(self.y_train):
+            raise ValueError(
+                f"reference set and labels disagree: {len(self.X_train)} rows, "
+                f"{len(self.y_train)} labels"
+            )
+        return self
+
+    def _distances(self, Q: np.ndarray) -> np.ndarray:
+        assert self.X_train is not None
+        R = self.X_train
+
+        if self.metric == "euclidean":
+            # d^2 = |q|^2 - 2 q.r + |r|^2, which turns the whole distance computation into
+            # one matrix product. Small negatives appear from cancellation when a query
+            # nearly coincides with a reference point, so the result is clipped at zero
+            # before the square root.
+            q_sq = np.einsum("ij,ij->i", Q, Q)[:, None]
+            r_sq = np.einsum("ij,ij->i", R, R)[None, :]
+            d2 = q_sq - 2.0 * (Q @ R.T) + r_sq
+            np.maximum(d2, 0.0, out=d2)
+            return np.sqrt(d2)
+
+        if self.metric == "manhattan":
+            return np.abs(Q[:, None, :] - R[None, :, :]).sum(axis=2)
+
+        diff = np.abs(Q[:, None, :] - R[None, :, :]) ** self.p
+        return diff.sum(axis=2) ** (1.0 / self.p)
+
+    def _neighbours(self, Q: np.ndarray) -> np.ndarray:
+        """Indices of the k nearest reference rows, ordered nearest-first."""
+        assert self.X_train is not None
+        n_ref = len(self.X_train)
+        k = min(self.k, n_ref)
+
+        distances = self._distances(Q)
+
+        if k < n_ref:
+            candidates = np.argpartition(distances, kth=k - 1, axis=1)[:, :k]
+        else:
+            candidates = np.tile(np.arange(n_ref), (len(Q), 1))
+
+        # Re-sort the selected k by distance. argpartition leaves them unordered, and the
+        # vote's tie-break depends on nearest-first order.
+        rows = np.arange(len(Q))[:, None]
+        order = np.argsort(distances[rows, candidates], axis=1, kind="stable")
+        return candidates[rows, order]
+
+    def _vote(self, neighbour_idx: np.ndarray) -> np.ndarray:
+        assert self.y_train is not None
+        labels = self.y_train[neighbour_idx]
+        out = np.empty(len(labels), dtype=np.int64)
+        for i, row in enumerate(labels):
+            # Counter preserves insertion order, and the row is nearest-first, so a tie
+            # resolves to the class of the nearest neighbour -- matching the original.
+            out[i] = Counter(row.tolist()).most_common(1)[0][0]
+        return out
+
+    def predict(self, X: np.ndarray) -> np.ndarray:
+        if self.X_train is None or self.y_train is None:
+            raise RuntimeError("ScratchKNN.predict called before fit()")
+
+        Q = as_float32(X)
+        predictions = np.empty(len(Q), dtype=np.int64)
+        for start in range(0, len(Q), CHUNK_SIZE):
+            chunk = Q[start : start + CHUNK_SIZE]
+            predictions[start : start + len(chunk)] = self._vote(self._neighbours(chunk))
+        return predictions
+
+    def score_phishing(self, X: np.ndarray) -> np.ndarray:
+        """Share of the k neighbours that are phishing.
+
+        A vote fraction, not a calibrated probability. It is monotone in the evidence and
+        that is all the threshold slider needs; presenting it as a calibrated posterior
+        would overstate what k=20 nearest neighbours can tell you.
+        """
+        if self.X_train is None or self.y_train is None:
+            raise RuntimeError("ScratchKNN.score_phishing called before fit()")
+
+        Q = as_float32(X)
+        scores = np.empty(len(Q), dtype=np.float64)
+        for start in range(0, len(Q), CHUNK_SIZE):
+            chunk = Q[start : start + CHUNK_SIZE]
+            neighbours = self._neighbours(chunk)
+            labels = self.y_train[neighbours]
+            scores[start : start + len(chunk)] = (labels == PHISHING_LABEL).mean(axis=1)
+        return scores
+
+
+class NaiveScratchKNN(ScratchKNN):
+    """The original row-at-a-time loop, kept as the reference for the vectorised version.
+
+    Used only by the equivalence test. It is far too slow to serve with -- which is the
+    entire reason the vectorised implementation exists -- but a fast implementation with
+    no slow one to check it against is just an assertion.
+    """
+
+    name = "KNN (from scratch, naive loop)"
+
+    def predict(self, X: np.ndarray) -> np.ndarray:
+        if self.X_train is None or self.y_train is None:
+            raise RuntimeError("NaiveScratchKNN.predict called before fit()")
+
+        Q = as_float32(X)
+        predictions = []
+        for row in Q:
+            match self.metric:
+                case "euclidean":
+                    distances = np.sqrt(np.sum(np.square(row - self.X_train), axis=1))
+                case "manhattan":
+                    distances = np.sum(np.abs(row - self.X_train), axis=1)
+                case "minkowski":
+                    distances = np.sum(np.abs(row - self.X_train) ** self.p, axis=1) ** (
+                        1 / self.p
+                    )
+                case _:
+                    raise ValueError("Invalid distance metric.")
+
+            neighbours = np.argsort(distances, kind="stable")[: self.k]
+            classes = Counter(self.y_train[neighbours].tolist())
+            predictions.append(classes.most_common(1)[0][0])
+        return np.asarray(predictions, dtype=np.int64)

phiusiil/preprocess/scaler.py

@@ -0,0 +1,97 @@
+"""Standardisation over a named subset of columns.
+
+Scales exactly the 30 numeric columns and passes the 19 binary indicators through
+bit-identically.
+
+WHY THE SUBSET IS DELIBERATE
+============================
+
+This started as an accident. In the original, the clipping helper and the standardisation
+path both closed over a notebook-level ``numerical_columns`` global, so the 19 binary
+columns were never in scope for either. Nobody decided this; it fell out of variable
+scoping.
+
+It is now a frozen decision, for two reasons:
+
+1. The 19 columns are already 0/1 indicators. Z-scoring a Bernoulli indicator maps it to
+   two points at -p/sqrt(p(1-p)) and (1-p)/sqrt(p(1-p)); it changes the scale of an
+   already-comparable quantity, and for a rare indicator it inflates the minority level's
+   magnitude substantially.
+2. Changing it would change every recorded metric. Reporting as-recorded beside
+   as-corrected is only meaningful if the two differ by one identified cause -- the
+   fit/transform fix -- rather than by a bundle of simultaneous changes.
+
+THE HONEST CAVEAT
+=================
+
+A Euclidean distance over 30 z-scored dimensions plus 19 raw 0/1 dimensions is not a
+principled metric. The unit-variance dimensions and the at-most-0.25-variance dimensions
+contribute on different scales, and the effective weighting is an accident of where the
+fix stopped rather than a modelling choice. KNN is the model that cares; Gaussian NB
+treats each dimension independently and its decision rule is invariant to per-feature
+affine rescaling. An ablation that scales all 49 is reported as a comparison, not adopted
+as the default -- changing it is a product decision with a metric-invalidation cost, not a
+silent bugfix.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+
+from phiusiil.preprocess.stats import ScalerParams
+
+
+class SubsetStandardScaler:
+    """Fit and apply a per-column affine map over a fixed column subset.
+
+    Note what this class does not have: a ``fit_transform``. Its absence is the point.
+    Serving-time code receives :class:`ScalerParams` -- plain numbers -- and can only
+    apply them.
+    """
+
+    def __init__(self, columns: tuple[str, ...]) -> None:
+        self.columns = tuple(columns)
+
+    def fit(self, X: pd.DataFrame) -> ScalerParams:
+        missing = [c for c in self.columns if c not in X.columns]
+        if missing:
+            raise KeyError(f"cannot fit scaler, columns absent from frame: {sorted(missing)}")
+
+        block = X[list(self.columns)].astype(np.float64)
+        mean_ = block.mean(axis=0).to_numpy()
+
+        # Population standard deviation (ddof=0), matching StandardScaler. Using the
+        # sample standard deviation here would shift every scaled value by a factor of
+        # sqrt(n/(n-1)) relative to the reference implementation.
+        scale_ = block.std(axis=0, ddof=0).to_numpy()
+
+        # A constant column has zero variance and would divide by zero. Mapping its scale
+        # to 1.0 turns it into a pure mean-shift, which is what StandardScaler does too.
+        scale_ = np.where(scale_ == 0.0, 1.0, scale_)
+
+        return ScalerParams(
+            columns=self.columns,
+            mean_=tuple(float(v) for v in mean_),
+            scale_=tuple(float(v) for v in scale_),
+        )
+
+    @staticmethod
+    def apply(X: pd.DataFrame, params: ScalerParams) -> pd.DataFrame:
+        """Apply a fitted affine map in place on a copy.
+
+        Elementwise and row-local by construction: the value written for row i depends
+        only on row i and on the persisted constants. That is what makes transforming one
+        row and transforming a batch produce bitwise-identical output.
+        """
+        missing = [c for c in params.columns if c not in X.columns]
+        if missing:
+            raise KeyError(f"cannot scale, columns absent from frame: {sorted(missing)}")
+
+        cols = list(params.columns)
+        mean_ = np.asarray(params.mean_, dtype=np.float64)
+        scale_ = np.asarray(params.scale_, dtype=np.float64)
+
+        X = X.copy()
+        X[cols] = (X[cols].astype(np.float64).to_numpy() - mean_) / scale_
+        return X

phiusiil/preprocess/stats.py

@@ -0,0 +1,177 @@
+"""Everything learned at fit time, in one immutable record.
+
+The governing rule of this package: **transform() may not compute any statistic.**
+
+Operationally, inside ``transform`` there is no call to ``.mean()``, ``.median()``,
+``.mode()``, ``.std()``, ``.skew()``, ``.quantile()``, ``.value_counts()``,
+``.groupby(...).agg(...)``, ``fit()`` or ``fit_transform()`` on the input frame. Anything
+depending on more than the current row is a fitted statistic: computed once in ``fit()``,
+stored here, serialized, and read back at serving time.
+
+This is not a style preference. The original recomputed every statistic from whatever
+batch it was handed, and standardized the training and validation splits by their own
+separate means -- so the scaling that produced its headline numbers does not exist at
+serving time and cannot be reconstructed, because there is no batch to take a mean over
+when a user pastes one URL. Making the statistics data rather than control flow is the
+precondition for the service existing at all.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Literal
+
+FillMethod = Literal["mean", "median"]
+NbDropReason = Literal["empty_contingency_cell", "zero_std"]
+FallbackSource = Literal["numeric_mean", "numeric_median", "categorical_mode"]
+
+
+@dataclass(frozen=True)
+class NumericFill:
+    """Imputation value for one numeric column.
+
+    ``skew`` is recorded even though ``transform`` never reads it, so a reader can audit
+    which branch was taken without re-running training.
+    """
+
+    column: str
+    method: FillMethod  # median iff abs(skew) > 3
+    value: float
+    skew: float
+
+
+@dataclass(frozen=True)
+class CategoricalFillStep:
+    """One step of the ordered categorical cascade.
+
+    ``groupby_cols`` is a snapshot of the group key *at this step*, persisted explicitly
+    rather than reconstructed as "the first k-1 filled columns plus HasObfuscation". If
+    the cascade is ever edited, the artifact still describes what actually ran.
+    """
+
+    column: str
+    groupby_cols: tuple[str, ...]
+    modes: dict[tuple[float, ...], float]
+    global_mode: float
+
+
+@dataclass(frozen=True)
+class ClipBound:
+    column: str
+    lower: float  # 1st percentile of the training split
+    upper: float  # 99th percentile of the training split
+
+
+@dataclass(frozen=True)
+class ScalerParams:
+    """An affine map, stored as plain numbers.
+
+    Deliberately not a pickled StandardScaler. A pickled estimator carries a
+    ``fit_transform`` method, and the entire class of defect this package exists to
+    eliminate is someone calling it at serving time. Numbers cannot be re-fitted.
+    """
+
+    columns: tuple[str, ...]  # exactly the 30 numeric columns, ordered
+    mean_: tuple[float, ...]
+    scale_: tuple[float, ...]  # zeros replaced by 1.0 so transform cannot divide by zero
+
+    def __post_init__(self) -> None:
+        if not (len(self.columns) == len(self.mean_) == len(self.scale_)):
+            raise ValueError(
+                f"scaler arity mismatch: {len(self.columns)} columns, "
+                f"{len(self.mean_)} means, {len(self.scale_)} scales"
+            )
+        if any(s == 0.0 for s in self.scale_):
+            raise ValueError("scale_ contains a zero; constant columns must be mapped to 1.0")
+
+
+@dataclass(frozen=True)
+class HtmlFallback:
+    """The value a page-derived feature takes when the fetch failed or the feature was
+    demoted.
+
+    Derived, not separately computed: each entry is exactly the value this column's own
+    fill rule already produces. It is lifted into its own record purely so the interface
+    can *display* the fallback and where it came from, rather than silently applying it.
+    A mismatch between this table and the fills it was derived from is a bug, and a test
+    asserts they agree.
+    """
+
+    column: str
+    value: float
+    source: FallbackSource
+
+
+@dataclass(frozen=True)
+class FittedStats:
+    """The complete serialized state of a fitted preprocessor."""
+
+    # --- URL character model -------------------------------------------------
+    char_prob: dict[str, float]
+
+    # --- TLD legitimacy ------------------------------------------------------
+    tld_prob_mean: dict[str, float]
+    tld_prob_global_fill: float
+    tld_skew: float
+    tld_fill_method: FillMethod  # median iff abs(tld_skew) > 1 -- note: 1, not 3
+
+    # --- numeric imputation --------------------------------------------------
+    numeric_fill: tuple[NumericFill, ...]
+
+    # --- categorical cascade. ORDER IS SEMANTIC ------------------------------
+    categorical_cascade: tuple[CategoricalFillStep, ...]
+
+    # --- outlier clipping ----------------------------------------------------
+    clip_bounds: tuple[ClipBound, ...]
+
+    # --- scaling -------------------------------------------------------------
+    scaler: ScalerParams
+
+    # --- Naive-Bayes column drop, fitted here for locality -------------------
+    nb_drop: tuple[str, ...]
+    nb_drop_reasons: dict[str, NbDropReason]
+
+    # --- page-feature fallbacks ---------------------------------------------
+    html_fallbacks: tuple[HtmlFallback, ...]
+
+    # --- frozen column lists, so nothing closes over a module global ---------
+    feature_order: tuple[str, ...]
+    numerical_columns: tuple[str, ...]
+    continuous_columns: tuple[str, ...]
+    discrete_columns: tuple[str, ...]
+    categorical_columns_filtered: tuple[str, ...]
+
+    # --- provenance ----------------------------------------------------------
+    n_train_rows: int = 0
+    demoted_features: tuple[str, ...] = field(default_factory=tuple)
+
+    def __post_init__(self) -> None:
+        if len(self.feature_order) != 49:
+            raise ValueError(f"feature_order must have 49 entries, got {len(self.feature_order)}")
+        if len(self.numerical_columns) != 30:
+            raise ValueError(
+                f"numerical_columns must have 30 entries, got {len(self.numerical_columns)}"
+            )
+        if len(self.categorical_columns_filtered) != 19:
+            raise ValueError(
+                "categorical_columns_filtered must have 19 entries, got "
+                f"{len(self.categorical_columns_filtered)}"
+            )
+        if len(self.categorical_cascade) != 18:
+            raise ValueError(
+                f"categorical_cascade must have 18 steps, got {len(self.categorical_cascade)}"
+            )
+        if tuple(self.scaler.columns) != tuple(self.numerical_columns):
+            raise ValueError(
+                "scaler.columns must be exactly numerical_columns, in the same order. "
+                "A mismatch here means the matrix was scaled on a different column set "
+                "than the one recorded, which silently changes every distance."
+            )
+
+    @property
+    def numeric_fill_map(self) -> dict[str, float]:
+        return {f.column: f.value for f in self.numeric_fill}
+
+    @property
+    def html_fallback_map(self) -> dict[str, float]:
+        return {f.column: f.value for f in self.html_fallbacks}

phiusiil/preprocess/transformer.py

@@ -0,0 +1,489 @@
+"""The fit/transform split.
+
+``fit`` learns; ``transform`` only reads. There is no ``transform_single``, no
+``predict_one`` shortcut, and no "skip clipping for a single row" fast path. Every such
+special case is a place where the serving path can drift from the training path, and drift
+of exactly that kind is the defect this module exists to eliminate.
+
+The proof obligation is a test, not a comment: transforming n rows individually must equal
+transforming them as one batch, bitwise on float32. Any tolerance-based comparison would
+pass even if a statistic were being recomputed from a batch that happens to resemble the
+training set. Exact equality can only hold if transform reads its numbers from FittedStats.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+from phiusiil import schema
+from phiusiil.features import url_features as uf
+from phiusiil.preprocess.scaler import SubsetStandardScaler
+from phiusiil.preprocess.stats import (
+    CategoricalFillStep,
+    ClipBound,
+    FallbackSource,
+    FillMethod,
+    FittedStats,
+    HtmlFallback,
+    NbDropReason,
+    NumericFill,
+)
+
+
+class FeatureContractError(RuntimeError):
+    """A feature column is absent at reindex time.
+
+    Distinct from a missing *value*, which is normal and is imputed. An absent column
+    means the extractor's contract broke, and failing loudly is correct.
+    """
+
+
+class NotFittedError(RuntimeError):
+    pass
+
+
+def _apply_url_fill_chain(
+    X: pd.DataFrame,
+    *,
+    char_prob: dict[str, float],
+    tld_prob_mean: pd.Series,
+    tld_prob_global_fill: float,
+) -> pd.DataFrame:
+    """The 26 fill_* functions, in the one order their dependencies allow.
+
+    Domain must precede DomainLength, TLD and NoOfSubDomain; TLD must precede TLDLength
+    and TLDLegitimateProb; URLLength must precede every ratio that divides by it;
+    IsDomainIP must precede TLDLegitimateProb.
+
+    Every function here is row-local given the three injected statistics, which is the
+    property the invariance test depends on.
+    """
+    X = uf.fill_url_length(X, url_col="URL", url_length_col="URLLength")
+    X = uf.fill_domain(X, url_col="URL", domain_col="Domain")
+    X = uf.fill_domain_length(X, domain_col="Domain", domain_length_col="DomainLength")
+    X = uf.fill_is_domain_ip(X, domain_col="Domain", is_domain_ip_col="IsDomainIP")
+    X = uf.fill_tld(X, domain_col="Domain", tld_col="TLD")
+    X = uf.fill_char_continuation_rate(X, url_col="URL", char_rate_col="CharContinuationRate")
+    X = uf.fill_tld_legitimate_prob(
+        X,
+        tld_prob_mean=tld_prob_mean,
+        global_fill_value=tld_prob_global_fill,
+        tld_col="TLD",
+        tld_prob_col="TLDLegitimateProb",
+        is_domain_ip_col="IsDomainIP",
+    )
+    X = uf.fill_url_char_prob(
+        X, url_col="URL", char_prob_col="URLCharProb", char_prob=char_prob
+    )
+    X = uf.fill_tld_length(X, tld_col="TLD", tld_length_col="TLDLength")
+    X = uf.fill_no_of_subdomains(X, domain_col="Domain", subdomain_col="NoOfSubDomain")
+    X = uf.fill_has_obfuscation(X, url_col="URL", obfuscation_col="HasObfuscation")
+    X = uf.fill_no_of_obfuscated_characters(X, url_col="URL", obf_char_col="NoOfObfuscatedChar")
+    X = uf.fill_obfuscation_ratio(
+        X,
+        obf_char_col="NoOfObfuscatedChar",
+        url_length_col="URLLength",
+        obf_ratio_col="ObfuscationRatio",
+    )
+    X = uf.fill_no_of_letters_in_url(X, url_col="URL", letters_col="NoOfLettersInURL")
+    X = uf.fill_letter_ratio_in_url(
+        X,
+        letters_col="NoOfLettersInURL",
+        url_length_col="URLLength",
+        ratio_col="LetterRatioInURL",
+    )
+    X = uf.fill_no_of_digits_in_url(X, url_col="URL", digits_col="NoOfDegitsInURL")
+    X = uf.fill_digits_ratio_in_url(X, url_col="URL", ratio_col="DegitRatioInURL")
+    X = uf.fill_no_of_equals_in_url(X, url_col="URL", equals_col="NoOfEqualsInURL")
+    X = uf.fill_no_of_qmark_in_url(X, url_col="URL", qmark_col="NoOfQMarkInURL")
+    X = uf.fill_no_of_ampersand_in_url(X, url_col="URL", ampersand_col="NoOfAmpersandInURL")
+    X = uf.fill_no_of_special_chars_in_url(
+        X, url_col="URL", special_chars_col="NoOfOtherSpecialCharsInURL"
+    )
+    X = uf.fill_specials_ratio_in_url(X, url_col="URL", ratio_col="SpacialCharRatioInURL")
+    X = uf.fill_is_https(X, url_col="URL", https_col="IsHTTPS")
+    X = uf.fill_has_title(X, title_col="Title", has_title_col="HasTitle")
+    X = uf.fill_domain_title_match_score(
+        X, title_col="Title", domain_col="Domain", score_col="DomainTitleMatchScore"
+    )
+    return uf.fill_url_title_match_score(
+        X, title_col="Title", url_col="URL", score_col="URLTitleMatchScore"
+    )
+
+
+def _group_mode(values: pd.Series) -> Any:
+    """Most frequent value in a group, or None when the group is entirely missing."""
+    mode = values.mode()
+    return mode.iloc[0] if not mode.empty else None
+
+
+def _cascade_key(row: Any, groupby_cols: tuple[str, ...]) -> tuple[float, ...]:
+    """Build the cascade lookup key for one row.
+
+    Values are coerced to float so that a key built from an int column and a key built
+    from the same column after a float round-trip compare equal. Without this, a one-row
+    frame (whose columns pandas may infer as int64) and a batch frame (float64 because
+    some other row had a NaN) would miss each other's mode tables, and invariance would
+    fail for reasons that have nothing to do with the statistics.
+    """
+    return tuple(float(row[c]) for c in groupby_cols)
+
+
+class Preprocessor:
+    """Learns from the training split, then applies what it learned and nothing else."""
+
+    def __init__(self, stats: FittedStats | None = None) -> None:
+        self._stats = stats
+        #: Per-cascade-step count of rows that fell back to the global mode. Instrumented
+        #: because a step with a high fallback rate is telling you its conditioning is
+        #: doing nothing useful -- worth knowing, not an error.
+        self.fallback_counts: Counter[str] = Counter()
+
+    @property
+    def stats(self) -> FittedStats:
+        if self._stats is None:
+            raise NotFittedError("Preprocessor.transform called before fit(); no statistics loaded")
+        return self._stats
+
+    @property
+    def is_fitted(self) -> bool:
+        return self._stats is not None
+
+    # -- fit ---------------------------------------------------------------
+
+    def fit(self, X_train: pd.DataFrame, y_train: pd.Series) -> FittedStats:
+        """Compute every statistic the serving path will need.
+
+        Returns FittedStats and nothing else -- not a transformed matrix. The caller
+        invokes transform(X_train) afterwards. That duplicates steps 1 and 2 and is worth
+        it: it makes "the training matrix is produced by the same code path as a single
+        live URL" true by construction rather than by inspection.
+        """
+        X = X_train.copy()
+
+        # 1. Character frequency table over the training corpus' URLs.
+        char_prob = uf.calculate_char_prob(X, "URL")
+
+        # 2. Materialise the string-derived columns so later steps see a filled TLD to
+        #    group by. This is a dry run of transform's first stage.
+        tld_prob_mean_series, tld_global_fill = uf.compute_tld_prob_statistics(X)
+        tld_skew = uf.scalar_float(X["TLDLegitimateProb"].skew())
+        tld_fill_method: FillMethod = "median" if abs(tld_skew) > 1 else "mean"
+
+        X = _apply_url_fill_chain(
+            X,
+            char_prob=char_prob,
+            tld_prob_mean=tld_prob_mean_series,
+            tld_prob_global_fill=tld_global_fill,
+        )
+
+        # 3/4. Numeric imputation. The threshold is 3 here and 1 for the TLD statistic
+        #      above. The two are genuinely different and are not unified.
+        numeric_fill: list[NumericFill] = []
+        for col in schema.NUMERICAL_COLUMNS:
+            skewness = uf.scalar_float(X[col].skew())
+            method: FillMethod
+            if abs(skewness) > 3:
+                method = "median"
+                value = uf.scalar_float(X[col].median())
+            else:
+                method = "mean"
+                value = uf.scalar_float(X[col].mean())
+            numeric_fill.append(
+                NumericFill(column=col, method=method, value=value, skew=skewness)
+            )
+            X[col] = X[col].fillna(value)
+
+        # 5. The ordered categorical cascade.
+        cascade = self._fit_cascade(X)
+        for step in cascade:
+            X[step.column] = self._fill_cascade_step(X, step, count_fallbacks=False)
+
+        # 6. Clip bounds, computed after imputation so the quantiles are over the filled
+        #    distribution.
+        clip_bounds = tuple(
+            ClipBound(
+                column=col,
+                lower=float(np.percentile(X[col].dropna(), 1.0)),
+                upper=float(np.percentile(X[col].dropna(), 99.0)),
+            )
+            for col in schema.NUMERICAL_COLUMNS
+        )
+        for b in clip_bounds:
+            X[b.column] = np.clip(X[b.column], b.lower, b.upper)
+
+        # 7. Scaler over the 30 numerics only.
+        X_features = X.drop(columns=list(schema.INTERMEDIATE_COLUMNS))
+        X_features = _validate_dtypes(X_features)
+        X_features = X_features[list(schema.FEATURE_ORDER)]
+        scaler_params = SubsetStandardScaler(schema.NUMERICAL_COLUMNS).fit(X_features)
+        X_scaled = SubsetStandardScaler.apply(X_features, scaler_params)
+
+        # 8. Naive-Bayes column drop, last because it needs the finished matrix.
+        nb_drop, nb_drop_reasons = naive_bayes_drop(X_scaled, y_train)
+
+        # 9. Page-feature fallbacks, derived from the fills computed above rather than
+        #    computed again. Lifted out so the interface can display what it fell back to.
+        numeric_map = {f.column: f for f in numeric_fill}
+        cascade_map = {s.column: s for s in cascade}
+        html_fallbacks: list[HtmlFallback] = []
+        for col in (*schema.HTML_FEATURES, *schema.TITLE_HYBRID_FEATURES):
+            if col in numeric_map:
+                nf = numeric_map[col]
+                source: FallbackSource = (
+                    "numeric_median" if nf.method == "median" else "numeric_mean"
+                )
+                html_fallbacks.append(
+                    HtmlFallback(column=col, value=nf.value, source=source)
+                )
+            elif col in cascade_map:
+                html_fallbacks.append(
+                    HtmlFallback(
+                        column=col,
+                        value=float(cascade_map[col].global_mode),
+                        source="categorical_mode",
+                    )
+                )
+
+        stats = FittedStats(
+            char_prob=char_prob,
+            tld_prob_mean={str(k): float(v) for k, v in tld_prob_mean_series.items()},
+            tld_prob_global_fill=float(tld_global_fill),
+            tld_skew=tld_skew,
+            tld_fill_method=tld_fill_method,
+            numeric_fill=tuple(numeric_fill),
+            categorical_cascade=cascade,
+            clip_bounds=clip_bounds,
+            scaler=scaler_params,
+            nb_drop=nb_drop,
+            nb_drop_reasons=nb_drop_reasons,
+            html_fallbacks=tuple(html_fallbacks),
+            feature_order=schema.FEATURE_ORDER,
+            numerical_columns=schema.NUMERICAL_COLUMNS,
+            continuous_columns=schema.CONTINUOUS_COLUMNS,
+            discrete_columns=schema.DISCRETE_COLUMNS,
+            categorical_columns_filtered=schema.CATEGORICAL_COLUMNS_FILTERED,
+            n_train_rows=int(len(X_train)),
+        )
+        self._stats = stats
+        return stats
+
+    def _fit_cascade(self, X: pd.DataFrame) -> tuple[CategoricalFillStep, ...]:
+        """Learn the 18 mode tables, growing the group key one column per step.
+
+        The key grows monotonically: step 1 groups by HasObfuscation alone, step 2 by
+        HasObfuscation and the column step 1 just filled, and so on. So column n is
+        imputed conditioned on columns 1..n-1, and permuting the list changes every mode
+        table from step 2 onward. It is not a cosmetic ordering.
+        """
+        steps: list[CategoricalFillStep] = []
+        groupby_cols: list[str] = list(schema.CATEGORICAL_FILL_INITIAL_GROUP_BY)
+
+        for col in schema.CATEGORICAL_COLUMNS_TO_FILL:
+            grouped = X.groupby(groupby_cols, dropna=False)[col].agg(_group_mode)
+            col_mode = X[col].mode()
+            global_mode = uf.scalar_float(col_mode.iloc[0]) if not col_mode.empty else 0.0
+
+            modes: dict[tuple[float, ...], float] = {}
+            for key, value in grouped.items():
+                if value is None or pd.isna(value):
+                    continue
+                key_tuple = key if isinstance(key, tuple) else (key,)
+                if any(pd.isna(k) for k in key_tuple):
+                    continue
+                modes[tuple(float(k) for k in key_tuple)] = float(value)
+
+            step = CategoricalFillStep(
+                column=col,
+                groupby_cols=tuple(groupby_cols),
+                modes=modes,
+                global_mode=global_mode,
+            )
+            steps.append(step)
+            X[col] = self._fill_cascade_step(X, step, count_fallbacks=False)
+            groupby_cols.append(col)
+
+        return tuple(steps)
+
+    # -- transform ---------------------------------------------------------
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        """Apply the fitted statistics. Computes nothing.
+
+        Accepts a frame in the raw schema shape -- the same shape a row of the training
+        CSV has -- whether it holds 28,081 rows or 1. There is no branch on len(X).
+        """
+        stats = self.stats
+
+        # Validate the input contract before touching anything. A missing feature column
+        # is a broken extractor, not an unknown value, and it must be reported by name
+        # rather than surfacing later as a bare KeyError from whichever stage happened to
+        # reach for it first.
+        required = {*stats.feature_order, *schema.INTERMEDIATE_COLUMNS}
+        missing = sorted(required - set(X.columns))
+        if missing:
+            raise FeatureContractError(f"input columns absent from frame: {missing}")
+
+        X = X.copy()
+
+        tld_prob_mean = pd.Series(stats.tld_prob_mean, dtype="float64")
+
+        # 1. The 26 row-local fill functions, with the three fitted statistics injected.
+        X = _apply_url_fill_chain(
+            X,
+            char_prob=stats.char_prob,
+            tld_prob_mean=tld_prob_mean,
+            tld_prob_global_fill=stats.tld_prob_global_fill,
+        )
+
+        # 2. Numeric imputation from recorded values.
+        for f in stats.numeric_fill:
+            X[f.column] = X[f.column].fillna(f.value)
+
+        # 3. Replay the cascade in recorded order.
+        for step in stats.categorical_cascade:
+            X[step.column] = self._fill_cascade_step(X, step, count_fallbacks=True)
+
+        # 4. Clip to recorded bounds.
+        for b in stats.clip_bounds:
+            X[b.column] = np.clip(X[b.column], b.lower, b.upper)
+
+        # 5. Drop the text intermediates.
+        X = X.drop(columns=[c for c in schema.INTERMEDIATE_COLUMNS if c in X.columns])
+
+        # 6. Reindex to the frozen order. A missing column is a broken contract, not an
+        #    unknown value, so this raises rather than filling.
+        missing = sorted(set(stats.feature_order) - set(X.columns))
+        if missing:
+            raise FeatureContractError(f"feature columns absent from frame: {missing}")
+        X = X[list(stats.feature_order)]
+
+        # 7. Dtype validation, with the column lists passed in explicitly rather than
+        #    closed over. A helper that closes over a module global is a train/serve skew
+        #    bug waiting for someone to import the module in a different order.
+        X = _validate_dtypes(
+            X,
+            binary_columns=(*stats.categorical_columns_filtered, *stats.discrete_columns),
+        )
+
+        # 8. Scale the 30 numerics.
+        X = SubsetStandardScaler.apply(X, stats.scaler)
+
+        return X
+
+    def transform_matrix(self, X: pd.DataFrame) -> np.ndarray:
+        """transform(), fixed to float32 at the boundary.
+
+        The dtype is pinned here so the KNN distance kernel never silently upcasts and
+        never sees a dtype it did not see during training.
+        """
+        return self.transform(X).to_numpy(dtype=np.float32)
+
+    def _fill_cascade_step(
+        self, X: pd.DataFrame, step: CategoricalFillStep, *, count_fallbacks: bool
+    ) -> pd.Series:
+        """Fill one cascade column from its recorded mode table.
+
+        An unseen group key is expected, not exceptional. At step 18 the key is 18 columns
+        wide, so the training frame observed only a small fraction of the combinatorial
+        space. The defined behaviour is to fall back to the recorded global mode -- not to
+        raise, not to warn at the user, and under no circumstances to recompute the mode
+        from the incoming batch, which would reintroduce exactly the defect this module
+        exists to remove.
+        """
+        col = X[step.column]
+        null_mask = col.isna()
+        if not null_mask.any():
+            return col
+
+        modes = step.modes
+        global_mode = step.global_mode
+
+        # Build every key at once rather than looping row by row. Row-wise .loc lookups on
+        # a 53-column frame cost enough that, across 18 cascade steps and the ~40% of
+        # 112,000 training rows that need filling, they dominate the whole fit. The
+        # arithmetic is unchanged: each key is still built only from its own row's values.
+        key_block = X.loc[null_mask, list(step.groupby_cols)]
+        key_values = key_block.to_numpy(dtype="float64", na_value=np.nan)
+
+        resolved = np.empty(len(key_values), dtype="float64")
+        missed = 0
+        for i, key_row in enumerate(key_values):
+            if np.isnan(key_row).any():
+                # A key component that is itself missing cannot address the table. It is a
+                # fallback, not an error -- the earlier steps of the cascade may leave a
+                # conditioning column unfilled on a sufficiently sparse row.
+                resolved[i] = global_mode
+                missed += 1
+                continue
+            value = modes.get(tuple(key_row.tolist()))
+            if value is None:
+                resolved[i] = global_mode
+                missed += 1
+            else:
+                resolved[i] = value
+
+        if count_fallbacks and missed:
+            self.fallback_counts[step.column] += missed
+
+        filled = col.copy()
+        filled.loc[null_mask] = resolved
+        return filled
+
+
+def _validate_dtypes(
+    X: pd.DataFrame, binary_columns: tuple[str, ...] | None = None
+) -> pd.DataFrame:
+    """Coerce the integral columns to int, as the original did.
+
+    Column lists arrive as a parameter. The original closed over notebook globals here,
+    which is how the scaler silently ended up touching only 30 of 49 columns.
+    """
+    if binary_columns is None:
+        binary_columns = (*schema.CATEGORICAL_COLUMNS_FILTERED, *schema.DISCRETE_COLUMNS)
+
+    X = X.copy()
+    cols = [c for c in binary_columns if c in X.columns]
+    if cols:
+        X[cols] = X[cols].astype("int64")
+    return X
+
+
+def naive_bayes_drop(
+    X: pd.DataFrame, y: pd.Series
+) -> tuple[tuple[str, ...], dict[str, NbDropReason]]:
+    """Columns Gaussian NB cannot use, with the reason recorded per column.
+
+    Two rules, ported from the original:
+
+      - a binary indicator with an empty (value, label) contingency cell, which would give
+        that cell zero likelihood;
+      - any column with zero standard deviation, whose Gaussian is degenerate.
+
+    The original computed this list and discarded it, so nothing downstream could say
+    which columns a served model had actually been fitted on. Here it is returned,
+    deduplicated with the first reason winning, and persisted.
+    """
+    nb_df = X.join(y.rename(schema.TARGET_COLUMN))
+    reasons: dict[str, NbDropReason] = {}
+    ordered: list[str] = []
+
+    for f in schema.CATEGORICAL_COLUMNS_FILTERED:
+        if f not in X.columns:
+            continue
+        for value, label in [(x // 2, x % 2) for x in range(4)]:
+            empty = len(nb_df[(nb_df[f] == value) & (nb_df[schema.TARGET_COLUMN] == label)]) == 0
+            if empty and f not in reasons:
+                reasons[f] = "empty_contingency_cell"
+                ordered.append(f)
+
+    for col in X.columns:
+        if uf.scalar_float(X[col].std()) == 0.0 and col not in reasons:
+            reasons[col] = "zero_std"
+            ordered.append(col)
+
+    return tuple(ordered), reasons

phiusiil/schema.py

@@ -0,0 +1,332 @@
+"""The frozen feature contract.
+
+Every column name, every column group, and the exact ordering of the 49 model features
+live here and nowhere else. Nothing in the codebase may hardcode a column list; it imports
+one of these constants instead.
+
+Why this module is written defensively, with an assertion after every group: the original
+notebook derived its column groups from whatever DataFrame happened to be in scope. The
+scaler consequently touched only the 30 numeric columns and left the 19 binaries raw --
+not by decision, but because ``numerical_columns`` was a notebook global that closed over
+the training frame. Every recorded metric depends on that accident. Making the groups
+explicit and asserting their sizes at import time means a typo fails immediately and
+loudly, rather than silently changing what the models were fitted on.
+
+Dataset column typos are real and preserved verbatim: ``NoOfDegitsInURL``,
+``DegitRatioInURL``, ``SpacialCharRatioInURL``.
+"""
+
+from __future__ import annotations
+
+from typing import Final
+
+# ---------------------------------------------------------------------------
+# Raw CSV schema
+# ---------------------------------------------------------------------------
+
+#: The 56 columns of the raw CSV, in file order.
+RAW_COLUMNS: Final[tuple[str, ...]] = (
+    "id",
+    "FILENAME",
+    "URL",
+    "URLLength",
+    "Domain",
+    "DomainLength",
+    "IsDomainIP",
+    "TLD",
+    "CharContinuationRate",
+    "TLDLegitimateProb",
+    "URLCharProb",
+    "TLDLength",
+    "NoOfSubDomain",
+    "HasObfuscation",
+    "NoOfObfuscatedChar",
+    "ObfuscationRatio",
+    "NoOfLettersInURL",
+    "LetterRatioInURL",
+    "NoOfDegitsInURL",
+    "DegitRatioInURL",
+    "NoOfEqualsInURL",
+    "NoOfQMarkInURL",
+    "NoOfAmpersandInURL",
+    "NoOfOtherSpecialCharsInURL",
+    "SpacialCharRatioInURL",
+    "IsHTTPS",
+    "LineOfCode",
+    "LargestLineLength",
+    "HasTitle",
+    "Title",
+    "DomainTitleMatchScore",
+    "URLTitleMatchScore",
+    "HasFavicon",
+    "Robots",
+    "IsResponsive",
+    "NoOfURLRedirect",
+    "NoOfSelfRedirect",
+    "HasDescription",
+    "NoOfPopup",
+    "NoOfiFrame",
+    "HasExternalFormSubmit",
+    "HasSocialNet",
+    "HasSubmitButton",
+    "HasHiddenFields",
+    "HasPasswordField",
+    "Bank",
+    "Pay",
+    "Crypto",
+    "HasCopyrightInfo",
+    "NoOfImage",
+    "NoOfCSS",
+    "NoOfJS",
+    "NoOfSelfRef",
+    "NoOfEmptyRef",
+    "NoOfExternalRef",
+    "label",
+)
+
+#: Dropped before modelling: identifiers and the target.
+DROPPED_IDENTIFIER_COLUMNS: Final[tuple[str, ...]] = ("id", "FILENAME")
+TARGET_COLUMN: Final[str] = "label"
+
+#: Text columns used to derive features and then dropped. They are not model inputs.
+INTERMEDIATE_COLUMNS: Final[tuple[str, ...]] = ("URL", "Domain", "TLD", "Title")
+
+#: The 53 columns the notebook's ``X_train`` carried, in file order.
+WORKING_COLUMNS: Final[tuple[str, ...]] = tuple(
+    c for c in RAW_COLUMNS if c not in DROPPED_IDENTIFIER_COLUMNS and c != TARGET_COLUMN
+)
+
+# ---------------------------------------------------------------------------
+# The 49 model features
+# ---------------------------------------------------------------------------
+
+#: The frozen model input, in order. This ordering is the wire format between training
+#: and serving: the matrix handed to every model has these columns, in exactly this
+#: sequence. Reordering it silently changes what each model coefficient refers to.
+FEATURE_ORDER: Final[tuple[str, ...]] = tuple(
+    c for c in WORKING_COLUMNS if c not in INTERMEDIATE_COLUMNS
+)
+
+# ---------------------------------------------------------------------------
+# Column groups, reproduced exactly as the notebook defined them
+# ---------------------------------------------------------------------------
+
+#: Notebook cell 12. Includes the four text intermediates, which is why it has 23
+#: members while the model-facing filtered list has 19.
+CATEGORICAL_COLUMNS: Final[tuple[str, ...]] = (
+    "IsDomainIP",
+    "HasObfuscation",
+    "IsHTTPS",
+    "HasTitle",
+    "HasFavicon",
+    "Robots",
+    "IsResponsive",
+    "HasDescription",
+    "HasSocialNet",
+    "HasSubmitButton",
+    "HasHiddenFields",
+    "HasPasswordField",
+    "Bank",
+    "Pay",
+    "Crypto",
+    "HasCopyrightInfo",
+    "Domain",
+    "URL",
+    "TLD",
+    "Title",
+    "HasExternalFormSubmit",
+    "NoOfSelfRedirect",
+    "NoOfURLRedirect",
+)
+
+#: Notebook cell 12: the same list without the four text intermediates. These are the
+#: columns the scaler must pass through bit-identically.
+CATEGORICAL_COLUMNS_FILTERED: Final[tuple[str, ...]] = tuple(
+    c for c in CATEGORICAL_COLUMNS if c not in INTERMEDIATE_COLUMNS
+)
+
+#: Notebook cell 104. Order is load-bearing, not cosmetic: the imputation cascade appends
+#: each just-filled column to the group-by key before moving to the next one, so column
+#: *n* is imputed conditioned on columns 1..n-1. Reordering this list changes the fill
+#: values it produces.
+CATEGORICAL_COLUMNS_TO_FILL: Final[tuple[str, ...]] = (
+    "IsDomainIP",
+    "IsHTTPS",
+    "HasTitle",
+    "IsResponsive",
+    "Pay",
+    "HasHiddenFields",
+    "Robots",
+    "Crypto",
+    "HasDescription",
+    "Bank",
+    "HasExternalFormSubmit",
+    "HasFavicon",
+    "HasSubmitButton",
+    "HasPasswordField",
+    "NoOfSelfRedirect",
+    "HasCopyrightInfo",
+    "NoOfURLRedirect",
+    "HasSocialNet",
+)
+
+#: The initial group-by key for the cascade above (notebook cell 104).
+CATEGORICAL_FILL_INITIAL_GROUP_BY: Final[tuple[str, ...]] = ("HasObfuscation",)
+
+#: Notebook cell 13: everything in the working set that is not categorical. This is the
+#: exact set the scaler touches, and no other.
+NUMERICAL_COLUMNS: Final[tuple[str, ...]] = tuple(
+    c for c in WORKING_COLUMNS if c not in CATEGORICAL_COLUMNS
+)
+
+#: Notebook cell 14.
+CONTINUOUS_COLUMNS: Final[tuple[str, ...]] = (
+    "CharContinuationRate",
+    "ObfuscationRatio",
+    "DegitRatioInURL",
+    "DomainTitleMatchScore",
+    "URLTitleMatchScore",
+    "TLDLegitimateProb",
+    "URLCharProb",
+    "LetterRatioInURL",
+    "SpacialCharRatioInURL",
+)
+
+#: Notebook cell 15.
+DISCRETE_COLUMNS: Final[tuple[str, ...]] = tuple(
+    c for c in NUMERICAL_COLUMNS if c not in CONTINUOUS_COLUMNS
+)
+
+# ---------------------------------------------------------------------------
+# The 21 / 3 / 25 split by what a feature needs in order to be computed
+# ---------------------------------------------------------------------------
+
+#: Derivable from the URL string alone. Available even when the fetch fails.
+URL_ONLY_FEATURES: Final[tuple[str, ...]] = (
+    "URLLength",
+    "DomainLength",
+    "IsDomainIP",
+    "CharContinuationRate",
+    "TLDLegitimateProb",
+    "URLCharProb",
+    "TLDLength",
+    "NoOfSubDomain",
+    "HasObfuscation",
+    "NoOfObfuscatedChar",
+    "ObfuscationRatio",
+    "NoOfLettersInURL",
+    "LetterRatioInURL",
+    "NoOfDegitsInURL",
+    "DegitRatioInURL",
+    "NoOfEqualsInURL",
+    "NoOfQMarkInURL",
+    "NoOfAmpersandInURL",
+    "NoOfOtherSpecialCharsInURL",
+    "SpacialCharRatioInURL",
+    "IsHTTPS",
+)
+
+#: Have a URL-side implementation, but consume a Title that only exists once the page has
+#: been fetched and parsed. With no successful fetch these fall through to imputation.
+TITLE_HYBRID_FEATURES: Final[tuple[str, ...]] = (
+    "HasTitle",
+    "DomainTitleMatchScore",
+    "URLTitleMatchScore",
+)
+
+#: Require the fetched page. Slightly more than half the model's inputs -- which is why
+#: single-URL inference fetches at all, and why it must abstain when the fetch fails.
+HTML_FEATURES: Final[tuple[str, ...]] = tuple(
+    c for c in FEATURE_ORDER if c not in URL_ONLY_FEATURES and c not in TITLE_HYBRID_FEATURES
+)
+
+# ---------------------------------------------------------------------------
+# Labels
+# ---------------------------------------------------------------------------
+
+#: Class 0 is phishing and is the positive class throughout this codebase. Class 1 is
+#: legitimate. Every confusion matrix is oriented rows = true (0, 1), cols = pred (0, 1).
+PHISHING_LABEL: Final[int] = 0
+LEGITIMATE_LABEL: Final[int] = 1
+
+#: A constant "always legitimate" predictor scores this on the full dataset. Any accuracy
+#: figure is meaningless unless read against it.
+MAJORITY_BASELINE_ACCURACY: Final[float] = 0.9248
+
+#: Provenance values recorded per feature for a single inference.
+PROVENANCE_URL: Final[str] = "url"
+PROVENANCE_SCRAPED: Final[str] = "scraped"
+PROVENANCE_IMPUTED: Final[str] = "imputed"
+PROVENANCE_DEMOTED: Final[str] = "demoted"
+PROVENANCE_VALUES: Final[frozenset[str]] = frozenset(
+    {PROVENANCE_URL, PROVENANCE_SCRAPED, PROVENANCE_IMPUTED, PROVENANCE_DEMOTED}
+)
+
+# ---------------------------------------------------------------------------
+# Import-time assertions. A typo above fails here, not three phases downstream.
+# ---------------------------------------------------------------------------
+
+assert len(RAW_COLUMNS) == 56, len(RAW_COLUMNS)
+assert len(set(RAW_COLUMNS)) == 56, "duplicate column name in RAW_COLUMNS"
+assert len(WORKING_COLUMNS) == 53, len(WORKING_COLUMNS)
+assert len(FEATURE_ORDER) == 49, len(FEATURE_ORDER)
+
+assert len(CATEGORICAL_COLUMNS) == 23, len(CATEGORICAL_COLUMNS)
+assert len(CATEGORICAL_COLUMNS_FILTERED) == 19, len(CATEGORICAL_COLUMNS_FILTERED)
+assert len(CATEGORICAL_COLUMNS_TO_FILL) == 18, len(CATEGORICAL_COLUMNS_TO_FILL)
+assert len(NUMERICAL_COLUMNS) == 30, len(NUMERICAL_COLUMNS)
+assert len(CONTINUOUS_COLUMNS) == 9, len(CONTINUOUS_COLUMNS)
+assert len(DISCRETE_COLUMNS) == 21, len(DISCRETE_COLUMNS)
+
+assert len(URL_ONLY_FEATURES) == 21, len(URL_ONLY_FEATURES)
+assert len(TITLE_HYBRID_FEATURES) == 3, len(TITLE_HYBRID_FEATURES)
+assert len(HTML_FEATURES) == 25, len(HTML_FEATURES)
+
+# The scaler's scope and the pass-through set must exactly partition the 49 features.
+assert set(NUMERICAL_COLUMNS) | set(CATEGORICAL_COLUMNS_FILTERED) == set(FEATURE_ORDER)
+assert not (set(NUMERICAL_COLUMNS) & set(CATEGORICAL_COLUMNS_FILTERED))
+
+# The three-way availability split must also partition the 49, with no overlap.
+assert (
+    set(URL_ONLY_FEATURES) | set(TITLE_HYBRID_FEATURES) | set(HTML_FEATURES)
+    == set(FEATURE_ORDER)
+)
+assert not (set(URL_ONLY_FEATURES) & set(TITLE_HYBRID_FEATURES))
+assert not (set(URL_ONLY_FEATURES) & set(HTML_FEATURES))
+assert not (set(TITLE_HYBRID_FEATURES) & set(HTML_FEATURES))
+
+assert set(CONTINUOUS_COLUMNS) <= set(NUMERICAL_COLUMNS)
+assert set(CATEGORICAL_COLUMNS_TO_FILL) <= set(CATEGORICAL_COLUMNS_FILTERED)
+assert set(CATEGORICAL_FILL_INITIAL_GROUP_BY) <= set(CATEGORICAL_COLUMNS_FILTERED)
+
+# The cascade fills every categorical except the one it initially groups by.
+assert set(CATEGORICAL_COLUMNS_FILTERED) - set(CATEGORICAL_COLUMNS_TO_FILL) == set(
+    CATEGORICAL_FILL_INITIAL_GROUP_BY
+)
+
+__all__ = [
+    "CATEGORICAL_COLUMNS",
+    "CATEGORICAL_COLUMNS_FILTERED",
+    "CATEGORICAL_COLUMNS_TO_FILL",
+    "CATEGORICAL_FILL_INITIAL_GROUP_BY",
+    "CONTINUOUS_COLUMNS",
+    "DISCRETE_COLUMNS",
+    "DROPPED_IDENTIFIER_COLUMNS",
+    "FEATURE_ORDER",
+    "HTML_FEATURES",
+    "INTERMEDIATE_COLUMNS",
+    "LEGITIMATE_LABEL",
+    "MAJORITY_BASELINE_ACCURACY",
+    "NUMERICAL_COLUMNS",
+    "PHISHING_LABEL",
+    "PROVENANCE_DEMOTED",
+    "PROVENANCE_IMPUTED",
+    "PROVENANCE_SCRAPED",
+    "PROVENANCE_URL",
+    "PROVENANCE_VALUES",
+    "RAW_COLUMNS",
+    "TARGET_COLUMN",
+    "TITLE_HYBRID_FEATURES",
+    "URL_ONLY_FEATURES",
+    "WORKING_COLUMNS",
+]

pyproject.toml

@@ -0,0 +1,31 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "fetiai-v1-phiusiil-binclf-knn-scratch-500k"
+version = "1.0.0"
+description = "KNN (from scratch) for PhiUSIIL phishing URL classification, served over HTTP"
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "MIT" }
+authors = [
+    { name = "Kelompok 16 ITB" },
+]
+
+# Runtime dependencies are pinned in requirements.txt. This list carries only the lower
+# bounds the code actually depends on.
+dependencies = [
+    "fastapi>=0.110",
+    "numpy>=1.26",
+    "pandas>=2.1",
+    "pydantic>=2.6",
+    "uvicorn>=0.27"
+]
+
+[tool.setuptools]
+packages = ["phiusiil", "phiusiil.features", "phiusiil.models", "phiusiil.preprocess", "server"]
+
+[tool.pytest.ini_options]
+pythonpath = ["."]
+testpaths = ["tests"]