phiusiil/schema.py

"""The frozen feature contract.

Every column name, every column group, and the exact ordering of the 49 model features
live here and nowhere else. Nothing in the codebase may hardcode a column list; it imports
one of these constants instead.

Why this module is written defensively, with an assertion after every group: the original
notebook derived its column groups from whatever DataFrame happened to be in scope. The
scaler consequently touched only the 30 numeric columns and left the 19 binaries raw --
not by decision, but because ``numerical_columns`` was a notebook global that closed over
the training frame. Every recorded metric depends on that accident. Making the groups
explicit and asserting their sizes at import time means a typo fails immediately and
loudly, rather than silently changing what the models were fitted on.

Dataset column typos are real and preserved verbatim: ``NoOfDegitsInURL``,
``DegitRatioInURL``, ``SpacialCharRatioInURL``.
"""

from __future__ import annotations

from typing import Final

# ---------------------------------------------------------------------------
# Raw CSV schema
# ---------------------------------------------------------------------------

#: The 56 columns of the raw CSV, in file order.
RAW_COLUMNS: Final[tuple[str, ...]] = (
    "id",
    "FILENAME",
    "URL",
    "URLLength",
    "Domain",
    "DomainLength",
    "IsDomainIP",
    "TLD",
    "CharContinuationRate",
    "TLDLegitimateProb",
    "URLCharProb",
    "TLDLength",
    "NoOfSubDomain",
    "HasObfuscation",
    "NoOfObfuscatedChar",
    "ObfuscationRatio",
    "NoOfLettersInURL",
    "LetterRatioInURL",
    "NoOfDegitsInURL",
    "DegitRatioInURL",
    "NoOfEqualsInURL",
    "NoOfQMarkInURL",
    "NoOfAmpersandInURL",
    "NoOfOtherSpecialCharsInURL",
    "SpacialCharRatioInURL",
    "IsHTTPS",
    "LineOfCode",
    "LargestLineLength",
    "HasTitle",
    "Title",
    "DomainTitleMatchScore",
    "URLTitleMatchScore",
    "HasFavicon",
    "Robots",
    "IsResponsive",
    "NoOfURLRedirect",
    "NoOfSelfRedirect",
    "HasDescription",
    "NoOfPopup",
    "NoOfiFrame",
    "HasExternalFormSubmit",
    "HasSocialNet",
    "HasSubmitButton",
    "HasHiddenFields",
    "HasPasswordField",
    "Bank",
    "Pay",
    "Crypto",
    "HasCopyrightInfo",
    "NoOfImage",
    "NoOfCSS",
    "NoOfJS",
    "NoOfSelfRef",
    "NoOfEmptyRef",
    "NoOfExternalRef",
    "label",
)

#: Dropped before modelling: identifiers and the target.
DROPPED_IDENTIFIER_COLUMNS: Final[tuple[str, ...]] = ("id", "FILENAME")
TARGET_COLUMN: Final[str] = "label"

#: Text columns used to derive features and then dropped. They are not model inputs.
INTERMEDIATE_COLUMNS: Final[tuple[str, ...]] = ("URL", "Domain", "TLD", "Title")

#: The 53 columns the notebook's ``X_train`` carried, in file order.
WORKING_COLUMNS: Final[tuple[str, ...]] = tuple(
    c for c in RAW_COLUMNS if c not in DROPPED_IDENTIFIER_COLUMNS and c != TARGET_COLUMN
)

# ---------------------------------------------------------------------------
# The 49 model features
# ---------------------------------------------------------------------------

#: The frozen model input, in order. This ordering is the wire format between training
#: and serving: the matrix handed to every model has these columns, in exactly this
#: sequence. Reordering it silently changes what each model coefficient refers to.
FEATURE_ORDER: Final[tuple[str, ...]] = tuple(
    c for c in WORKING_COLUMNS if c not in INTERMEDIATE_COLUMNS
)

# ---------------------------------------------------------------------------
# Column groups, reproduced exactly as the notebook defined them
# ---------------------------------------------------------------------------

#: Notebook cell 12. Includes the four text intermediates, which is why it has 23
#: members while the model-facing filtered list has 19.
CATEGORICAL_COLUMNS: Final[tuple[str, ...]] = (
    "IsDomainIP",
    "HasObfuscation",
    "IsHTTPS",
    "HasTitle",
    "HasFavicon",
    "Robots",
    "IsResponsive",
    "HasDescription",
    "HasSocialNet",
    "HasSubmitButton",
    "HasHiddenFields",
    "HasPasswordField",
    "Bank",
    "Pay",
    "Crypto",
    "HasCopyrightInfo",
    "Domain",
    "URL",
    "TLD",
    "Title",
    "HasExternalFormSubmit",
    "NoOfSelfRedirect",
    "NoOfURLRedirect",
)

#: Notebook cell 12: the same list without the four text intermediates. These are the
#: columns the scaler must pass through bit-identically.
CATEGORICAL_COLUMNS_FILTERED: Final[tuple[str, ...]] = tuple(
    c for c in CATEGORICAL_COLUMNS if c not in INTERMEDIATE_COLUMNS
)

#: Notebook cell 104. Order is load-bearing, not cosmetic: the imputation cascade appends
#: each just-filled column to the group-by key before moving to the next one, so column
#: *n* is imputed conditioned on columns 1..n-1. Reordering this list changes the fill
#: values it produces.
CATEGORICAL_COLUMNS_TO_FILL: Final[tuple[str, ...]] = (
    "IsDomainIP",
    "IsHTTPS",
    "HasTitle",
    "IsResponsive",
    "Pay",
    "HasHiddenFields",
    "Robots",
    "Crypto",
    "HasDescription",
    "Bank",
    "HasExternalFormSubmit",
    "HasFavicon",
    "HasSubmitButton",
    "HasPasswordField",
    "NoOfSelfRedirect",
    "HasCopyrightInfo",
    "NoOfURLRedirect",
    "HasSocialNet",
)

#: The initial group-by key for the cascade above (notebook cell 104).
CATEGORICAL_FILL_INITIAL_GROUP_BY: Final[tuple[str, ...]] = ("HasObfuscation",)

#: Notebook cell 13: everything in the working set that is not categorical. This is the
#: exact set the scaler touches, and no other.
NUMERICAL_COLUMNS: Final[tuple[str, ...]] = tuple(
    c for c in WORKING_COLUMNS if c not in CATEGORICAL_COLUMNS
)

#: Notebook cell 14.
CONTINUOUS_COLUMNS: Final[tuple[str, ...]] = (
    "CharContinuationRate",
    "ObfuscationRatio",
    "DegitRatioInURL",
    "DomainTitleMatchScore",
    "URLTitleMatchScore",
    "TLDLegitimateProb",
    "URLCharProb",
    "LetterRatioInURL",
    "SpacialCharRatioInURL",
)

#: Notebook cell 15.
DISCRETE_COLUMNS: Final[tuple[str, ...]] = tuple(
    c for c in NUMERICAL_COLUMNS if c not in CONTINUOUS_COLUMNS
)

# ---------------------------------------------------------------------------
# The 21 / 3 / 25 split by what a feature needs in order to be computed
# ---------------------------------------------------------------------------

#: Derivable from the URL string alone. Available even when the fetch fails.
URL_ONLY_FEATURES: Final[tuple[str, ...]] = (
    "URLLength",
    "DomainLength",
    "IsDomainIP",
    "CharContinuationRate",
    "TLDLegitimateProb",
    "URLCharProb",
    "TLDLength",
    "NoOfSubDomain",
    "HasObfuscation",
    "NoOfObfuscatedChar",
    "ObfuscationRatio",
    "NoOfLettersInURL",
    "LetterRatioInURL",
    "NoOfDegitsInURL",
    "DegitRatioInURL",
    "NoOfEqualsInURL",
    "NoOfQMarkInURL",
    "NoOfAmpersandInURL",
    "NoOfOtherSpecialCharsInURL",
    "SpacialCharRatioInURL",
    "IsHTTPS",
)

#: Have a URL-side implementation, but consume a Title that only exists once the page has
#: been fetched and parsed. With no successful fetch these fall through to imputation.
TITLE_HYBRID_FEATURES: Final[tuple[str, ...]] = (
    "HasTitle",
    "DomainTitleMatchScore",
    "URLTitleMatchScore",
)

#: Require the fetched page. Slightly more than half the model's inputs -- which is why
#: single-URL inference fetches at all, and why it must abstain when the fetch fails.
HTML_FEATURES: Final[tuple[str, ...]] = tuple(
    c for c in FEATURE_ORDER if c not in URL_ONLY_FEATURES and c not in TITLE_HYBRID_FEATURES
)

# ---------------------------------------------------------------------------
# Labels
# ---------------------------------------------------------------------------

#: Class 0 is phishing and is the positive class throughout this codebase. Class 1 is
#: legitimate. Every confusion matrix is oriented rows = true (0, 1), cols = pred (0, 1).
PHISHING_LABEL: Final[int] = 0
LEGITIMATE_LABEL: Final[int] = 1

#: A constant "always legitimate" predictor scores this on the full dataset. Any accuracy
#: figure is meaningless unless read against it.
MAJORITY_BASELINE_ACCURACY: Final[float] = 0.9248

#: Provenance values recorded per feature for a single inference.
PROVENANCE_URL: Final[str] = "url"
PROVENANCE_SCRAPED: Final[str] = "scraped"
PROVENANCE_IMPUTED: Final[str] = "imputed"
PROVENANCE_DEMOTED: Final[str] = "demoted"
PROVENANCE_VALUES: Final[frozenset[str]] = frozenset(
    {PROVENANCE_URL, PROVENANCE_SCRAPED, PROVENANCE_IMPUTED, PROVENANCE_DEMOTED}
)

# ---------------------------------------------------------------------------
# Import-time assertions. A typo above fails here, not three phases downstream.
# ---------------------------------------------------------------------------

assert len(RAW_COLUMNS) == 56, len(RAW_COLUMNS)
assert len(set(RAW_COLUMNS)) == 56, "duplicate column name in RAW_COLUMNS"
assert len(WORKING_COLUMNS) == 53, len(WORKING_COLUMNS)
assert len(FEATURE_ORDER) == 49, len(FEATURE_ORDER)

assert len(CATEGORICAL_COLUMNS) == 23, len(CATEGORICAL_COLUMNS)
assert len(CATEGORICAL_COLUMNS_FILTERED) == 19, len(CATEGORICAL_COLUMNS_FILTERED)
assert len(CATEGORICAL_COLUMNS_TO_FILL) == 18, len(CATEGORICAL_COLUMNS_TO_FILL)
assert len(NUMERICAL_COLUMNS) == 30, len(NUMERICAL_COLUMNS)
assert len(CONTINUOUS_COLUMNS) == 9, len(CONTINUOUS_COLUMNS)
assert len(DISCRETE_COLUMNS) == 21, len(DISCRETE_COLUMNS)

assert len(URL_ONLY_FEATURES) == 21, len(URL_ONLY_FEATURES)
assert len(TITLE_HYBRID_FEATURES) == 3, len(TITLE_HYBRID_FEATURES)
assert len(HTML_FEATURES) == 25, len(HTML_FEATURES)

# The scaler's scope and the pass-through set must exactly partition the 49 features.
assert set(NUMERICAL_COLUMNS) | set(CATEGORICAL_COLUMNS_FILTERED) == set(FEATURE_ORDER)
assert not (set(NUMERICAL_COLUMNS) & set(CATEGORICAL_COLUMNS_FILTERED))

# The three-way availability split must also partition the 49, with no overlap.
assert (
    set(URL_ONLY_FEATURES) | set(TITLE_HYBRID_FEATURES) | set(HTML_FEATURES)
    == set(FEATURE_ORDER)
)
assert not (set(URL_ONLY_FEATURES) & set(TITLE_HYBRID_FEATURES))
assert not (set(URL_ONLY_FEATURES) & set(HTML_FEATURES))
assert not (set(TITLE_HYBRID_FEATURES) & set(HTML_FEATURES))

assert set(CONTINUOUS_COLUMNS) <= set(NUMERICAL_COLUMNS)
assert set(CATEGORICAL_COLUMNS_TO_FILL) <= set(CATEGORICAL_COLUMNS_FILTERED)
assert set(CATEGORICAL_FILL_INITIAL_GROUP_BY) <= set(CATEGORICAL_COLUMNS_FILTERED)

# The cascade fills every categorical except the one it initially groups by.
assert set(CATEGORICAL_COLUMNS_FILTERED) - set(CATEGORICAL_COLUMNS_TO_FILL) == set(
    CATEGORICAL_FILL_INITIAL_GROUP_BY
)

__all__ = [
    "CATEGORICAL_COLUMNS",
    "CATEGORICAL_COLUMNS_FILTERED",
    "CATEGORICAL_COLUMNS_TO_FILL",
    "CATEGORICAL_FILL_INITIAL_GROUP_BY",
    "CONTINUOUS_COLUMNS",
    "DISCRETE_COLUMNS",
    "DROPPED_IDENTIFIER_COLUMNS",
    "FEATURE_ORDER",
    "HTML_FEATURES",
    "INTERMEDIATE_COLUMNS",
    "LEGITIMATE_LABEL",
    "MAJORITY_BASELINE_ACCURACY",
    "NUMERICAL_COLUMNS",
    "PHISHING_LABEL",
    "PROVENANCE_DEMOTED",
    "PROVENANCE_IMPUTED",
    "PROVENANCE_SCRAPED",
    "PROVENANCE_URL",
    "PROVENANCE_VALUES",
    "RAW_COLUMNS",
    "TARGET_COLUMN",
    "TITLE_HYBRID_FEATURES",
    "URL_ONLY_FEATURES",
    "WORKING_COLUMNS",
]