Upload 6 files

README.md

@@ -1,3 +1,87 @@
----
-license: apache-2.0
----
+# Hybrid RoBERTa + Linguistic-Features Detector
+
+Best checkpoint from the master's thesis on hybrid machine learning approaches for detection of LLM-generated English texts by Bohdan Zhvalevskyi. 
+It combines a fine-tuned `roberta-base` CLS embedding with 25 normalized
+linguistic features through a feature-attention module and a learned gated
+fusion, followed by a 2-class classifier (`0 = human`, `1 = machine`).
+
+## Files
+
+| File | Purpose |
+|------|---------|
+| `hybrid_model_best.pt` | PyTorch `state_dict` for `HybridClassifier` (~500 MB). |
+| `model.py` | Self-contained architecture definition + `load_model()` helper. |
+| `features.py` | Self-contained linguistic feature extraction (normalize → chunk → 25 features). |
+| `ling_scaler.pkl` | Fitted training `StandardScaler` used to normalize the 25 features. **Required** for valid predictions. |
+| `example_usage.py` | Runnable end-to-end scoring example (raw text → prediction). |
+
+## Install
+
+```bash
+pip install torch transformers spacy scikit-learn
+python -m spacy download en_core_web_lg
+```
+
+## Quick start
+
+Score raw text end-to-end (extraction + normalization + document-level scoring
+are handled for you):
+
+```bash
+python example_usage.py --text "Your transcript goes here."
+python example_usage.py --file document.txt
+```
+
+Or from Python:
+
+```python
+import torch
+from transformers import RobertaTokenizer, GPT2LMHeadModel, GPT2TokenizerFast
+import spacy, pickle
+from model import load_model, CONFIG
+from features import extract_raw_features, prepare_document
+from example_usage import score_text
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+model = load_model("hybrid_model_best.pt", device=device)
+tokenizer = RobertaTokenizer.from_pretrained(CONFIG["roberta_model"])
+nlp = spacy.load("en_core_web_lg", disable=["ner"])
+gpt2_tokenizer = GPT2TokenizerFast.from_pretrained("gpt2")
+gpt2_model = GPT2LMHeadModel.from_pretrained("gpt2").to(device).eval()
+scaler = pickle.load(open("ling_scaler.pkl", "rb"))
+
+p_llm, chunk_probs = score_text(
+    "Your transcript goes here.",
+    model, tokenizer, nlp, gpt2_model, gpt2_tokenizer, scaler, device=device,
+)
+print(p_llm, "->", "machine" if p_llm >= 0.5 else "human")
+```
+
+## Linguistic features
+
+The 25 features, in the exact order the model expects, are extracted by
+`features.py` and normalized with the fitted training scaler (`ling_scaler.pkl`):
+
+```
+msttr, avg_word_len, hapax_ratio, function_ratio, punct_density, char_entropy,
+burstiness, repetition_ratio, avg_sent_len, sent_len_std, noun_ratio,
+verb_ratio, adj_ratio, adv_ratio, pron_ratio, pos_diversity, avg_tree_depth,
+max_tree_depth, sub_clause_ratio, dm_density, sent_len_cv, fp_ratio,
+num_sentences, words_per_sent, perplexity
+```
+
+`features.py` reproduces the training/testing pipeline exactly: `normalize_text`
+→ `sliding_window_chunk` (450-word windows, 350-word stride) → 24 spaCy features
++ GPT-2 perplexity → `StandardScaler.transform`. Document-level scores are the
+mean of per-chunk `P(machine)`. The original notebooks
+(`01_data_preprocessing_v2.py`, `02_feature_extraction.py`) are at
+https://github.com/foggyghost0/Hybrid-Machine-Learning-Approaches-for-Detection-of-LLM-Generated-Texts .
+
+## Notes
+
+- The checkpoint is a plain `state_dict`; load it with the `HybridClassifier`
+  in `model.py` (see `load_model()`).
+- Long documents were chunked to 512 tokens at training time; the reported
+  document-level score is the mean chunk probability per document.
+- `return_gate=True` in `forward()` also returns the fusion gate values and the
+  feature-attention weights for interpretability.

example_usage.py

@@ -0,0 +1,135 @@
+"""Runnable end-to-end example: score raw text with the hybrid detector.
+
+Unlike a placeholder demo, this extracts the 25 linguistic features *exactly* the
+way they were produced for training/testing, normalizes them with the fitted
+training scaler (``ling_scaler.pkl``), and aggregates chunk probabilities into a
+document-level score (mean chunk probability), which is how the thesis reports
+per-document predictions.
+
+Pipeline (see features.py for details):
+    raw text -> normalize -> sliding-window chunks -> per-chunk 25 features
+             -> StandardScaler.transform -> model -> mean chunk P(machine)
+
+Requirements:
+    pip install torch transformers spacy scikit-learn
+    python -m spacy download en_core_web_lg
+
+Run:
+    python example_usage.py
+    python example_usage.py --text "Some text to classify..."
+    python example_usage.py --file path/to/document.txt
+"""
+import argparse
+import pickle
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+from transformers import (
+    GPT2LMHeadModel,
+    GPT2TokenizerFast,
+    RobertaTokenizer,
+)
+
+from model import load_model, CONFIG, LING_FEATURE_NAMES
+from features import extract_raw_features, prepare_document, FEATURE_NAMES
+
+# Sanity check: features.py and model.py must agree on feature order.
+assert FEATURE_NAMES == LING_FEATURE_NAMES, "Feature order mismatch."
+
+DEFAULT_TEXT = (
+    "This is an example transcript to classify as human- or machine-written. "
+    "It is deliberately short, so it forms a single chunk. Provide your own "
+    "longer document via --text or --file to see multi-chunk aggregation."
+)
+
+
+def load_components(device):
+    """Load every model/resource needed to reproduce the test-time pipeline."""
+    print("Loading hybrid model ...")
+    model = load_model("hybrid_model_best.pt", device=device)
+
+    print("Loading RoBERTa tokenizer ...")
+    tokenizer = RobertaTokenizer.from_pretrained(CONFIG["roberta_model"])
+
+    print("Loading spaCy (en_core_web_lg) ...")
+    import spacy
+    try:
+        nlp = spacy.load("en_core_web_lg", disable=["ner"])
+    except OSError:
+        print("  en_core_web_lg not found -> downloading ...")
+        spacy.cli.download("en_core_web_lg")
+        nlp = spacy.load("en_core_web_lg", disable=["ner"])
+
+    print("Loading GPT-2 (perplexity) ...")
+    gpt2_tokenizer = GPT2TokenizerFast.from_pretrained("gpt2")
+    gpt2_model = GPT2LMHeadModel.from_pretrained("gpt2").to(device)
+    gpt2_model.eval()
+
+    print("Loading training feature scaler (ling_scaler.pkl) ...")
+    with open("ling_scaler.pkl", "rb") as f:
+        scaler = pickle.load(f)
+
+    return model, tokenizer, nlp, gpt2_model, gpt2_tokenizer, scaler
+
+
+def score_text(text, model, tokenizer, nlp, gpt2_model, gpt2_tokenizer, scaler, device):
+    """Return (doc_prob, per_chunk_probs) for a raw document."""
+    chunks = prepare_document(text)
+    if not chunks:
+        raise ValueError("Text is empty after normalization; nothing to score.")
+
+    chunk_probs = []
+    for chunk in chunks:
+        # 1. Extract the 25 raw features exactly as during training.
+        raw = extract_raw_features(chunk, nlp, gpt2_model, gpt2_tokenizer, device)
+        # 2. Normalize with the fitted training scaler.
+        norm = scaler.transform(raw.reshape(1, -1)).astype(np.float32)
+        ling = torch.from_numpy(norm).to(device)
+        # 3. Tokenize text for RoBERTa (same 512-token truncation as training).
+        enc = tokenizer(chunk, max_length=512, truncation=True, return_tensors="pt")
+        input_ids = enc["input_ids"].to(device)
+        attention_mask = enc["attention_mask"].to(device)
+        # 4. Forward pass -> P(machine) for this chunk.
+        with torch.no_grad():
+            logits = model(input_ids, attention_mask, ling)
+            p_llm = F.softmax(logits, dim=1)[0, 1].item()
+        chunk_probs.append(p_llm)
+
+    doc_prob = float(np.mean(chunk_probs))
+    return doc_prob, chunk_probs
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    src = parser.add_mutually_exclusive_group()
+    src.add_argument("--text", type=str, help="Raw text to classify.")
+    src.add_argument("--file", type=str, help="Path to a UTF-8 text file to classify.")
+    args = parser.parse_args()
+
+    if args.file:
+        with open(args.file, "r", encoding="utf-8") as f:
+            text = f.read()
+    elif args.text:
+        text = args.text
+    else:
+        text = DEFAULT_TEXT
+
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    print(f"Device: {device}\n")
+
+    components = load_components(device)
+    doc_prob, chunk_probs = score_text(text, *components, device=device)
+
+    print("\n" + "=" * 60)
+    print(f"Chunks scored: {len(chunk_probs)}")
+    for i, p in enumerate(chunk_probs):
+        print(f"  chunk {i:>2}: P(machine) = {p:.4f}")
+    print("-" * 60)
+    print(f"Document P(machine-generated) = {doc_prob:.4f}")
+    print(f"Prediction: {'machine-generated' if doc_prob >= 0.5 else 'human-written'}")
+    print("=" * 60)
+
+
+if __name__ == "__main__":
+    main()

features.py

@@ -0,0 +1,302 @@
+"""Self-contained linguistic feature extraction for the hybrid detector.
+
+This reproduces *exactly* the preprocessing + feature pipeline used to train and
+evaluate the released ``hybrid_model_best.pt`` checkpoint, so that a raw piece of
+text can be scored the same way it was during testing:
+
+    raw text
+      -> normalize_text()            (lowercase, strip HTML/markdown, collapse ws)
+      -> sliding_window_chunk()      (450-word windows, 350-word stride)
+      -> extract_raw_features()      (24 spaCy features + 1 GPT-2 perplexity = 25)
+      -> StandardScaler.transform()  (the fitted training scaler, ling_scaler.pkl)
+      -> model(...)                  (per chunk)
+      -> mean chunk probability      (document-level score)
+
+The 25 features, in order, match ``model.LING_FEATURE_NAMES``:
+
+    msttr, avg_word_len, hapax_ratio, function_ratio, punct_density, char_entropy,
+    burstiness, repetition_ratio, avg_sent_len, sent_len_std, noun_ratio,
+    verb_ratio, adj_ratio, adv_ratio, pron_ratio, pos_diversity, avg_tree_depth,
+    max_tree_depth, sub_clause_ratio, dm_density, sent_len_cv, fp_ratio,
+    num_sentences, words_per_sent, perplexity
+
+Requirements:
+    pip install torch transformers spacy scikit-learn
+    python -m spacy download en_core_web_lg
+"""
+import math
+import re
+from collections import Counter
+
+import numpy as np
+import torch
+
+# --------------------------------------------------------------------------- #
+# Feature order (must match model.LING_FEATURE_NAMES).
+# --------------------------------------------------------------------------- #
+FEATURE_NAMES = [
+    "msttr", "avg_word_len", "hapax_ratio", "function_ratio", "punct_density",
+    "char_entropy", "burstiness", "repetition_ratio",
+    "avg_sent_len", "sent_len_std", "noun_ratio", "verb_ratio", "adj_ratio",
+    "adv_ratio", "pron_ratio", "pos_diversity", "avg_tree_depth",
+    "max_tree_depth", "sub_clause_ratio",
+    "dm_density", "sent_len_cv", "fp_ratio", "num_sentences", "words_per_sent",
+    "perplexity",
+]
+
+# --------------------------------------------------------------------------- #
+# Preprocessing (mirrors 01_data_preprocessing_v2.py).
+# --------------------------------------------------------------------------- #
+WINDOW_SIZE = 450
+STRIDE = 350
+MIN_WINDOW_TOKENS = 50
+
+
+def normalize_text(text: str) -> str:
+    """Light normalization matching the training preprocessing.
+
+    Lowercase, strip HTML/markdown artifacts, normalize quotes, collapse
+    whitespace. Applied to every document before chunking / feature extraction.
+    """
+    if not text or not isinstance(text, str):
+        return ""
+
+    text = text.lower()
+    text = re.sub(r"<[^>]+>", " ", text)
+    text = re.sub(r"#{1,6}\s+", " ", text)
+    text = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", text)
+    text = re.sub(r"!\[[^\]]+\]\([^)]+\)", " ", text)
+    text = text.replace('"', '"').replace('"', '"')
+    text = text.replace("'", "'").replace("'", "'")
+    text = re.sub(r"\s+", " ", text).strip()
+    return text
+
+
+def sliding_window_chunk(text, window_size=WINDOW_SIZE, stride=STRIDE):
+    """Split text into overlapping word windows (same as training)."""
+    if not text:
+        return []
+
+    words = text.split()
+    total_words = len(words)
+
+    if total_words <= window_size:
+        return [text]
+
+    chunks = []
+    start = 0
+    while start < total_words:
+        end = min(start + window_size, total_words)
+        chunk_words = words[start:end]
+        if len(chunk_words) >= MIN_WINDOW_TOKENS:
+            chunks.append(" ".join(chunk_words))
+        start += stride
+        if end == total_words:
+            break
+    return chunks
+
+
+# --------------------------------------------------------------------------- #
+# Lexicons (identical to 02_feature_extraction.py).
+# --------------------------------------------------------------------------- #
+DISCOURSE_MARKERS = {
+    "however", "therefore", "moreover", "furthermore", "nevertheless",
+    "consequently", "meanwhile", "additionally", "similarly", "likewise",
+    "thus", "hence", "accordingly", "otherwise", "instead",
+    "first", "second", "third", "finally", "next", "then",
+    "in conclusion", "in summary", "to summarize", "overall",
+    "for example", "for instance", "specifically", "in particular",
+    "on the other hand", "in contrast", "conversely", "although",
+    "because", "since", "while", "whereas", "unless", "if",
+}
+
+FUNCTION_WORDS = {
+    "the", "a", "an", "and", "or", "but", "if", "then", "else",
+    "when", "where", "how", "what", "who", "which", "that", "this",
+    "is", "are", "was", "were", "be", "been", "being",
+    "have", "has", "had", "do", "does", "did",
+    "will", "would", "could", "should", "may", "might", "must",
+    "to", "of", "in", "for", "on", "with", "at", "by", "from",
+    "as", "into", "through", "during", "before", "after",
+    "above", "below", "between", "under", "over",
+    "i", "you", "he", "she", "it", "we", "they", "me", "him", "her", "us", "them",
+    "my", "your", "his", "her", "its", "our", "their",
+    "not", "no", "yes", "so", "very", "just", "also", "only",
+}
+
+SUB_MARKERS = {
+    "that", "which", "who", "whom", "whose", "where", "when", "while",
+    "because", "although", "if", "unless",
+}
+
+FIRST_PERSON = {
+    "i", "me", "my", "mine", "myself", "we", "us", "our", "ours", "ourselves",
+}
+
+
+# --------------------------------------------------------------------------- #
+# Feature helpers (identical to 02_feature_extraction.py).
+# --------------------------------------------------------------------------- #
+def get_tree_depth(token):
+    depth = 0
+    current = token
+    while current.head != current:
+        depth += 1
+        current = current.head
+        if depth > 100:
+            break
+    return depth
+
+
+def calculate_entropy(text):
+    if not text:
+        return 0.0
+    counts = Counter(text)
+    total = len(text)
+    probs = [c / total for c in counts.values()]
+    return -sum(p * math.log2(p) for p in probs)
+
+
+def calculate_burstiness(words):
+    if not words:
+        return 0.0
+    word_counts = list(Counter(words).values())
+    if not word_counts:
+        return 0.0
+    return np.std(word_counts) / np.mean(word_counts) if np.mean(word_counts) > 0 else 0.0
+
+
+def calculate_repetition_ratio(words):
+    if not words:
+        return 0.0
+    counts = Counter(words)
+    repeated = sum(c for w, c in counts.items() if c > 1)
+    return repeated / len(words)
+
+
+def calculate_msttr(words, window_size=50):
+    if not words:
+        return 0.0
+    if len(words) < window_size:
+        return len(set(words)) / len(words)
+    ttrs = []
+    for i in range(0, len(words), window_size):
+        segment = words[i:i + window_size]
+        if len(segment) == window_size:
+            ttrs.append(len(set(segment)) / len(segment))
+    return np.mean(ttrs) if ttrs else 0.0
+
+
+def calculate_pos_diversity(doc):
+    pos_counts = Counter([token.pos_ for token in doc])
+    total = len(doc)
+    if total == 0:
+        return 0.0
+    probs = [count / total for count in pos_counts.values()]
+    return -sum(p * math.log2(p) for p in probs)
+
+
+def calculate_perplexity(text, model, tokenizer, device):
+    """GPT-2 perplexity (single truncated window, matching training)."""
+    max_length = model.config.n_positions
+    encodings = tokenizer(text, return_tensors="pt", truncation=True, max_length=max_length)
+    seq_len = encodings.input_ids.size(1)
+    if seq_len < 2:
+        return 0.0
+    input_ids = encodings.input_ids.to(device)
+    with torch.no_grad():
+        outputs = model(input_ids, labels=input_ids)
+        neg_log_likelihood = outputs.loss
+    if neg_log_likelihood is None:
+        return 0.0
+    return torch.exp(neg_log_likelihood).item()
+
+
+def _cpu_features_from_doc(doc):
+    """The 24 spaCy-based features for a single spaCy Doc."""
+    text = doc.text
+    words = [token.text.lower() for token in doc if token.is_alpha]
+    sentences = list(doc.sents)
+
+    features = {}
+    if len(words) == 0:
+        return {}
+
+    features["msttr"] = calculate_msttr(words)
+    features["avg_word_len"] = np.mean([len(w) for w in words]) if words else 0
+
+    word_counts = Counter(words)
+    hapax = sum(1 for w, c in word_counts.items() if c == 1)
+    features["hapax_ratio"] = hapax / len(words) if words else 0
+
+    function_count = sum(1 for w in words if w in FUNCTION_WORDS)
+    features["function_ratio"] = function_count / len(words) if words else 0
+
+    punct_count = sum(1 for token in doc if token.is_punct)
+    features["punct_density"] = (punct_count / len(words)) * 100 if words else 0
+
+    features["char_entropy"] = calculate_entropy(text)
+    features["burstiness"] = calculate_burstiness(words)
+    features["repetition_ratio"] = calculate_repetition_ratio(words)
+
+    sent_lengths = [len([t for t in sent if t.is_alpha]) for sent in sentences]
+    features["avg_sent_len"] = np.mean(sent_lengths) if sent_lengths else 0
+    features["sent_len_std"] = np.std(sent_lengths) if len(sent_lengths) > 1 else 0
+
+    pos_counts = Counter([token.pos_ for token in doc])
+    total_tokens = len(doc)
+    features["noun_ratio"] = pos_counts.get("NOUN", 0) / total_tokens if total_tokens else 0
+    features["verb_ratio"] = pos_counts.get("VERB", 0) / total_tokens if total_tokens else 0
+    features["adj_ratio"] = pos_counts.get("ADJ", 0) / total_tokens if total_tokens else 0
+    features["adv_ratio"] = pos_counts.get("ADV", 0) / total_tokens if total_tokens else 0
+    features["pron_ratio"] = pos_counts.get("PRON", 0) / total_tokens if total_tokens else 0
+
+    features["pos_diversity"] = calculate_pos_diversity(doc)
+
+    depths = [get_tree_depth(token) for token in doc if token.dep_ != "punct"]
+    features["avg_tree_depth"] = np.mean(depths) if depths else 0
+    features["max_tree_depth"] = max(depths) if depths else 0
+
+    sub_count = sum(1 for token in doc if token.text.lower() in SUB_MARKERS)
+    features["sub_clause_ratio"] = sub_count / len(sentences) if sentences else 0
+
+    text_lower = text.lower()
+    dm_count = sum(1 for dm in DISCOURSE_MARKERS if dm in text_lower)
+    features["dm_density"] = (dm_count / len(words)) * 100 if words else 0
+
+    features["sent_len_cv"] = (
+        features["sent_len_std"] / features["avg_sent_len"]
+        if features["avg_sent_len"] > 0 else 0
+    )
+
+    fp_count = sum(1 for w in words if w in FIRST_PERSON)
+    features["fp_ratio"] = fp_count / len(words) if words else 0
+
+    features["num_sentences"] = len(sentences)
+    features["words_per_sent"] = len(words) / len(sentences) if sentences else 0
+
+    return features
+
+
+def extract_raw_features(text, nlp, gpt2_model, gpt2_tokenizer, device):
+    """Return the 25-dim *raw* (un-normalized) feature vector for one chunk.
+
+    ``text`` is expected to be a single normalized chunk (see normalize_text /
+    sliding_window_chunk). Order matches FEATURE_NAMES.
+    """
+    doc = nlp(text)
+    feats = _cpu_features_from_doc(doc)
+    ppl = calculate_perplexity(text, gpt2_model, gpt2_tokenizer, device)
+
+    vec = []
+    for name in FEATURE_NAMES:
+        if name == "perplexity":
+            vec.append(ppl)
+        else:
+            vec.append(feats.get(name, 0.0))
+    return np.array(vec, dtype=np.float32)
+
+
+def prepare_document(text):
+    """Normalize a raw document and split it into the chunks used at test time."""
+    return sliding_window_chunk(normalize_text(text))

hybrid_model_best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2d8dae1640731018bdae408478a118f80b83eb36d626c9a6491d7035732dc54
+size 506593731

ling_scaler.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07bc0594c10eab762389550c377296c28cd6d8c8785fc4b7dc9742fd80c7c877
+size 1050

model.py

@@ -0,0 +1,115 @@
+"""Self-contained definition of the hybrid RoBERTa + linguistic-features model.
+
+The released checkpoint ``hybrid_model_best.pt`` is a plain PyTorch
+``state_dict`` for the ``HybridClassifier`` defined here. Import this module to
+rebuild the architecture and load the weights.
+"""
+import torch
+import torch.nn as nn
+from transformers import RobertaModel
+
+CONFIG = {
+    "roberta_model": "roberta-base",
+    "roberta_dim": 768,
+    "ling_dim": 25,
+    "ling_hidden": 256,
+    "hidden_dim": 768,
+    "dropout": 0.15,
+    "freeze_bottom_layers": 6,
+}
+
+LING_FEATURE_NAMES = [
+    "msttr", "avg_word_len", "hapax_ratio", "function_ratio", "punct_density",
+    "char_entropy", "burstiness", "repetition_ratio", "avg_sent_len",
+    "sent_len_std", "noun_ratio", "verb_ratio", "adj_ratio", "adv_ratio",
+    "pron_ratio", "pos_diversity", "avg_tree_depth", "max_tree_depth",
+    "sub_clause_ratio", "dm_density", "sent_len_cv", "fp_ratio",
+    "num_sentences", "words_per_sent", "perplexity",
+]
+
+
+class FeatureAttention(nn.Module):
+    def __init__(self, num_features):
+        super().__init__()
+        self.attention = nn.Sequential(
+            nn.Linear(num_features, num_features),
+            nn.Tanh(),
+            nn.Linear(num_features, num_features),
+            nn.Softmax(dim=-1),
+        )
+
+    def forward(self, features):
+        attn_weights = self.attention(features)
+        weighted_features = features * attn_weights
+        return weighted_features, attn_weights
+
+
+class GatedFusion(nn.Module):
+    def __init__(self, dim):
+        super().__init__()
+        self.gate = nn.Sequential(
+            nn.Linear(dim * 2, dim),
+            nn.Sigmoid(),
+        )
+
+    def forward(self, cls_emb, ling_emb):
+        combined = torch.cat([cls_emb, ling_emb], dim=-1)
+        g = self.gate(combined)
+        fused = g * cls_emb + (1 - g) * ling_emb
+        return fused, g
+
+
+class HybridClassifier(nn.Module):
+    """RoBERTa CLS embedding gated-fused with attended linguistic features."""
+
+    def __init__(self, config=CONFIG):
+        super().__init__()
+        self.roberta = RobertaModel.from_pretrained(config["roberta_model"])
+        if config["freeze_bottom_layers"] > 0:
+            for param in self.roberta.embeddings.parameters():
+                param.requires_grad = False
+            for i in range(config["freeze_bottom_layers"]):
+                for param in self.roberta.encoder.layer[i].parameters():
+                    param.requires_grad = False
+        self.feature_attention = FeatureAttention(config["ling_dim"])
+        self.ling_projection = nn.Sequential(
+            nn.Linear(config["ling_dim"], config["ling_hidden"]),
+            nn.LayerNorm(config["ling_hidden"]),
+            nn.GELU(),
+            nn.Dropout(config["dropout"]),
+            nn.Linear(config["ling_hidden"], config["roberta_dim"]),
+            nn.LayerNorm(config["roberta_dim"]),
+            nn.GELU(),
+            nn.Dropout(config["dropout"]),
+        )
+        self.gated_fusion = GatedFusion(config["roberta_dim"])
+        self.classifier = nn.Sequential(
+            nn.Linear(config["roberta_dim"], config["hidden_dim"]),
+            nn.GELU(),
+            nn.Dropout(config["dropout"]),
+            nn.Linear(config["hidden_dim"], 2),
+        )
+        self.dropout = nn.Dropout(config["dropout"])
+        self.config = config
+
+    def forward(self, input_ids, attention_mask, ling_features, return_gate=False):
+        outputs = self.roberta(input_ids=input_ids, attention_mask=attention_mask)
+        cls_embedding = outputs.last_hidden_state[:, 0, :]
+        cls_embedding = self.dropout(cls_embedding)
+        attended_features, attn_weights = self.feature_attention(ling_features)
+        ling_proj = self.ling_projection(attended_features)
+        fused, gate_values = self.gated_fusion(cls_embedding, ling_proj)
+        logits = self.classifier(fused)
+        if return_gate:
+            return logits, gate_values, attn_weights
+        return logits
+
+
+def load_model(checkpoint_path="hybrid_model_best.pt", device="cpu"):
+    """Build the model and load the released weights. Returns an eval-mode model."""
+    model = HybridClassifier(CONFIG)
+    state_dict = torch.load(checkpoint_path, map_location=device)
+    model.load_state_dict(state_dict)
+    model.to(device)
+    model.eval()
+    return model