Initial deploy — AI text detector API

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,6 @@
+venv/
+__pycache__/
+*.pyc
+*.pyo
+.env
+*.egg-info/

README.md

@@ -0,0 +1,11 @@
+---
+title: AIdetector
+emoji: 🏃
+colorFrom: red
+colorTo: pink
+sdk: docker
+pinned: false
+license: mit
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

api/main.py

@@ -0,0 +1,75 @@
+# api/main.py
+# Application entry point.
+# Handles startup (model loading), CORS, and router registration.
+
+import logging
+from contextlib import asynccontextmanager
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from api.routes import router
+from core.bert_scorer import bert_scorer
+from config.settings import settings
+from core.perplexity_scorer import perplexity_scorer
+
+# Configure logging format
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
+)
+logger = logging.getLogger(__name__)
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """
+    Lifespan context manager — runs setup before the app starts
+    accepting requests, and cleanup when it shuts down.
+    
+    We load the model HERE, not on first request, so the first
+    user doesn't wait 10 seconds for a cold start.
+    """
+    logger.info("Starting up — loading model...")
+    bert_scorer.load()
+    perplexity_scorer.load()
+    logger.info("Model ready. API is live.")
+    
+    yield  # App runs here
+    
+    # Shutdown cleanup (if needed in future)
+    logger.info("Shutting down.")
+
+
+# Create FastAPI app
+app = FastAPI(
+    title="AI Text Detector API",
+    description=(
+        "Detects AI-generated text using a hybrid approach: "
+        "BERT-based neural classification + stylometric feature analysis. "
+        "Output is a calibrated probability — NOT a definitive verdict."
+    ),
+    version="1.0.0",
+    lifespan=lifespan,
+)
+
+# CORS — allow frontend to call this API
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=settings.ALLOWED_ORIGINS,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Register routes under /api prefix
+app.include_router(router, prefix="/api")
+
+
+# Root endpoint
+@app.get("/")
+async def root():
+    return {
+        "name": "AI Text Detector API",
+        "version": "1.0.0",
+        "docs": "/docs",
+        "health": "/api/health",
+    }

api/routes.py

@@ -0,0 +1,121 @@
+# api/routes.py
+# API endpoints — thin layer that validates input, calls core logic,
+# returns structured responses. No business logic lives here.
+
+import logging
+from fastapi import APIRouter, HTTPException
+from api.schemas import (
+    AnalyzeRequest, AnalyzeResponse,
+    BatchAnalyzeRequest, BatchAnalyzeResponse,
+    HealthResponse,
+)
+from core.preprocessor import preprocess, split_sentences
+from core.bert_scorer import bert_scorer
+from core.aggregator import aggregate_scores
+from config.settings import settings
+
+# Set up logging — every request is logged for debugging
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+@router.get("/health", response_model=HealthResponse)
+async def health_check():
+    """
+    Health check endpoint.
+    Render/Railway ping this to know if the service is alive.
+    """
+    return HealthResponse(
+        status="ok",
+        model_loaded=bert_scorer._pipeline is not None,
+        model_id=settings.MODEL_ID,
+        environment=settings.ENVIRONMENT,
+    )
+
+
+@router.post("/analyze", response_model=AnalyzeResponse)
+async def analyze_text(request: AnalyzeRequest):
+    """
+    Main endpoint — analyze a single text for AI authorship.
+    
+    Steps:
+    1. Clean and validate input
+    2. Score full text with BERT
+    3. Score each sentence individually
+    4. Compute stylometric features
+    5. Aggregate into final score + explanation
+    """
+    try:
+        # Step 1: Clean text
+        cleaned_text = preprocess(request.text)
+        logger.info(f"Analyzing text of length {len(cleaned_text)}")
+        
+        # Step 2: BERT score on full text
+        bert_result = bert_scorer.score_text(cleaned_text)
+        
+        # Step 3: Sentence-level scores (for highlighting)
+        sentences = split_sentences(
+            cleaned_text,
+            min_length=settings.MIN_SENTENCE_LENGTH,
+        )
+        # Cap sentences to avoid slow responses on very long texts
+        sentences = sentences[:settings.MAX_SENTENCES]
+        sentence_scores = bert_scorer.score_sentences(sentences)
+        
+        # Step 4 + 5: Aggregate everything
+        result = aggregate_scores(
+            bert_result=bert_result,
+            text=cleaned_text,
+            sentence_scores=sentence_scores,
+        )
+        
+        # Add metadata
+        result["word_count"] = len(cleaned_text.split())
+        result["character_count"] = len(cleaned_text)
+        
+        return AnalyzeResponse(**result)
+    
+    except ValueError as e:
+        # Input validation errors → 400
+        raise HTTPException(status_code=400, detail=str(e))
+    
+    except Exception as e:
+        # Unexpected errors → 500 with logging
+        logger.error(f"Analysis failed: {e}", exc_info=True)
+        raise HTTPException(
+            status_code=500,
+            detail="Analysis failed. Please try again."
+        )
+
+
+@router.post("/batch", response_model=BatchAnalyzeResponse)
+async def batch_analyze(request: BatchAnalyzeRequest):
+    """
+    Batch endpoint — analyze multiple texts in one call.
+    Max 20 texts per request to prevent timeout on free tier hosting.
+    """
+    results = []
+    
+    for text in request.texts:
+        try:
+            cleaned = preprocess(text)
+            bert_result = bert_scorer.score_text(cleaned)
+            sentences = split_sentences(cleaned, min_length=settings.MIN_SENTENCE_LENGTH)
+            sentences = sentences[:settings.MAX_SENTENCES]
+            sentence_scores = bert_scorer.score_sentences(sentences)
+            result = aggregate_scores(
+                bert_result=bert_result,
+                text=cleaned,
+                sentence_scores=sentence_scores,
+            )
+            result["word_count"] = len(cleaned.split())
+            result["character_count"] = len(cleaned)
+            results.append(AnalyzeResponse(**result))
+        
+        except Exception as e:
+            logger.error(f"Batch item failed: {e}")
+            # Skip failed items — don't crash entire batch
+            continue
+    
+    return BatchAnalyzeResponse(results=results, total=len(results))

api/schemas.py

@@ -0,0 +1,92 @@
+# api/schemas.py
+# Pydantic models define the exact shape of every request and response.
+# This gives us automatic validation + auto-generated API docs for free.
+
+from pydantic import BaseModel, Field, field_validator
+from typing import Optional
+
+
+class AnalyzeRequest(BaseModel):
+    text: str = Field(
+        ...,
+        min_length=20,
+        max_length=50000,
+        description="Text to analyze for AI authorship",
+    )
+    
+    @field_validator("text")
+    @classmethod
+    def text_must_not_be_empty(cls, v):
+        if not v.strip():
+            raise ValueError("Text cannot be empty or whitespace only.")
+        return v
+
+
+class BatchAnalyzeRequest(BaseModel):
+    texts: list[str] = Field(
+        ...,
+        min_length=1,
+        max_length=20,
+        description="List of texts to analyze (max 20)",
+    )
+
+
+class SentenceScore(BaseModel):
+    sentence: str
+    ai_probability: float
+    reliable: bool
+
+
+class ExplanationReason(BaseModel):
+    signal: str
+    detail: str
+    weight: str  # "high", "medium", "low"
+
+
+class Explanation(BaseModel):
+    verdict: str
+    ai_probability: float
+    reasons: list[ExplanationReason]
+    top_suspicious_sentences: list[SentenceScore]
+    uncertainty_note: str
+
+
+class ScoreBreakdown(BaseModel):
+    bert_score: float
+    stylometric_score: float
+    bert_weight: float
+    stylometric_weight: float
+
+
+class AnalyzeResponse(BaseModel):
+    ai_probability: float = Field(
+        description="Final calibrated probability (0-1) that text is AI-generated"
+    )
+    is_ai: bool = Field(
+        description="True if probability exceeds confidence threshold"
+    )
+    verdict: str = Field(
+        description="Human-readable classification label"
+    )
+    confidence_threshold: float
+    scores: ScoreBreakdown
+    stylometric_features: dict
+    sentence_scores: list[SentenceScore]
+    explanation: Explanation
+    chunks_analyzed: int
+    word_count: int
+    character_count: int
+
+
+class BatchAnalyzeResponse(BaseModel):
+    results: list[AnalyzeResponse]
+    total: int
+
+
+class HealthResponse(BaseModel):
+    model_config = {"protected_namespaces": ()}  # ← add this line
+    
+    status: str
+    model_loaded: bool
+    model_id: str
+    environment: str

config/settings.py

@@ -0,0 +1,25 @@
+from pathlib import Path
+from dotenv import load_dotenv
+import os
+
+load_dotenv(Path(__file__).parent.parent / ".env")
+
+class Settings:
+    MODEL_ID: str = os.getenv("MODEL_ID", "gouwsxander/slop-detector-bert")
+    MAX_TOKENS: int = int(os.getenv("MAX_TOKENS", 500))
+    CONFIDENCE_THRESHOLD: float = 0.68
+    MIN_SENTENCE_LENGTH: int = 20
+    MAX_SENTENCES: int = 40
+    TEMPERATURE: float = 2.5   # ← make sure this is 2.5
+    ENVIRONMENT: str = os.getenv("ENVIRONMENT", "development")
+    ALLOWED_ORIGINS: list = [
+    "http://localhost:3000",
+    "http://localhost:5500",
+    "http://127.0.0.1:5500",
+    "http://localhost:5501",
+    "http://127.0.0.1:5501",
+    "https://*.vercel.app",
+    "https://*.hf.space",
+    "https://huggingface.co",
+]
+settings = Settings()

core/aggregator.py

@@ -0,0 +1,60 @@
+# core/aggregator.py
+# Combines BERT score + stylometric score into a final probability.
+
+from core.stylometrics import compute_stylometric_score
+from core.explainer import find_ai_phrases, generate_explanation
+from config.settings import settings
+
+
+def aggregate_scores(
+    bert_result: dict,
+    text: str,
+    sentence_scores: list[dict],
+) -> dict:
+    """
+    Two-signal hybrid:
+    - BERT (70%): neural, trained on Wikipedia pairs
+    - Stylometrics (30%): model-agnostic structural features
+    """
+    stylo_result = compute_stylometric_score(text)
+    stylo_score = stylo_result["stylometric_ai_score"]
+
+    bert_prob = bert_result["ai_probability"]
+
+    bert_weight  = 0.70
+    stylo_weight = 0.30
+
+    final_probability = (
+        bert_prob   * bert_weight +
+        stylo_score * stylo_weight
+    )
+
+    final_probability = round(max(0.0, min(1.0, final_probability)), 4)
+
+    ai_phrases = find_ai_phrases(text)
+
+    explanation = generate_explanation(
+        ai_probability=final_probability,
+        stylometric_data=stylo_result,
+        sentence_scores=sentence_scores,
+        ai_phrases=ai_phrases,
+    )
+
+    is_ai = final_probability >= settings.CONFIDENCE_THRESHOLD
+
+    return {
+        "ai_probability": final_probability,
+        "is_ai": is_ai,
+        "verdict": explanation["verdict"],
+        "confidence_threshold": settings.CONFIDENCE_THRESHOLD,
+        "scores": {
+            "bert_score": bert_prob,
+            "stylometric_score": round(stylo_score, 4),
+            "bert_weight": bert_weight,
+            "stylometric_weight": stylo_weight,
+        },
+        "stylometric_features": stylo_result["features"],
+        "sentence_scores": sentence_scores,
+        "explanation": explanation,
+        "chunks_analyzed": bert_result.get("chunks_analyzed", 1),
+    }

core/bert_scorer.py

@@ -0,0 +1,428 @@
+# core/bert_scorer.py
+# Loads slop-detector-bert by merging LoRA weights directly into BERT.
+# Key findings from adapter_config.json:
+#   - lora_alpha=16, r=16, scaling=1.0
+#   - modules_to_save=["classifier"] — saved with different key format
+#   - LABEL_0 = AI, LABEL_1 = Human (inverted from model card)
+
+import numpy as np
+import torch
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+from huggingface_hub import hf_hub_download
+from safetensors.torch import load_file
+from config.settings import settings
+from core.preprocessor import chunk_for_bert
+
+
+class BertScorer:
+
+    def __init__(self):
+        self.model_id = settings.MODEL_ID
+        self.max_tokens = settings.MAX_TOKENS
+        self.temperature = settings.TEMPERATURE
+        self._model = None
+        self._tokenizer = None
+
+    def load(self):
+        print(f"Loading model: {self.model_id}")
+
+        # Load tokenizer from base model
+        self._tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+
+        # Load base BERT with classification head
+        self._model = AutoModelForSequenceClassification.from_pretrained(
+            "bert-base-cased",
+            num_labels=2,
+            ignore_mismatched_sizes=True,
+        )
+
+        # Download and apply LoRA adapter weights
+        try:
+            adapter_path = hf_hub_download(
+                repo_id=self.model_id,
+                filename="adapter_model.safetensors",
+            )
+            print(f"Adapter downloaded: {adapter_path}")
+            self._apply_lora_weights(adapter_path)
+        except Exception as e:
+            print(f"Warning: Could not load adapter weights: {e}")
+            print("Running with base BERT — scores will be less accurate.")
+
+        self._model.eval()
+        print("Model loaded successfully.")
+
+    def _apply_lora_weights(self, adapter_path: str):
+        """
+        Merge LoRA weights into base model.
+
+        Key format in safetensors:
+          LoRA:       base_model.model.bert.encoder.layer.X....lora_A.weight
+          Classifier: base_model.model.classifier.modules_to_save.default.weight
+                      base_model.model.classifier.modules_to_save.default.bias
+
+        Scaling = lora_alpha / r = 16 / 16 = 1.0
+
+        LABEL_0 = AI-generated
+        LABEL_1 = Human-written
+        (confirmed by empirical testing — inverted from model card)
+        """
+        lora_weights = load_file(adapter_path)
+
+        # Print all keys for debugging
+        classifier_keys = [k for k in lora_weights.keys() if "classifier" in k]
+        print(f"Classifier keys found: {classifier_keys}")
+
+        # Collect lora_A and lora_B tensors
+        lora_A = {}
+        lora_B = {}
+
+        for key, tensor in lora_weights.items():
+            if key.endswith("lora_A.weight"):
+                base = key.replace("base_model.model.", "").replace(".lora_A.weight", "")
+                lora_A[base] = tensor
+            elif key.endswith("lora_B.weight"):
+                base = key.replace("base_model.model.", "").replace(".lora_B.weight", "")
+                lora_B[base] = tensor
+
+        # scaling = lora_alpha / r = 16 / 16 = 1.0
+        scaling = 16 / 16
+
+        # Get current model state dict
+        state_dict = self._model.state_dict()
+        merged = 0
+
+        # Merge: W = W_base + scaling * (B @ A)
+        for base_key in lora_A:
+            if base_key in lora_B:
+                weight_key = base_key + ".weight"
+                if weight_key in state_dict:
+                    A = lora_A[base_key].float()
+                    B = lora_B[base_key].float()
+                    delta = scaling * (B @ A)
+                    state_dict[weight_key] = state_dict[weight_key].float() + delta
+                    merged += 1
+
+        print(f"Merged {merged} LoRA layers into base model.")
+
+        # Load classifier weights
+        # Format: base_model.model.classifier.modules_to_save.default.weight/bias
+        for key, tensor in lora_weights.items():
+            if "classifier" in key and "modules_to_save" in key:
+                # Map to model's classifier.weight / classifier.bias
+                if key.endswith(".weight"):
+                    state_dict["classifier.weight"] = tensor.float()
+                    print("Loaded classifier.weight")
+                elif key.endswith(".bias"):
+                    state_dict["classifier.bias"] = tensor.float()
+                    print("Loaded classifier.bias")
+
+        self._model.load_state_dict(state_dict)
+
+    def _temperature_scale(self, prob: float) -> float:
+        """
+        Calibrate raw softmax probability.
+        T=1.0 = no change. T>1.0 softens overconfident predictions.
+        """
+        p = max(min(prob, 0.9999), 0.0001)
+        raw_logit = np.log(p / (1 - p))
+        scaled_logit = raw_logit / self.temperature
+        return float(1 / (1 + np.exp(-scaled_logit)))
+
+    def _predict(self, text: str) -> float:
+        """
+        Run single inference pass.
+        Returns probability that text is AI-generated.
+        LABEL_0 = AI, LABEL_1 = Human
+        So we return probs[0][0] for AI probability.
+        """
+        inputs = self._tokenizer(
+            text,
+            return_tensors="pt",
+            truncation=True,
+            max_length=self.max_tokens,
+            padding=True,
+        )
+
+        with torch.no_grad():
+            outputs = self._model(**inputs)
+
+        probs = torch.softmax(outputs.logits, dim=-1)
+
+        # LABEL_0 = AI probability
+        ai_prob = probs[0][0].item()
+        return ai_prob
+
+    def score_text(self, text: str) -> dict:
+        """
+        Score full text. Chunks if longer than max_tokens.
+        Returns calibrated AI probability.
+        """
+        if self._model is None:
+            raise RuntimeError("Model not loaded. Call load() first.")
+
+        chunks = chunk_for_bert(text, self._tokenizer, self.max_tokens)
+        chunk_scores = []
+
+        for chunk_text in chunks:
+            raw_prob = self._predict(chunk_text)
+            chunk_scores.append(raw_prob)
+
+        raw_ai_probability = float(np.mean(chunk_scores))
+        calibrated_probability = self._temperature_scale(raw_ai_probability)
+
+        return {
+            "raw_ai_probability": round(raw_ai_probability, 4),
+            "ai_probability": round(calibrated_probability, 4),
+            "chunks_analyzed": len(chunks),
+        }
+
+    def score_sentences(self, sentences: list[str]) -> list[dict]:
+        """
+        Score each sentence individually for frontend highlighting.
+        """
+        if self._model is None:
+            raise RuntimeError("Model not loaded. Call load() first.")
+
+        results = []
+
+        for sentence in sentences:
+            if len(sentence) < settings.MIN_SENTENCE_LENGTH:
+                results.append({
+                    "sentence": sentence,
+                    "ai_probability": 0.5,
+                    "reliable": False,
+                })
+                continue
+
+            try:
+                raw_prob = self._predict(sentence)
+                calibrated = self._temperature_scale(raw_prob)
+                results.append({
+                    "sentence": sentence,
+                    "ai_probability": round(calibrated, 4),
+                    "reliable": True,
+                })
+            except Exception:
+                results.append({
+                    "sentence": sentence,
+                    "ai_probability": 0.5,
+                    "reliable": False,
+                })
+
+        return results
+
+
+# Module-level singleton — imported by routes
+bert_scorer = BertScorer()# core/bert_scorer.py
+# Loads slop-detector-bert by merging LoRA weights directly into BERT.
+# Key findings from adapter_config.json:
+#   - lora_alpha=16, r=16, scaling=1.0
+#   - modules_to_save=["classifier"] — saved with different key format
+#   - LABEL_0 = AI, LABEL_1 = Human (inverted from model card)
+
+import numpy as np
+import torch
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+from huggingface_hub import hf_hub_download
+from safetensors.torch import load_file
+from config.settings import settings
+from core.preprocessor import chunk_for_bert
+
+
+class BertScorer:
+
+    def __init__(self):
+        self.model_id = settings.MODEL_ID
+        self.max_tokens = settings.MAX_TOKENS
+        self.temperature = settings.TEMPERATURE
+        self._model = None
+        self._tokenizer = None
+
+    def load(self):
+        print(f"Loading model: {self.model_id}")
+
+        # Load tokenizer from base model
+        self._tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+
+        # Load base BERT with classification head
+        self._model = AutoModelForSequenceClassification.from_pretrained(
+            "bert-base-cased",
+            num_labels=2,
+            ignore_mismatched_sizes=True,
+        )
+
+        # Download and apply LoRA adapter weights
+        try:
+            adapter_path = hf_hub_download(
+                repo_id=self.model_id,
+                filename="adapter_model.safetensors",
+            )
+            print(f"Adapter downloaded: {adapter_path}")
+            self._apply_lora_weights(adapter_path)
+        except Exception as e:
+            print(f"Warning: Could not load adapter weights: {e}")
+            print("Running with base BERT — scores will be less accurate.")
+
+        self._model.eval()
+        print("Model loaded successfully.")
+
+    def _apply_lora_weights(self, adapter_path: str):
+        """
+        Merge LoRA weights into base model.
+
+        Key format in safetensors:
+          LoRA:       base_model.model.bert.encoder.layer.X....lora_A.weight
+          Classifier: base_model.model.classifier.modules_to_save.default.weight
+                      base_model.model.classifier.modules_to_save.default.bias
+
+        Scaling = lora_alpha / r = 16 / 16 = 1.0
+
+        LABEL_0 = AI-generated
+        LABEL_1 = Human-written
+        (confirmed by empirical testing — inverted from model card)
+        """
+        lora_weights = load_file(adapter_path)
+
+        # Print all keys for debugging
+        classifier_keys = [k for k in lora_weights.keys() if "classifier" in k]
+        print(f"Classifier keys found: {classifier_keys}")
+
+        # Collect lora_A and lora_B tensors
+        lora_A = {}
+        lora_B = {}
+
+        for key, tensor in lora_weights.items():
+            if key.endswith("lora_A.weight"):
+                base = key.replace("base_model.model.", "").replace(".lora_A.weight", "")
+                lora_A[base] = tensor
+            elif key.endswith("lora_B.weight"):
+                base = key.replace("base_model.model.", "").replace(".lora_B.weight", "")
+                lora_B[base] = tensor
+
+        # scaling = lora_alpha / r = 16 / 16 = 1.0
+        scaling = 16 / 16
+
+        # Get current model state dict
+        state_dict = self._model.state_dict()
+        merged = 0
+
+        # Merge: W = W_base + scaling * (B @ A)
+        for base_key in lora_A:
+            if base_key in lora_B:
+                weight_key = base_key + ".weight"
+                if weight_key in state_dict:
+                    A = lora_A[base_key].float()
+                    B = lora_B[base_key].float()
+                    delta = scaling * (B @ A)
+                    state_dict[weight_key] = state_dict[weight_key].float() + delta
+                    merged += 1
+
+        print(f"Merged {merged} LoRA layers into base model.")
+
+        # Load classifier weights
+        # Format: base_model.model.classifier.weight / bias
+        for key, tensor in lora_weights.items():
+            if "classifier" in key:
+                clean_key = key.replace("base_model.model.", "")
+                if clean_key in state_dict:
+                    state_dict[clean_key] = tensor.float()
+                    print(f"Loaded {clean_key} {tensor.shape}")
+
+        self._model.load_state_dict(state_dict)
+
+    def _temperature_scale(self, prob: float) -> float:
+        """
+        Calibrate raw softmax probability.
+        T=1.0 = no change. T>1.0 softens overconfident predictions.
+        """
+        p = max(min(prob, 0.9999), 0.0001)
+        raw_logit = np.log(p / (1 - p))
+        scaled_logit = raw_logit / self.temperature
+        return float(1 / (1 + np.exp(-scaled_logit)))
+
+    def _predict(self, text: str) -> float:
+        """
+        Run single inference pass.
+        Returns probability that text is AI-generated.
+        LABEL_0 = AI, LABEL_1 = Human
+        So we return probs[0][0] for AI probability.
+        """
+        inputs = self._tokenizer(
+            text,
+            return_tensors="pt",
+            truncation=True,
+            max_length=self.max_tokens,
+            padding=True,
+        )
+
+        with torch.no_grad():
+            outputs = self._model(**inputs)
+
+        probs = torch.softmax(outputs.logits, dim=-1)
+
+        # LABEL_0 = AI probability
+        ai_prob = probs[0][0].item()
+        return ai_prob
+
+    def score_text(self, text: str) -> dict:
+        """
+        Score full text. Chunks if longer than max_tokens.
+        Returns calibrated AI probability.
+        """
+        if self._model is None:
+            raise RuntimeError("Model not loaded. Call load() first.")
+
+        chunks = chunk_for_bert(text, self._tokenizer, self.max_tokens)
+        chunk_scores = []
+
+        for chunk_text in chunks:
+            raw_prob = self._predict(chunk_text)
+            chunk_scores.append(raw_prob)
+
+        raw_ai_probability = float(np.mean(chunk_scores))
+        calibrated_probability = self._temperature_scale(raw_ai_probability)
+
+        return {
+            "raw_ai_probability": round(raw_ai_probability, 4),
+            "ai_probability": round(calibrated_probability, 4),
+            "chunks_analyzed": len(chunks),
+        }
+
+    def score_sentences(self, sentences: list[str]) -> list[dict]:
+        """
+        Score each sentence individually for frontend highlighting.
+        """
+        if self._model is None:
+            raise RuntimeError("Model not loaded. Call load() first.")
+
+        results = []
+
+        for sentence in sentences:
+            if len(sentence) < settings.MIN_SENTENCE_LENGTH:
+                results.append({
+                    "sentence": sentence,
+                    "ai_probability": 0.5,
+                    "reliable": False,
+                })
+                continue
+
+            try:
+                raw_prob = self._predict(sentence)
+                calibrated = self._temperature_scale(raw_prob)
+                results.append({
+                    "sentence": sentence,
+                    "ai_probability": round(calibrated, 4),
+                    "reliable": True,
+                })
+            except Exception:
+                results.append({
+                    "sentence": sentence,
+                    "ai_probability": 0.5,
+                    "reliable": False,
+                })
+
+        return results
+
+
+# Module-level singleton — imported by routes
+bert_scorer = BertScorer()

core/explainer.py

@@ -0,0 +1,184 @@
+# core/explainer.py
+# Generates human-readable explanations for WHY text was flagged.
+# This is what separates a production tool from a tutorial project.
+
+from config.settings import settings
+
+
+# AI writing markers — phrases that appear disproportionately in LLM output.
+# These were identified by analyzing the dataset manually.
+AI_PHRASES = [
+    "it is worth noting",
+    "it is important to note",
+    "in the context of",
+    "plays a crucial role",
+    "plays an important role",
+    "serves as a",
+    "as a result of",
+    "in order to",
+    "due to the fact that",
+    "with respect to",
+    "in terms of",
+    "as mentioned",
+    "furthermore",
+    "moreover",
+    "additionally",
+    "consequently",
+    "nevertheless",
+    "it should be noted",
+    "it can be seen",
+    "in conclusion",
+    "to summarize",
+    "overall,",
+    "notably,",
+    "significantly,",
+    "importantly,",
+    "reflects",
+    "highlights",
+    "underscores",
+    "illustrates",
+]
+
+
+def find_ai_phrases(text: str) -> list[str]:
+    """Find known AI marker phrases in the text."""
+    text_lower = text.lower()
+    found = [phrase for phrase in AI_PHRASES if phrase in text_lower]
+    return found
+
+
+def generate_explanation(
+    ai_probability: float,
+    stylometric_data: dict,
+    sentence_scores: list[dict],
+    ai_phrases: list[str],
+) -> dict:
+    """
+    Builds a structured explanation object that the frontend renders.
+    
+    Design principle: always explain uncertainty.
+    We never say "this IS AI" — we say "these signals suggest AI".
+    """
+    
+    reasons = []
+    confidence_label = _confidence_label(ai_probability)
+    features = stylometric_data.get("features", {})
+    feature_scores = stylometric_data.get("feature_scores", {})
+    
+    # --- Reason 1: BERT model signal ---
+    if ai_probability > 0.75:
+        reasons.append({
+            "signal": "Neural classifier",
+            "detail": f"The AI detection model assigned a {ai_probability:.0%} probability of AI authorship.",
+            "weight": "high",
+        })
+    elif ai_probability > 0.55:
+        reasons.append({
+            "signal": "Neural classifier",
+            "detail": f"The AI detection model found moderate signals of AI authorship ({ai_probability:.0%}).",
+            "weight": "medium",
+        })
+    
+    # --- Reason 2: Burstiness (sentence length variation) ---
+    burst = features.get("burstiness", 0.5)
+    if burst < 0.25:
+        reasons.append({
+            "signal": "Uniform sentence rhythm",
+            "detail": f"Sentence lengths are unusually uniform (burstiness: {burst:.2f}). Human writing varies more.",
+            "weight": "high",
+        })
+    elif burst < 0.35:
+        reasons.append({
+            "signal": "Low sentence variation",
+            "detail": f"Sentence lengths show limited variation (burstiness: {burst:.2f}).",
+            "weight": "medium",
+        })
+    
+    # --- Reason 3: Type-token ratio (vocabulary diversity) ---
+    ttr = features.get("type_token_ratio", 0.7)
+    if ttr < 0.5:
+        reasons.append({
+            "signal": "Limited vocabulary diversity",
+            "detail": f"The text reuses words frequently (TTR: {ttr:.2f}). AI text tends to be lexically repetitive.",
+            "weight": "medium",
+        })
+    
+    # --- Reason 4: AI marker phrases ---
+    if len(ai_phrases) >= 3:
+        phrase_list = ", ".join(f'"{p}"' for p in ai_phrases[:4])
+        reasons.append({
+            "signal": "AI marker phrases",
+            "detail": f"Found {len(ai_phrases)} common AI phrases: {phrase_list}.",
+            "weight": "medium",
+        })
+    elif len(ai_phrases) >= 1:
+        phrase_list = ", ".join(f'"{p}"' for p in ai_phrases[:2])
+        reasons.append({
+            "signal": "AI transition words",
+            "detail": f"Found phrases common in AI text: {phrase_list}.",
+            "weight": "low",
+        })
+    
+    # --- Reason 5: Repetition ---
+    rep = features.get("repetition_score", 0)
+    if rep > 0.1:
+        reasons.append({
+            "signal": "Phrase repetition",
+            "detail": f"Repeated phrases detected (score: {rep:.2f}). LLMs often loop back to the same constructions.",
+            "weight": "medium",
+        })
+    
+    # --- Highlight the most suspicious sentences ---
+    suspicious = [
+        s for s in sentence_scores
+        if s.get("ai_probability", 0) > 0.7 and s.get("reliable", False)
+    ]
+    suspicious.sort(key=lambda x: x["ai_probability"], reverse=True)
+    top_suspicious = suspicious[:3]
+    
+    return {
+        "verdict": confidence_label,
+        "ai_probability": ai_probability,
+        "reasons": reasons,
+        "top_suspicious_sentences": top_suspicious,
+        "uncertainty_note": _uncertainty_note(ai_probability),
+    }
+
+
+def _confidence_label(prob: float) -> str:
+    """Convert probability to human-readable verdict."""
+    if prob >= 0.85:
+        return "Very likely AI-generated"
+    elif prob >= 0.70:
+        return "Likely AI-generated"
+    elif prob >= 0.55:
+        return "Possibly AI-generated"
+    elif prob >= 0.45:
+        return "Uncertain — could be either"
+    elif prob >= 0.30:
+        return "Possibly human-written"
+    else:
+        return "Likely human-written"
+
+
+def _uncertainty_note(prob: float) -> str:
+    """
+    Always include an uncertainty note.
+    This is non-negotiable in any honest AI detection system.
+    """
+    if 0.4 <= prob <= 0.6:
+        return (
+            "This text falls in the uncertain range. "
+            "The model cannot confidently distinguish AI from human authorship here. "
+            "Do not use this result for any consequential decision."
+        )
+    elif prob > 0.8:
+        return (
+            "While the model is fairly confident, no AI detector is perfect. "
+            "Heavily edited AI text and formal human writing can both score high."
+        )
+    else:
+        return (
+            "AI detectors have meaningful false-positive and false-negative rates. "
+            "Treat this as one signal among many, not a definitive verdict."
+        )

core/perplexity_scorer.py

@@ -0,0 +1,83 @@
+# core/perplexity_scorer.py
+# Perplexity-based AI detection using GPT-2.
+#
+# Key insight: AI text is MORE predictable than human text.
+# GPT-2 assigns LOWER perplexity to AI-generated text
+# because LLMs generate high-probability token sequences.
+#
+# This signal is MODEL-AGNOSTIC — works regardless of which
+# AI wrote the text, unlike our BERT model which learned
+# GPT-5 Nano patterns specifically.
+
+import torch
+import math
+import numpy as np
+from transformers import GPT2LMHeadModel, GPT2TokenizerFast
+
+
+class PerplexityScorer:
+    """
+    Scores text using GPT-2 perplexity.
+    Lower perplexity = more predictable = more likely AI.
+    """
+
+    def __init__(self):
+        self._model = None
+        self._tokenizer = None
+        self._loaded = False
+
+    def load(self):
+        print("Loading GPT-2 for perplexity scoring...")
+        self._tokenizer = GPT2TokenizerFast.from_pretrained("gpt2")
+        self._model = GPT2LMHeadModel.from_pretrained("gpt2")
+        self._model.eval()
+        print("GPT-2 loaded.")
+        self._loaded = True
+
+    def get_perplexity(self, text: str) -> float:
+        """
+        Compute perplexity of text under GPT-2.
+        Lower = more predictable = more AI-like.
+        Typical ranges:
+          AI text:    30 - 80
+          Human text: 80 - 200+
+        """
+        if not self._loaded:
+            return None
+
+        encodings = self._tokenizer(
+            text,
+            return_tensors="pt",
+            truncation=True,
+            max_length=512,
+        )
+
+        input_ids = encodings.input_ids
+
+        with torch.no_grad():
+            outputs = self._model(input_ids, labels=input_ids)
+            loss = outputs.loss
+
+        return math.exp(loss.item())
+
+    def perplexity_to_ai_score(self, perplexity: float) -> float:
+        """
+        Convert perplexity to 0-1 AI probability.
+        Lower perplexity = higher AI score.
+
+        Calibrated ranges:
+          perplexity < 50  → score > 0.8 (very likely AI)
+          perplexity 50-100 → score 0.5-0.8
+          perplexity > 150 → score < 0.3 (likely human)
+        """
+        if perplexity is None:
+            return 0.5
+
+        # Sigmoid-like mapping
+        # Anchor: perplexity=50 → score=0.75, perplexity=150 → score=0.25
+        score = 1 / (1 + (perplexity / 80) ** 1.5)
+        return round(float(max(0.0, min(1.0, score))), 4)
+
+
+# Singleton
+perplexity_scorer = PerplexityScorer()

core/preprocessor.py

@@ -0,0 +1,94 @@
+# core/preprocessor.py
+# Handles all text cleaning and sentence splitting.
+# Kept separate so it can be tested and improved independently.
+
+import re
+import nltk
+from nltk.tokenize import sent_tokenize
+
+# Download the sentence tokenizer model on first run
+# This is a one-time ~400KB download
+nltk.download("punkt", quiet=True)
+nltk.download("punkt_tab", quiet=True)
+
+
+def clean_text(text: str) -> str:
+    """
+    Light cleaning — removes junk but preserves writing style.
+    We intentionally DON'T remove punctuation or normalize case
+    because those stylometric signals matter for detection.
+    """
+    if not text or not text.strip():
+        raise ValueError("Input text is empty.")
+    
+    # Collapse multiple spaces/newlines into single space
+    text = re.sub(r"\s+", " ", text)
+    
+    # Remove invisible unicode characters (zero-width spaces etc.)
+    text = re.sub(r"[\u200b\u200c\u200d\ufeff]", "", text)
+    
+    # Strip leading/trailing whitespace
+    text = text.strip()
+    
+    if len(text) < 20:
+        raise ValueError("Text too short to analyze (minimum 20 characters).")
+    
+    return text
+
+
+def split_sentences(text: str, min_length: int = 20) -> list[str]:
+    """
+    Split text into sentences using NLTK's Punkt tokenizer.
+    Filters out sentences that are too short to be meaningful.
+    
+    Why NLTK over simple split(".")?
+    → Handles abbreviations (U.S.A., Dr., etc.)
+    → Handles quoted speech
+    → More accurate on real-world text
+    """
+    sentences = sent_tokenize(text)
+    
+    # Filter trivially short fragments
+    sentences = [s.strip() for s in sentences if len(s.strip()) >= min_length]
+    
+    return sentences
+
+
+def chunk_for_bert(text: str, tokenizer, max_tokens: int = 500) -> list[str]:
+    """
+    BERT has a hard 512-token limit. For long texts we split into
+    overlapping chunks so no content is silently dropped.
+    
+    Overlap of 50 tokens ensures sentence boundaries aren't cut mid-thought.
+    We score each chunk separately, then average the scores.
+    """
+    tokens = tokenizer.encode(text, add_special_tokens=False)
+    
+    if len(tokens) <= max_tokens:
+        # Short enough — no chunking needed
+        return [text]
+    
+    # Split token IDs into overlapping windows
+    chunks = []
+    stride = max_tokens - 50  # 50-token overlap between chunks
+    
+    for start in range(0, len(tokens), stride):
+        end = start + max_tokens
+        chunk_tokens = tokens[start:end]
+        
+        # Decode back to text
+        chunk_text = tokenizer.decode(chunk_tokens, skip_special_tokens=True)
+        chunks.append(chunk_text)
+        
+        if end >= len(tokens):
+            break
+    
+    return chunks
+
+
+def preprocess(text: str) -> str:
+    """
+    Main entry point — clean and validate.
+    Returns cleaned text ready for scoring.
+    """
+    return clean_text(text)

core/stylometrics.py

@@ -0,0 +1,193 @@
+# core/stylometrics.py
+# Stylometric feature extraction — model-agnostic AI detection signals.
+#
+# WHY this matters:
+# BERT learned GPT-5 Nano patterns. These features catch ALL LLMs
+# because they measure structural writing habits, not learned tokens.
+
+import re
+import math
+import nltk
+from nltk.tokenize import sent_tokenize, word_tokenize
+
+nltk.download("punkt", quiet=True)
+nltk.download("punkt_tab", quiet=True)
+
+
+def avg_sentence_length(text: str) -> float:
+    """
+    Average words per sentence.
+    AI text tends to have suspiciously uniform, medium-length sentences.
+    Human text varies wildly — some very short, some very long.
+    """
+    sentences = sent_tokenize(text)
+    if not sentences:
+        return 0.0
+    
+    lengths = [len(word_tokenize(s)) for s in sentences]
+    return sum(lengths) / len(lengths)
+
+
+def burstiness(text: str) -> float:
+    """
+    Burstiness = coefficient of variation of sentence lengths.
+    
+    High burstiness → human (varied rhythm)
+    Low burstiness → AI (robotic uniformity)
+    
+    Formula: std_dev / mean of sentence lengths
+    A value near 0 means all sentences are the same length (AI signal).
+    A value > 0.5 is typical human writing.
+    """
+    sentences = sent_tokenize(text)
+    if len(sentences) < 3:
+        return 0.5  # Not enough data — return neutral
+    
+    lengths = [len(word_tokenize(s)) for s in sentences]
+    mean = sum(lengths) / len(lengths)
+    
+    if mean == 0:
+        return 0.0
+    
+    variance = sum((l - mean) ** 2 for l in lengths) / len(lengths)
+    std_dev = math.sqrt(variance)
+    
+    return std_dev / mean
+
+
+def type_token_ratio(text: str) -> float:
+    """
+    TTR = unique words / total words.
+    
+    Low TTR → repetitive vocabulary (AI signal — LLMs reuse safe words)
+    High TTR → diverse vocabulary (human signal)
+    
+    We use a windowed version (first 200 words) to avoid length bias.
+    """
+    words = word_tokenize(text.lower())
+    
+    # Use first 200 words to normalize for text length
+    words = words[:200]
+    
+    if not words:
+        return 0.0
+    
+    unique = set(words)
+    return len(unique) / len(words)
+
+
+def punctuation_diversity(text: str) -> float:
+    """
+    Ratio of punctuation variety to text length.
+    
+    AI text tends to use periods and commas almost exclusively.
+    Humans use dashes, ellipses, semicolons, exclamation marks more freely.
+    """
+    diverse_punct = re.findall(r"[;:—–…!?]", text)
+    total_chars = len(text)
+    
+    if total_chars == 0:
+        return 0.0
+    
+    return len(diverse_punct) / total_chars * 100
+
+
+def repetition_score(text: str) -> float:
+    """
+    Detects repeated phrases (3+ word n-grams that appear more than once).
+    
+    AI text often repeats phrases like "in the context of", 
+    "it is worth noting", "plays a crucial role".
+    """
+    words = word_tokenize(text.lower())
+    
+    if len(words) < 6:
+        return 0.0
+    
+    # Build trigrams
+    trigrams = [
+        " ".join(words[i:i+3])
+        for i in range(len(words) - 2)
+    ]
+    
+    # Count how many trigrams appear more than once
+    seen = {}
+    for tg in trigrams:
+        seen[tg] = seen.get(tg, 0) + 1
+    
+    repeated = sum(1 for count in seen.values() if count > 1)
+    
+    return repeated / len(trigrams) if trigrams else 0.0
+
+
+def compute_stylometric_score(text: str) -> dict:
+    """
+    Combines all features into a single AI-likelihood score (0–1)
+    plus a breakdown of each signal.
+    
+    Scoring logic:
+    - Low burstiness → more AI-like
+    - Low TTR → more AI-like
+    - Low punctuation diversity → more AI-like
+    - High repetition → more AI-like
+    
+    Each feature is normalized and combined with empirically tuned weights.
+    These weights are NOT magic — they're starting points. The evaluation
+    phase will show whether they need adjustment.
+    """
+    burst = burstiness(text)
+    ttr = type_token_ratio(text)
+    punct = punctuation_diversity(text)
+    rep = repetition_score(text)
+    avg_len = avg_sentence_length(text)
+    
+    # --- Normalize each feature to 0-1 AI likelihood ---
+    
+    # Burstiness: human ~0.4-0.8, AI ~0.1-0.3
+    # Lower burstiness = higher AI score
+    burst_score = max(0, min(1, 1 - (burst / 0.6)))
+    
+    # TTR: human ~0.6-0.8, AI ~0.4-0.6
+    # Lower TTR = higher AI score
+    ttr_score = max(0, min(1, 1 - (ttr / 0.7)))
+    
+    # Punctuation diversity: human > AI
+    # Lower diversity = higher AI score
+    punct_score = max(0, min(1, 1 - (punct / 0.5)))
+    
+    # Repetition: AI > human
+    # Higher repetition = higher AI score
+    rep_score = min(1.0, rep * 5)
+    
+    # Weighted combination
+    # BERT is our primary signal — stylometrics is secondary/supporting
+    weights = {
+        "burstiness": 0.35,
+        "ttr": 0.30,
+        "punctuation": 0.20,
+        "repetition": 0.15,
+    }
+    
+    combined = (
+        burst_score * weights["burstiness"] +
+        ttr_score * weights["ttr"] +
+        punct_score * weights["punctuation"] +
+        rep_score * weights["repetition"]
+    )
+    
+    return {
+        "stylometric_ai_score": round(combined, 4),
+        "features": {
+            "burstiness": round(burst, 4),
+            "avg_sentence_length": round(avg_len, 2),
+            "type_token_ratio": round(ttr, 4),
+            "punctuation_diversity": round(punct, 4),
+            "repetition_score": round(rep, 4),
+        },
+        "feature_scores": {
+            "burstiness_ai_signal": round(burst_score, 4),
+            "ttr_ai_signal": round(ttr_score, 4),
+            "punctuation_ai_signal": round(punct_score, 4),
+            "repetition_ai_signal": round(rep_score, 4),
+        }
+    }

dockerfile

@@ -0,0 +1,31 @@
+FROM python:3.11-slim
+
+# Set working directory
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    gcc \
+    g++ \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements first (for Docker layer caching)
+COPY requirements.txt .
+
+# Install Python dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy all backend code
+COPY . .
+
+# Set Python path so imports resolve correctly
+ENV PYTHONPATH=/app
+
+# Download NLTK data at build time
+RUN python -c "import nltk; nltk.download('punkt'); nltk.download('punkt_tab')"
+
+# HuggingFace Spaces uses port 7860
+EXPOSE 7860
+
+# Start the API
+CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "7860"]

requirements.txt

@@ -0,0 +1,14 @@
+fastapi==0.115.0
+uvicorn[standard]==0.30.6
+transformers==4.46.0
+torch==2.11.0
+peft==0.13.0
+sentencepiece==0.2.0
+nltk==3.9.1
+numpy==1.26.4
+scipy==1.13.1
+pydantic==2.8.2
+python-multipart==0.0.9
+httpx==0.27.2
+python-dotenv==1.0.1
+accelerate==1.1.0

run.py

@@ -0,0 +1,17 @@
+import sys
+import os
+
+# This must happen before any other import
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+os.environ["PYTHONPATH"] = os.path.dirname(os.path.abspath(__file__))
+
+import uvicorn
+
+if __name__ == "__main__":
+    uvicorn.run(
+        "api.main:app",
+        host="0.0.0.0",
+        port=8000,
+        reload=False,  # Disabled — solves the subprocess path issue
+    )
+