# core/preprocessor.py
# Handles all text cleaning and sentence splitting.
# Kept separate so it can be tested and improved independently.

import re
import nltk
from nltk.tokenize import sent_tokenize

# Download the sentence tokenizer model on first run
# This is a one-time ~400KB download
nltk.download("punkt", quiet=True)
nltk.download("punkt_tab", quiet=True)


def clean_text(text: str) -> str:
    """
    Light cleaning — removes junk but preserves writing style.
    We intentionally DON'T remove punctuation or normalize case
    because those stylometric signals matter for detection.
    """
    if not text or not text.strip():
        raise ValueError("Input text is empty.")
    
    # Collapse multiple spaces/newlines into single space
    text = re.sub(r"\s+", " ", text)
    
    # Remove invisible unicode characters (zero-width spaces etc.)
    text = re.sub(r"[\u200b\u200c\u200d\ufeff]", "", text)
    
    # Strip leading/trailing whitespace
    text = text.strip()
    
    if len(text) < 20:
        raise ValueError("Text too short to analyze (minimum 20 characters).")
    
    return text


def split_sentences(text: str, min_length: int = 20) -> list[str]:
    """
    Split text into sentences using NLTK's Punkt tokenizer.
    Filters out sentences that are too short to be meaningful.
    
    Why NLTK over simple split(".")?
    → Handles abbreviations (U.S.A., Dr., etc.)
    → Handles quoted speech
    → More accurate on real-world text
    """
    sentences = sent_tokenize(text)
    
    # Filter trivially short fragments
    sentences = [s.strip() for s in sentences if len(s.strip()) >= min_length]
    
    return sentences


def chunk_for_bert(text: str, tokenizer, max_tokens: int = 500) -> list[str]:
    """
    BERT has a hard 512-token limit. For long texts we split into
    overlapping chunks so no content is silently dropped.
    
    Overlap of 50 tokens ensures sentence boundaries aren't cut mid-thought.
    We score each chunk separately, then average the scores.
    """
    tokens = tokenizer.encode(text, add_special_tokens=False)
    
    if len(tokens) <= max_tokens:
        # Short enough — no chunking needed
        return [text]
    
    # Split token IDs into overlapping windows
    chunks = []
    stride = max_tokens - 50  # 50-token overlap between chunks
    
    for start in range(0, len(tokens), stride):
        end = start + max_tokens
        chunk_tokens = tokens[start:end]
        
        # Decode back to text
        chunk_text = tokenizer.decode(chunk_tokens, skip_special_tokens=True)
        chunks.append(chunk_text)
        
        if end >= len(tokens):
            break
    
    return chunks


def preprocess(text: str) -> str:
    """
    Main entry point — clean and validate.
    Returns cleaned text ready for scoring.
    """
    return clean_text(text)