Add Dutch legal audit candidate review layer

candidate_scanner.py

@@ -0,0 +1,316 @@
+"""Candidate scanner for SolidPrivacy Scrub Dutch Legal Strict mode.
+
+This module is deliberately a review/audit layer, not an automatic redaction
+layer. It looks for suspicious reference-like values that were not already found
+by Presidio recognizers. The UI can show them as unchecked rows in the editable
+replacement table, so the user can decide whether to include them.
+
+Design rules:
+- Preserve context words such as dossiernummer, kenteken, factuurnummer.
+- Suggest only the suspicious value, not the whole sentence.
+- Do not suggest legal article references, dates, money amounts, pages or annexes.
+- Prefer category-level review over one-off hotfixes.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Iterable, List, Sequence, Tuple
+
+try:
+    from legal_reference_taxonomy import LEGAL_REFERENCE_CATEGORIES
+except Exception:  # keep the app usable while files are being copied
+    LEGAL_REFERENCE_CATEGORIES = []
+
+
+@dataclass(frozen=True)
+class Candidate:
+    text: str
+    entity_type: str
+    placeholder: str
+    score: float
+    start: int
+    end: int
+    reason: str
+    context: str
+
+    def as_dict(self) -> dict:
+        return {
+            "text": self.text,
+            "entity_type": self.entity_type,
+            "placeholder": self.placeholder,
+            "score": self.score,
+            "start": self.start,
+            "end": self.end,
+            "reason": self.reason,
+            "context": self.context,
+        }
+
+
+# Broad value shapes used only after a context cue has been found.
+# This catches values like CL-FAM-55201, WR-KLANT-2026-7712, FACT-2026-4481,
+# DOSS/2026/1189 and compact context-bound values such as XX123X after "kenteken".
+CONTEXTUAL_VALUE_RE = re.compile(
+    r"\b(?:"
+    r"(?=[A-Z0-9][A-Z0-9./_-]{4,49}\b)(?=[A-Z0-9./_-]*[A-Z])(?=[A-Z0-9./_-]*\d)"
+    r"[A-Z0-9]+(?:[./_-][A-Z0-9]+){0,8}"
+    r"|"
+    r"\d{3}\.\d{3}\.\d{3}/\d{2}\s+[A-Z]{1,5}"
+    r")\b"
+)
+
+# Stand-alone suspicious codes. These are shown as candidates only when they are
+# not already detected and not obviously a date/article/amount.
+STANDALONE_CODE_RE = re.compile(
+    r"\b(?=[A-Z0-9][A-Z0-9./_-]{5,49}\b)(?=[A-Z0-9./_-]*[A-Z])(?=[A-Z0-9./_-]*\d)"
+    r"[A-Z0-9]+(?:[./_-][A-Z0-9]+){1,8}\b"
+)
+
+DUTCH_PLATE_CONTEXT = {
+    "kenteken",
+    "kentekennummer",
+    "nummerplaat",
+    "voertuig",
+    "auto",
+    "leaseauto",
+    "bedrijfsauto",
+    "bestelbus",
+    "rdw",
+}
+
+GENERIC_CONTEXT_CUES = {
+    "nummer",
+    "referentie",
+    "kenmerk",
+    "dossier",
+    "code",
+    "registratie",
+    "zaak",
+    "factuur",
+    "contract",
+    "polis",
+    "claim",
+    "school",
+    "uwv",
+    "ind",
+    "gemeente",
+    "politie",
+    "proces-verbaal",
+    "pv",
+}
+
+NEGATIVE_NEARBY_CUES = {
+    "artikel",
+    "art.",
+    "lid",
+    "sub",
+    "pagina",
+    "bladzijde",
+    "bijlage",
+    "productie",
+    "randnummer",
+    "paragraaf",
+}
+
+
+def _overlaps(start: int, end: int, spans: Sequence[Tuple[int, int]]) -> bool:
+    for other_start, other_end in spans:
+        if start < other_end and end > other_start:
+            return True
+    return False
+
+
+def _window(text: str, start: int, end: int, radius: int = 60) -> str:
+    return text[max(0, start - radius) : min(len(text), end + radius)]
+
+
+def _normalise_space(value: str) -> str:
+    return re.sub(r"\s+", " ", value or "").strip()
+
+
+def _looks_like_date_or_time(value: str) -> bool:
+    v = (value or "").strip()
+    if re.fullmatch(r"\d{1,2}[-/.]\d{1,2}[-/.]\d{2,4}", v):
+        return True
+    if re.fullmatch(r"\d{4}[-/.]\d{1,2}[-/.]\d{1,2}", v):
+        return True
+    if re.fullmatch(r"\d{1,2}[.:]\d{2}", v):
+        return True
+    return False
+
+
+def _looks_like_money_or_article(value: str, context: str) -> bool:
+    low = (context or "").lower()
+    v = (value or "").strip()
+    if "€" in low or "eur" in low or "euro" in low:
+        return True
+    if re.fullmatch(r"\d{1,2}:\d{1,4}[a-z]?", v, flags=re.IGNORECASE):
+        return True
+    # Avoid legal article references and document navigation references.
+    if any(cue in low for cue in NEGATIVE_NEARBY_CUES) and not any(cue in low for cue in GENERIC_CONTEXT_CUES):
+        return True
+    return False
+
+
+def _is_negative_candidate(value: str, context: str) -> bool:
+    v = (value or "").strip(" .,;:\n\t")
+    if len(v) < 5:
+        return True
+    if _looks_like_date_or_time(v) or _looks_like_money_or_article(v, context):
+        return True
+    # Plain Dutch postcode is already handled by NL_POSTCODE; do not duplicate it
+    # as a suspicious reference.
+    if re.fullmatch(r"[1-9][0-9]{3}\s?[A-Z]{2}", v, flags=re.IGNORECASE):
+        return True
+    return False
+
+
+def _keyword_regex(keyword: str) -> re.Pattern:
+    escaped = re.escape(keyword.strip()).replace(r"\ ", r"[ \t]+")
+    return re.compile(rf"(?<!\w){escaped}(?!\w)", flags=re.IGNORECASE | re.MULTILINE)
+
+
+def _search_boundary(text: str, start: int, max_chars: int = 120) -> int:
+    hard_end = min(len(text), start + max_chars)
+    candidates = [hard_end]
+    for sep in ["\n", "\r", ";"]:
+        idx = text.find(sep, start, hard_end)
+        if idx != -1:
+            candidates.append(idx)
+    dot = text.find(".", start, hard_end)
+    if dot != -1 and dot - start > 25:
+        candidates.append(dot)
+    return min(candidates)
+
+
+def _placeholder_for(entity_type: str) -> str:
+    labels = {
+        "NL_SUSPICIOUS_REFERENCE_CANDIDATE": "<MOGELIJKE_REFERENTIE>",
+        "NL_POSSIBLE_LICENSE_PLATE": "<MOGELIJK_KENTEKEN>",
+        "NL_VEHICLE_REFERENCE": "<VOERTUIG_OF_KENTEKENREFERENTIE>",
+        "NL_OBJECT_REFERENCE": "<OBJECTREFERENTIE>",
+    }
+    for category in LEGAL_REFERENCE_CATEGORIES:
+        if category.get("entity_type") == entity_type:
+            return f"<{category.get('placeholder', entity_type)}>"
+    return labels.get(entity_type, f"<{entity_type}>")
+
+
+def _has_context_cue(context: str) -> bool:
+    low = (context or "").lower()
+    return any(cue in low for cue in GENERIC_CONTEXT_CUES) or any(cue in low for cue in DUTCH_PLATE_CONTEXT)
+
+
+def _dedupe(candidates: Iterable[Candidate]) -> List[Candidate]:
+    by_span = {}
+    for candidate in candidates:
+        key = (candidate.start, candidate.end, candidate.text)
+        existing = by_span.get(key)
+        if existing is None or candidate.score > existing.score:
+            by_span[key] = candidate
+    return sorted(by_span.values(), key=lambda item: (item.start, -item.score))
+
+
+def scan_unmasked_candidates(text: str, analyzer_results=None, max_candidates: int = 50) -> List[dict]:
+    """Return suspicious unmasked candidate values for review.
+
+    analyzer_results may be Presidio RecognizerResult objects. Their spans are
+    excluded so this scanner focuses on what likely remained unhandled.
+    """
+    source = text or ""
+    existing_spans = []
+    for res in analyzer_results or []:
+        start = getattr(res, "start", None)
+        end = getattr(res, "end", None)
+        if isinstance(start, int) and isinstance(end, int):
+            existing_spans.append((start, end))
+
+    candidates: List[Candidate] = []
+
+    # 1) Taxonomy-driven contextual values that were not detected. This uses the
+    # same categories as the recognizer but keeps them as unchecked candidates in
+    # case thresholds/entity filters missed them.
+    for category in LEGAL_REFERENCE_CATEGORIES:
+        entity_type = category.get("entity_type", "NL_CONTEXTUAL_REFERENCE")
+        keywords = category.get("keywords", [])
+        base_score = min(float(category.get("score", 0.70)), 0.74)
+        for keyword in keywords:
+            for kw_match in _keyword_regex(keyword).finditer(source):
+                search_start = kw_match.end()
+                search_end = _search_boundary(source, search_start)
+                local_text = source[search_start:search_end]
+                value_match = CONTEXTUAL_VALUE_RE.search(local_text)
+                if not value_match:
+                    continue
+                start = search_start + value_match.start()
+                end = search_start + value_match.end()
+                raw = source[start:end]
+                trim_l = len(raw) - len(raw.lstrip(" \t:=-#"))
+                trim_r = len(raw) - len(raw.rstrip(" \t.,;:"))
+                start += trim_l
+                if trim_r:
+                    end -= trim_r
+                value = source[start:end]
+                ctx = _window(source, start, end)
+                if _overlaps(start, end, existing_spans) or _is_negative_candidate(value, ctx):
+                    continue
+                candidates.append(
+                    Candidate(
+                        text=value,
+                        entity_type=entity_type,
+                        placeholder=_placeholder_for(entity_type),
+                        score=base_score,
+                        start=start,
+                        end=end,
+                        reason=f"Possible unmasked value after context keyword '{keyword}'",
+                        context=_normalise_space(ctx),
+                    )
+                )
+
+    # 2) License plate / vehicle compact candidates. These are context-bound, not
+    # blind plate recognition, because fake/test material often uses compact
+    # examples such as XX123X.
+    for match in re.finditer(r"\b(?=[A-Z0-9]{5,12}\b)(?=[A-Z0-9]*[A-Z])(?=[A-Z0-9]*\d)[A-Z0-9]{5,12}\b", source):
+        start, end = match.span()
+        value = match.group(0)
+        ctx = _window(source, start, end)
+        if _overlaps(start, end, existing_spans) or _is_negative_candidate(value, ctx):
+            continue
+        if any(cue in ctx.lower() for cue in DUTCH_PLATE_CONTEXT):
+            candidates.append(
+                Candidate(
+                    text=value,
+                    entity_type="NL_POSSIBLE_LICENSE_PLATE",
+                    placeholder=_placeholder_for("NL_POSSIBLE_LICENSE_PLATE"),
+                    score=0.66,
+                    start=start,
+                    end=end,
+                    reason="Compact alphanumeric value near vehicle/kenteken context",
+                    context=_normalise_space(ctx),
+                )
+            )
+
+    # 3) Remaining standalone codes with generic legal/admin context nearby.
+    for match in STANDALONE_CODE_RE.finditer(source):
+        start, end = match.span()
+        value = match.group(0)
+        ctx = _window(source, start, end)
+        if _overlaps(start, end, existing_spans) or _is_negative_candidate(value, ctx):
+            continue
+        if not _has_context_cue(ctx):
+            continue
+        candidates.append(
+            Candidate(
+                text=value,
+                entity_type="NL_SUSPICIOUS_REFERENCE_CANDIDATE",
+                placeholder=_placeholder_for("NL_SUSPICIOUS_REFERENCE_CANDIDATE"),
+                score=0.52,
+                start=start,
+                end=end,
+                reason="Reference-like code near legal/administrative context but not auto-masked",
+                context=_normalise_space(ctx),
+            )
+        )
+
+    return [candidate.as_dict() for candidate in _dedupe(candidates)[:max_candidates]]