clean version without LFS

.env

@@ -0,0 +1 @@
+POPPLER_PATH=C:\Program Files\poppler-25.12.0\Library\bin

Dockerfile

@@ -0,0 +1,37 @@
+FROM python:3.10-slim
+
+ENV PYTHONUNBUFFERED=1
+ENV DEBIAN_FRONTEND=noninteractive
+ENV PADDLE_PDX_DISABLE_MODEL_SOURCE_CHECK=True
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libglib2.0-0 \
+    libsm6 \
+    libxext6 \
+    libxrender1 \
+    libgl1 \
+    libgomp1 \
+    poppler-utils \
+    tesseract-ocr \
+    tesseract-ocr-eng \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt .
+
+RUN pip install --no-cache-dir \
+    torch \
+    torchvision \
+    --index-url https://download.pytorch.org/whl/cpu
+
+RUN pip install --no-cache-dir paddlepaddle
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+RUN python -c "from paddleocr import PaddleOCR; PaddleOCR(lang='en'); print('PaddleOCR models cached')"
+
+COPY . .
+
+EXPOSE 7860
+CMD ["python", "-u", "app.py"]

IAM_train.py

@@ -0,0 +1,332 @@
+"""
+IAM_train.py
+============
+Fine-tune the CRNN model using the IAM Handwriting Word Database.
+Builds on top of EMNIST-trained model (best_model_emnist.pth).
+
+FIXES vs old version:
+  - IMG_WIDTH 400 -> 512 (must match pipeline)
+  - Added log_softmax before CTCLoss (was missing — caused catastrophic forgetting)
+  - Phase 1: CNN FROZEN — only RNN+FC trained
+  - Phase 2: Full model at very low LR
+  - Loads from best_model_emnist.pth, falls back to best_model.pth
+  - Uses get_crnn_model() with correct architecture from checkpoint config
+
+DATASET:
+  Download from: https://www.kaggle.com/datasets/nibinv23/iam-handwriting-word-database
+  Expected structure:
+    data/IAM/iam_words/
+      words/        <- word image folders (a01, a02, ...)
+      words.txt     <- annotation file
+
+USAGE:
+  python IAM_train.py --prepare          # convert IAM -> annotation JSON
+  python IAM_train.py --train            # fine-tune model
+  python IAM_train.py --prepare --train  # do both
+"""
+
+import os
+import sys
+import json
+import argparse
+import random
+from pathlib import Path
+
+import torch
+import torch.nn.functional as F
+import torch.optim as optim
+from torch.utils.data import DataLoader, ConcatDataset
+
+sys.path.append('.')
+from crnn_model import get_crnn_model
+from dataset import CivilRegistryDataset, collate_fn
+
+# ─────────────────────────────────────────────
+#  CONFIG
+# ─────────────────────────────────────────────
+IAM_ROOT      = "data/IAM/iam_words"
+IAM_WORDS_TXT = f"{IAM_ROOT}/words.txt"
+IAM_WORDS_DIR = f"{IAM_ROOT}/words"
+
+TRAIN_ANN     = "data/iam_train_annotations.json"
+IAM_VAL_ANN   = "data/iam_val_annotations.json"   # written by --prepare (IAM word images)
+SYNTH_VAL_ANN = "data/val_annotations.json"       # real civil registry val set — never overwritten
+TRAIN_IMG_DIR = "data/train/iam"
+VAL_IMG_DIR   = "data/val/iam"
+
+IMG_HEIGHT    = 64
+IMG_WIDTH     = 512       # FIXED: was 400 — must match pipeline
+BATCH_SIZE    = 32
+VAL_SPLIT     = 0.1
+MAX_SAMPLES   = 50000
+
+# Load from EMNIST checkpoint, fall back to synthetic if not found
+CHECKPOINT_IN  = "checkpoints/best_model_emnist.pth"
+CHECKPOINT_IN2 = "checkpoints/best_model.pth"   # fallback
+CHECKPOINT_OUT = "checkpoints/best_model_iam.pth"
+
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+# ─────────────────────────────────────────────
+#  STEP 1 — PREPARE
+# ─────────────────────────────────────────────
+def prepare_iam():
+    from PIL import Image
+
+    print("\n" + "=" * 50)
+    print("STEP 1 — Preparing IAM dataset")
+    print("=" * 50)
+
+    if not os.path.exists(IAM_WORDS_TXT):
+        print(f"ERROR: {IAM_WORDS_TXT} not found!")
+        print("Download from: https://www.kaggle.com/datasets/nibinv23/iam-handwriting-word-database")
+        print("Expected structure:")
+        print("  data/IAM/iam_words/words.txt")
+        print("  data/IAM/iam_words/words/")
+        sys.exit(1)
+
+    os.makedirs(TRAIN_IMG_DIR, exist_ok=True)
+    os.makedirs(VAL_IMG_DIR,   exist_ok=True)
+
+    entries = []
+    print(f"  Reading {IAM_WORDS_TXT} ...")
+    with open(IAM_WORDS_TXT, "r") as f:
+        for line in f:
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+            parts = line.split(" ")
+            if len(parts) < 9:
+                continue
+            word_id    = parts[0]
+            seg_result = parts[1]
+            text       = parts[-1]
+            if seg_result != "ok":
+                continue
+            if len(text) < 1 or len(text) > 32:
+                continue
+            parts_id = word_id.split("-")
+            img_path = os.path.join(
+                IAM_WORDS_DIR,
+                parts_id[0],
+                f"{parts_id[0]}-{parts_id[1]}",
+                f"{word_id}.png"
+            )
+            if not os.path.exists(img_path):
+                continue
+            entries.append((img_path, text))
+
+    print(f"  Found {len(entries)} valid word entries")
+
+    if MAX_SAMPLES and len(entries) > MAX_SAMPLES:
+        random.shuffle(entries)
+        entries = entries[:MAX_SAMPLES]
+        print(f"  Limiting to {MAX_SAMPLES} samples")
+
+    random.shuffle(entries)
+    split_idx     = int(len(entries) * (1 - VAL_SPLIT))
+    train_entries = entries[:split_idx]
+    val_entries   = entries[split_idx:]
+    print(f"  Train: {len(train_entries)} | Val: {len(val_entries)}")
+    print("  Copying and resizing images...")
+
+    def process_entries(entry_list, out_dir, prefix):
+        annotations = []
+        for i, (src_path, text) in enumerate(entry_list):
+            try:
+                img = Image.open(src_path).convert("RGB")
+                img = img.resize((IMG_WIDTH, IMG_HEIGHT))  # FIXED: 512x64
+                fname    = f"iam_{prefix}_{i:06d}.jpg"
+                out_path = os.path.join(out_dir, fname)
+                img.save(out_path, quality=90)
+                annotations.append({"image_path": f"iam/{fname}", "text": text})
+            except Exception:
+                continue
+            if i % 5000 == 0:
+                print(f"    {i}/{len(entry_list)} processed...")
+        return annotations
+
+    train_ann = process_entries(train_entries, TRAIN_IMG_DIR, "train")
+    val_ann   = process_entries(val_entries,   VAL_IMG_DIR,   "val")
+
+    with open(TRAIN_ANN, "w") as f:
+        json.dump(train_ann, f, indent=2)
+    with open(IAM_VAL_ANN, "w") as f:
+        json.dump(val_ann, f, indent=2)
+
+    print(f"\n  Train annotations -> {TRAIN_ANN} ({len(train_ann)} entries)")
+    print(f"  Val annotations   -> {IAM_VAL_ANN} ({len(val_ann)} entries)")
+    print("\n  Done! Now run: python IAM_train.py --train")
+
+
+# ─────────────────────────────────────────────
+#  STEP 2 — TRAIN
+# ─────────────────────────────────────────────
+def train_iam():
+    print("\n" + "=" * 55)
+    print("STEP 2 — Fine-tuning CRNN with IAM dataset")
+    print("=" * 55)
+    print(f"  Device : {DEVICE}")
+
+    for ann_file in [TRAIN_ANN, SYNTH_VAL_ANN]:
+        if not os.path.exists(ann_file):
+            print(f"ERROR: {ann_file} not found! Run --prepare first.")
+            sys.exit(1)
+
+    train_dataset = CivilRegistryDataset(
+        data_dir="data/train", annotations_file=TRAIN_ANN,
+        img_height=IMG_HEIGHT, img_width=IMG_WIDTH, augment=True
+    )
+    # FIXED: mix synthetic data in so the model never forgets Filipino multi-word sequences
+    synth_dataset = CivilRegistryDataset(
+        data_dir="data/train", annotations_file="data/train_annotations.json",
+        img_height=IMG_HEIGHT, img_width=IMG_WIDTH, augment=True
+    )
+    mixed_train = ConcatDataset([train_dataset, synth_dataset])
+    val_dataset = CivilRegistryDataset(
+        data_dir="data/val", annotations_file=SYNTH_VAL_ANN,
+        img_height=IMG_HEIGHT, img_width=IMG_WIDTH, augment=False
+    )
+    print(f"  IAM train     : {len(train_dataset)}")
+    print(f"  Synthetic train: {len(synth_dataset)}")
+    print(f"  Mixed train   : {len(mixed_train)}")
+    print(f"  Val           : {len(val_dataset)}")
+
+    train_loader = DataLoader(mixed_train, batch_size=BATCH_SIZE,
+                              shuffle=True,  num_workers=0, collate_fn=collate_fn)
+    val_loader   = DataLoader(val_dataset,   batch_size=BATCH_SIZE,
+                              shuffle=False, num_workers=0, collate_fn=collate_fn)
+
+    # ── Load checkpoint (EMNIST preferred, synthetic fallback) ──
+    ckpt_path = CHECKPOINT_IN if os.path.exists(CHECKPOINT_IN) else CHECKPOINT_IN2
+    if not os.path.exists(ckpt_path):
+        print(f"ERROR: No checkpoint found at {CHECKPOINT_IN} or {CHECKPOINT_IN2}")
+        print("Run: python train.py  then  python train_with_emnist.py")
+        sys.exit(1)
+
+    print(f"  Loading: {ckpt_path}")
+    ckpt   = torch.load(ckpt_path, map_location=DEVICE, weights_only=False)
+    config = ckpt.get('config', {})
+
+    model = get_crnn_model(
+        model_type      = config.get('model_type', 'standard'),
+        img_height      = config.get('img_height', 64),
+        num_chars       = train_dataset.num_chars,
+        hidden_size     = config.get('hidden_size', 128),
+        num_lstm_layers = config.get('num_lstm_layers', 1),
+    ).to(DEVICE)
+
+    missing, _ = model.load_state_dict(ckpt['model_state_dict'], strict=False)
+    if missing:
+        print(f"  Note: {len(missing)} layers re-initialized")
+    print(f"  Loaded epoch {ckpt.get('epoch', 'N/A')} "
+          f"val_loss={ckpt.get('val_loss', ckpt.get('val_cer', 0)):.4f}")
+
+    criterion = torch.nn.CTCLoss(blank=0, reduction='mean', zero_infinity=True)
+    os.makedirs("checkpoints", exist_ok=True)
+
+    def run_epoch(loader, training, optimizer=None):
+        model.train() if training else model.eval()
+        total, n = 0, 0
+        ctx = torch.enable_grad() if training else torch.no_grad()
+        with ctx:
+            for images, targets, target_lengths, _ in loader:
+                images        = images.to(DEVICE)
+                batch_size    = images.size(0)
+                if training:
+                    optimizer.zero_grad()
+                # CRITICAL: log_softmax before CTCLoss
+                outputs       = F.log_softmax(model(images), dim=2)
+                seq_len       = outputs.size(0)
+                input_lengths = torch.full((batch_size,), seq_len, dtype=torch.long)
+                loss = criterion(outputs, targets, input_lengths, target_lengths)
+                if not torch.isnan(loss) and not torch.isinf(loss):
+                    if training:
+                        loss.backward()
+                        torch.nn.utils.clip_grad_norm_(model.parameters(), 5)
+                        optimizer.step()
+                    total += loss.item()
+                    n     += 1
+        return total / max(n, 1)
+
+    def run_phase(num, epochs, lr, freeze_cnn, patience):
+        print(f"\n{'='*55}")
+        print(f"  PHASE {num} — "
+              f"{'CNN FROZEN  (RNN+FC only)' if freeze_cnn else 'FULL MODEL  (all layers)'}"
+              f"   LR={lr}")
+        print(f"{'='*55}")
+
+        for name, param in model.named_parameters():
+            param.requires_grad = not (freeze_cnn and 'cnn' in name)
+
+        trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
+        print(f"  Trainable params : {trainable:,}")
+
+        opt     = optim.Adam(
+            filter(lambda p: p.requires_grad, model.parameters()), lr=lr)
+        sched   = optim.lr_scheduler.ReduceLROnPlateau(opt, patience=3, factor=0.5)
+        best    = float('inf')
+        counter = 0
+
+        for epoch in range(1, epochs + 1):
+            tr = run_epoch(train_loader, True,  opt)
+            vl = run_epoch(val_loader,   False, None)
+            sched.step(vl)
+
+            if vl < best:
+                best    = vl
+                counter = 0
+                torch.save({
+                    'model_state_dict': model.state_dict(),
+                    'config':           config,
+                    'char_to_idx':      train_dataset.char_to_idx,
+                    'idx_to_char':      train_dataset.idx_to_char,
+                    'epoch':            epoch,
+                    'val_loss':         vl,   # FIXED: renamed from val_cer — this is val loss, not CER%
+                }, CHECKPOINT_OUT)
+                print(f"  Epoch {epoch:02d}/{epochs}  "
+                      f"Train={tr:.4f}  Val={vl:.4f}  <- saved")
+            else:
+                counter += 1
+                print(f"  Epoch {epoch:02d}/{epochs}  "
+                      f"Train={tr:.4f}  Val={vl:.4f}  "
+                      f"(patience {counter}/{patience})")
+                if counter >= patience:
+                    print(f"  Early stopping at epoch {epoch}.")
+                    break
+        return best
+
+    # Phase 1: Freeze CNN
+    p1 = run_phase(1, epochs=30, lr=1e-4, freeze_cnn=True,  patience=7)
+    # Phase 2: Full model, very low LR
+    p2 = run_phase(2, epochs=20, lr=1e-6, freeze_cnn=False, patience=5)
+
+    print(f"\n{'='*55}")
+    print(f"IAM fine-tuning complete!")
+    print(f"  Phase 1 best val loss : {p1:.4f}")
+    print(f"  Phase 2 best val loss : {p2:.4f}")
+    print(f"  Saved : {CHECKPOINT_OUT}")
+    print(f"\nNext step: collect physical certificate scans")
+
+
+# ─────────────────────────────────────────────
+#  MAIN
+# ─────────────────────────────────────────────
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--prepare", action="store_true")
+    parser.add_argument("--train",   action="store_true")
+    args = parser.parse_args()
+
+    if not args.prepare and not args.train:
+        print("Usage:")
+        print("  python IAM_train.py --prepare          # prepare dataset")
+        print("  python IAM_train.py --train            # train model")
+        print("  python IAM_train.py --prepare --train  # do both")
+        sys.exit(0)
+
+    if args.prepare:
+        prepare_iam()
+    if args.train:
+        train_iam()

README.md

@@ -0,0 +1,459 @@
+---
+title: LCR OCR API
+emoji: 📄
+colorFrom: blue
+colorTo: green
+sdk: docker
+app_file: app.py
+pinned: false
+---
+
+# Local Civil Registry Document Digitization and Data Extraction
+
+## Using CRNN+CTC, Multinomial Naive Bayes, and Named Entity Recognition
+
+**Thesis Project by:**
+- Shane Mark C. Blanco
+- Princess A. Pasamonte  
+- Irish Faith G. Ramirez
+
+**Institution:** Tarlac State University, College of Computer Studies
+
+---
+
+## 📋 Project Overview
+
+This system automates the digitization and data extraction of Philippine Civil Registry documents using advanced machine learning algorithms:
+
+### Target Documents:
+- **Form 1A** - Birth Certificate
+- **Form 2A** - Death Certificate
+- **Form 3A** - Marriage Certificate
+- **Form 90** - Application of Marriage License
+
+### Key Features:
+✅ OCR for printed and handwritten text  
+✅ Automatic document classification  
+✅ Named entity extraction (names, dates, places)  
+✅ Auto-fill digital forms  
+✅ MySQL database storage  
+✅ Searchable digital archive  
+✅ Data visualization dashboard  
+
+---
+
+## 🏗️ System Architecture
+
+```
+Input: Scanned Civil Registry Form
+    ↓
+1. Image Preprocessing
+    ↓
+2. CRNN+CTC → Text Recognition
+    ↓
+3. Multinomial Naive Bayes → Document Classification
+    ↓
+4. spaCy NER → Entity Extraction
+    ↓
+5. Data Validation & Storage → MySQL Database
+    ↓
+Output: Digitized & Searchable Record
+```
+
+---
+
+## 🚀 Quick Start
+
+### Prerequisites
+
+- Python 3.8+
+- CUDA-capable GPU (recommended) or CPU
+- 8GB RAM minimum
+
+### Installation
+
+```bash
+# 1. Clone or download the project
+cd civil_registry_ocr
+
+# 2. Create virtual environment
+python -m venv venv
+source venv/bin/activate  # Linux/Mac
+venv\Scripts\activate     # Windows
+
+# 3. Install dependencies
+pip install -r requirements.txt
+
+# 4. Download spaCy model
+python -m spacy download en_core_web_sm
+```
+
+### Quick Test
+
+```python
+from inference import CivilRegistryOCR
+
+# Load model
+ocr = CivilRegistryOCR('checkpoints/best_model.pth')
+
+# Recognize text
+text = ocr.predict('test_images/sample_name.jpg')
+print(f"Recognized: {text}")
+```
+
+---
+
+## 📁 Project Files
+
+### Core Implementation Files:
+
+1. **crnn_model.py** - CRNN+CTC neural network architecture
+2. **dataset.py** - Data loading and preprocessing  
+3. **train.py** - Model training script
+4. **inference.py** - Prediction and inference
+5. **utils.py** - Helper functions and metrics
+6. **requirements.txt** - Python dependencies
+7. **IMPLEMENTATION_GUIDE.md** - Detailed implementation guide
+
+### Additional Components (To be created):
+
+8. **document_classifier.py** - Multinomial Naive Bayes classifier
+9. **ner_extractor.py** - Named Entity Recognition
+10. **web_app.py** - Web application (Flask/FastAPI)
+11. **database.py** - MySQL integration
+
+---
+
+## 📊 Training the Model
+
+### 1. Prepare Your Data
+
+Organize images and labels:
+```
+data/
+  train/
+    form1a/
+      name_001.jpg
+      name_001.txt
+    form2a/
+      ...
+  val/
+    ...
+```
+
+### 2. Create Annotations
+
+```python
+from dataset import create_annotation_file
+
+create_annotation_file('data/train', 'data/train_annotations.json')
+create_annotation_file('data/val', 'data/val_annotations.json')
+```
+
+### 3. Train Model
+
+```bash
+python train.py
+```
+
+Monitor metrics:
+- Character Error Rate (CER)
+- Word Error Rate (WER)  
+- Training/Validation Loss
+
+### 4. Evaluate
+
+```python
+from utils import calculate_cer, calculate_wer
+
+predictions = [ocr.predict(img) for img in test_images]
+cer = calculate_cer(predictions, ground_truths)
+print(f"CER: {cer:.2f}%")
+```
+
+---
+
+## 🌐 Web Application
+
+### Start the Server
+
+```bash
+python web_app.py
+```
+
+### API Endpoints
+
+**POST /api/ocr** - Process document
+```bash
+curl -X POST -F "file=@birth_cert.jpg" http://localhost:8000/api/ocr
+```
+
+**Response:**
+```json
+{
+  "text": "Juan Dela Cruz\n01/15/1990\nTarlac City",
+  "form_type": "form1a",
+  "entities": {
+    "persons": ["Juan Dela Cruz"],
+    "dates": ["01/15/1990"],
+    "locations": ["Tarlac City"]
+  }
+}
+```
+
+---
+
+## 🎯 Expected Performance
+
+Based on thesis objectives:
+
+### CRNN+CTC Model:
+- **Target CER:** < 5%
+- **Target Accuracy:** > 95%
+- Handles both printed and handwritten text
+
+### Document Classifier (MNB):
+- **Target Accuracy:** > 90%
+- Fast classification (< 100ms)
+
+### NER (spaCy):
+- **F1 Score:** > 85%
+- Extracts: Names, Dates, Places
+
+---
+
+## 🧪 Testing
+
+### ISO 25010 Evaluation
+
+**Usability Testing:**
+```python
+# Metrics to measure:
+- Task completion rate
+- Average time per task
+- User satisfaction score (SUS)
+```
+
+**Reliability Testing:**
+```python
+# Metrics to measure:
+- System uptime %
+- Error rate
+- Recovery time
+```
+
+### Confusion Matrix
+
+```python
+from sklearn.metrics import confusion_matrix
+import seaborn as sns
+
+cm = confusion_matrix(true_labels, predicted_labels)
+sns.heatmap(cm, annot=True)
+```
+
+---
+
+## 💾 Database Schema
+
+### Birth Certificates Table
+```sql
+CREATE TABLE birth_certificates (
+    id INT PRIMARY KEY AUTO_INCREMENT,
+    child_name VARCHAR(255),
+    date_of_birth DATE,
+    place_of_birth VARCHAR(255),
+    sex CHAR(1),
+    father_name VARCHAR(255),
+    mother_name VARCHAR(255),
+    raw_text TEXT,
+    form_image LONGBLOB,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+```
+
+---
+
+## 📈 System Requirements
+
+### Minimum:
+- CPU: Intel i5 or equivalent
+- RAM: 8GB
+- Storage: 10GB
+- OS: Windows 10, Ubuntu 18.04, macOS 10.14
+
+### Recommended:
+- CPU: Intel i7 or equivalent
+- GPU: NVIDIA GTX 1060 or better
+- RAM: 16GB
+- Storage: 50GB SSD
+
+---
+
+## 🔒 Data Privacy & Security
+
+Following Philippine Data Privacy Act (RA 10173):
+
+- ✅ Encrypted data transmission
+- ✅ Access control and authentication
+- ✅ Audit logging
+- ✅ Regular security updates
+- ✅ Data retention policies
+
+---
+
+## 📚 Key Algorithms
+
+### 1. CRNN+CTC
+**Purpose:** Text recognition from images  
+**Strengths:** Handles variable-length sequences, no character segmentation needed  
+**Reference:** Shi et al. (2016)
+
+### 2. Multinomial Naive Bayes
+**Purpose:** Document classification  
+**Strengths:** Fast, efficient, works well with text data  
+**Reference:** McCallum & Nigam (1998)
+
+### 3. Named Entity Recognition
+**Purpose:** Extract entities (names, dates, places)  
+**Strengths:** Pre-trained, accurate, easy to use  
+**Reference:** spaCy (Honnibal & Montani, 2017)
+
+---
+
+## 🛠️ Troubleshooting
+
+### Low Accuracy?
+1. Increase training data (target: 10,000+ samples)
+2. Use data augmentation
+3. Train longer (100+ epochs)
+4. Clean your dataset
+
+### Out of Memory?
+1. Reduce batch size
+2. Use smaller image dimensions
+3. Use gradient accumulation
+4. Enable mixed precision
+
+### Slow Inference?
+1. Use GPU if available
+2. Batch process images
+3. Optimize model (ONNX)
+4. Cache frequent results
+
+---
+
+## 📖 Documentation
+
+- **IMPLEMENTATION_GUIDE.md** - Complete step-by-step guide
+- **API_DOCUMENTATION.md** - API reference (to be created)
+- **USER_MANUAL.md** - End-user guide (to be created)
+
+---
+
+## 🎓 Academic References
+
+### Key Papers:
+
+1. **CRNN**  
+   Shi, B., Bai, X., & Yao, C. (2016). An end-to-end trainable neural network for image-based sequence recognition and its application to scene text recognition. *IEEE TPAMI*.
+
+2. **CTC Loss**  
+   Graves, A., et al. (2006). Connectionist temporal classification: Labelling unsegmented sequence data with recurrent neural networks. *ICML*.
+
+3. **Naive Bayes**  
+   McCallum, A., & Nigam, K. (1998). A comparison of event models for naive bayes text classification. *AAAI Workshop*.
+
+4. **spaCy**  
+   Honnibal, M., & Montani, I. (2017). spaCy 2: Natural language understanding with Bloom embeddings, convolutional neural networks and incremental parsing.
+
+---
+
+## 👥 Contributors
+
+**Researchers:**
+- Shane Mark C. Blanco
+- Princess A. Pasamonte
+- Irish Faith G. Ramirez
+
+**Advisers:**
+- Mr. Rengel V. Corpuz (Technical Adviser)
+- Mr. Joselito T. Tan (Subject Teacher)
+
+**Institution:**  
+Tarlac State University  
+College of Computer Studies  
+Bachelor of Science in Computer Science
+
+---
+
+## 📞 Support
+
+For questions regarding this implementation:
+
+1. Review IMPLEMENTATION_GUIDE.md
+2. Check code documentation
+3. Consult with thesis advisers
+
+---
+
+## 📄 License
+
+This project is for academic purposes as part of a thesis requirement.
+
+---
+
+## ✅ Implementation Checklist
+
+### Phase 1: Setup ✓
+- [x] Install dependencies
+- [x] Set up project structure
+- [x] Prepare development environment
+
+### Phase 2: Data Preparation
+- [ ] Collect civil registry form images
+- [ ] Create annotations
+- [ ] Split into train/val/test sets
+
+### Phase 3: Model Development
+- [ ] Train CRNN+CTC model
+- [ ] Train document classifier
+- [ ] Integrate NER system
+
+### Phase 4: Web Application
+- [ ] Develop Flask/FastAPI backend
+- [ ] Create frontend interface
+- [ ] Implement database integration
+
+### Phase 5: Testing
+- [ ] Accuracy testing
+- [ ] Black-box testing
+- [ ] ISO 25010 evaluation
+- [ ] User acceptance testing
+
+### Phase 6: Deployment
+- [ ] Optimize for production
+- [ ] Set up server
+- [ ] Deploy application
+- [ ] Monitor performance
+
+---
+
+## 🎯 Success Metrics
+
+Target metrics for thesis evaluation:
+
+| Metric | Target | Status |
+|--------|--------|--------|
+| OCR Accuracy | > 95% | Pending |
+| CER | < 5% | Pending |
+| Classifier Accuracy | > 90% | Pending |
+| NER F1 Score | > 85% | Pending |
+| Response Time | < 2s | Pending |
+| System Uptime | > 99% | Pending |
+
+---
+
+**Good luck with your thesis defense! 🎓✨**
+
+For detailed implementation instructions, see **IMPLEMENTATION_GUIDE.md**

app.py

@@ -0,0 +1,1234 @@
+# app.py
+# ============================================================
+#  Flask API — Civil Registry Pipeline
+#
+#  TWO MODES (switch via USE_REAL_PIPELINE below):
+#
+#  USE_REAL_PIPELINE = False  →  fake data (safe, always works)
+#  USE_REAL_PIPELINE = True   →  calls pipeline.py (real models)
+#
+#  HOW TO ENABLE THE REAL PIPELINE:
+#  1. Set USE_REAL_PIPELINE = True
+#  2. Set PIPELINE_REPO_PATH to the absolute path of your repo
+#     e.g. r"C:\Users\YourName\Documents\thesis-repo"
+#  3. Make sure venv has all model dependencies installed
+#  4. Run: python app.py
+# ============================================================
+
+from flask import Flask, request, jsonify
+from flask_cors import CORS
+import os
+import sys
+import traceback
+from datetime import datetime
+
+# ── sys.path setup ────────────────────────────────────────────
+_BASE = os.path.dirname(os.path.abspath(__file__))
+for _p in [
+    _BASE,
+    os.path.join(_BASE, 'CRNN+CTC'),
+    os.path.join(_BASE, 'MNB'),
+    os.path.join(_BASE, 'spacyNER'),
+]:
+    if _p not in sys.path:
+        sys.path.insert(0, _p)
+
+app = Flask(__name__)
+CORS(app)
+
+# ── CONFIGURATION ─────────────────────────────────────────────
+USE_REAL_PIPELINE = False
+USE_TEMPLATE_MATCHING = True
+PIPELINE_REPO_PATH = r"C:\xampp\htdocs\python"
+# ─────────────────────────────────────────────────────────────
+
+# ── Load template matcher ─────────────────────────────────────
+try:
+    from template_matcher import (
+        extract_fields,
+        pdf_to_image,
+        detect_form_type,
+        _get_crnn,
+        _get_paddleocr,
+    )
+    _template_matcher_ok = True
+    print("[app.py] Template matcher loaded")
+
+    print("[app.py] Preloading CRNN+CTC model...")
+    _get_crnn()
+    print("[app.py] CRNN+CTC preloaded.")
+
+    print("[app.py] Preloading PaddleOCR...")
+    _get_paddleocr()
+    print("[app.py] PaddleOCR preloaded.")
+
+except Exception as _tm_err:
+    _template_matcher_ok = False
+    print(f"[app.py] Template matcher unavailable: {_tm_err}")
+
+# ── Load bridge (MNB + spaCyNER) ──────────────────────────────
+_bridge = None
+try:
+    from bridge import CivilRegistryBridge
+    print("[app.py] Loading MNB + spaCyNER bridge...")
+    _bridge = CivilRegistryBridge()
+    print("[app.py] Bridge (MNB + spaCyNER) ready.")
+except Exception as _br_err:
+    print(f"[app.py] Bridge unavailable (MNB/NER disabled): {_br_err}")
+
+TEMP_DIR = os.environ.get('TEMP_DIR', os.path.join('/tmp', 'uploads', 'temp'))
+
+# ── Load real pipeline (only if enabled) ─────────────────────
+_pipeline = None
+_pipeline_error = None
+
+if USE_REAL_PIPELINE:
+    try:
+        if PIPELINE_REPO_PATH not in sys.path:
+            sys.path.insert(0, PIPELINE_REPO_PATH)
+        from pipeline import CivilRegistryPipeline
+        print("[app.py] Loading pipeline models — this may take a moment...")
+        _pipeline = CivilRegistryPipeline()
+        print("[app.py] ✅ Pipeline ready")
+    except Exception:
+        _pipeline_error = traceback.format_exc()
+        print(f"[app.py] ❌ Pipeline failed to load:\n{_pipeline_error}")
+        print("[app.py] ⚠️ Falling back to fake data")
+
+
+# ── /process endpoint ─────────────────────────────────────────
+@app.route('/process', methods=['POST'])
+def process_document():
+    if 'file' not in request.files:
+        return jsonify({'status': 'error', 'message': 'No file provided'}), 400
+
+    file = request.files['file']
+    file2 = request.files.get('file2')
+    form_hint = request.form.get('form_hint', '1A')
+
+    hint_to_type = {
+        '1A': 'birth',
+        '2A': 'death',
+        '3A': 'marriage',
+        '90': 'marriage',
+    }
+    form_type = hint_to_type.get(form_hint, 'birth')
+
+    os.makedirs(TEMP_DIR, exist_ok=True)
+    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+
+    ext = os.path.splitext(file.filename)[1] or '.pdf'
+    saved_path = os.path.join(TEMP_DIR, f'upload_{timestamp}{ext}')
+    file.save(saved_path)
+
+    saved_path2 = None
+    if file2 and file2.filename:
+        ext2 = os.path.splitext(file2.filename)[1] or '.pdf'
+        saved_path2 = os.path.join(TEMP_DIR, f'upload_{timestamp}_bride{ext2}')
+        file2.save(saved_path2)
+
+    try:
+        if USE_REAL_PIPELINE and _pipeline is not None:
+            fields, confidence, form_class = _run_real_pipeline(
+                saved_path, form_hint, form_type, file2_path=saved_path2
+            )
+        elif USE_TEMPLATE_MATCHING and _template_matcher_ok:
+            fields, confidence, form_class = _run_template_pipeline(
+                saved_path, form_hint, file2_path=saved_path2
+            )
+        else:
+            fields, confidence, form_class = _run_fake_pipeline(form_hint)
+
+    except Exception as e:
+        tb = traceback.format_exc()
+        print(f"[app.py] ❌ Processing error:\n{tb}")
+        is_user_error = isinstance(e, ValueError)
+        return jsonify({
+            'status': 'error',
+            'message': str(e),
+            'trace': '' if is_user_error else tb,
+        }), 200 if is_user_error else 500
+
+    finally:
+        try:
+            os.remove(saved_path)
+        except Exception:
+            pass
+        if saved_path2:
+            try:
+                os.remove(saved_path2)
+            except Exception:
+                pass
+
+    preview_file = f'form_{form_class}_{timestamp}.html'
+    preview_path = os.path.join(TEMP_DIR, preview_file)
+    with open(preview_path, 'w', encoding='utf-8') as fh:
+        fh.write(_build_preview_html(form_class, fields))
+
+    mode_label = "pipeline" if (USE_REAL_PIPELINE and _pipeline) else "template/fake"
+
+    return jsonify({
+        'status': 'success',
+        'form_class': form_class,
+        'raw_text': f'Processed via {mode_label} — Form {form_class}',
+        'fields': fields,
+        'confidence': confidence,
+        'saved_file': preview_file,
+        'preview_url': f'/uploads/temp/{preview_file}',
+    })
+
+
+# ── /status endpoint ──────────────────────────────────────────
+@app.route('/status', methods=['GET'])
+def status():
+    return jsonify({
+        'mode': 'real_pipeline' if (USE_REAL_PIPELINE and _pipeline) else 'fake_data',
+        'pipeline_ready': _pipeline is not None,
+        'pipeline_error': _pipeline_error,
+        'repo_path': PIPELINE_REPO_PATH if USE_REAL_PIPELINE else None,
+    })
+
+
+# ── /debug endpoint ───────────────────────────────────────────
+@app.route('/debug', methods=['GET'])
+def debug():
+    try:
+        import pipeline as _pl_module  # noqa: F401
+        return jsonify({'import': 'ok', 'sys_path': sys.path[:6]})
+    except Exception:
+        return jsonify({
+            'import': 'FAILED',
+            'trace': traceback.format_exc(),
+            'sys_path': sys.path[:6]
+        }), 500
+
+
+# ═════════════════════════════════════════════════════════════
+#  REAL PIPELINE
+# ═════════════════════════════════════════════════════════════
+def _run_real_pipeline(file_path, form_hint, form_type, file2_path=None):
+    if form_hint == '90':
+        raw_groom = _pipeline.process_pdf(file_path, form_type='marriage')
+        groom_fields, groom_conf = _map_pipeline_output_form90(raw_groom, role='groom')
+
+        bride_fields = {}
+        bride_conf = {}
+        if file2_path:
+            raw_bride = _pipeline.process_pdf(file2_path, form_type='marriage')
+            bride_fields, bride_conf = _map_pipeline_output_form90(raw_bride, role='bride')
+
+        fields = {**bride_fields, **groom_fields}
+        confidence = {**bride_conf, **groom_conf}
+
+        for key in [
+            'registry_no', 'city_municipality', 'date_issuance', 'license_no',
+            'marriage_day', 'marriage_month', 'marriage_year',
+            'marriage_venue', 'marriage_city',
+            'groom_first', 'groom_middle', 'groom_last', 'groom_age',
+            'groom_citizenship', 'groom_mother_first', 'groom_mother_last',
+            'groom_father_first', 'groom_father_last',
+            'bride_first', 'bride_middle', 'bride_last', 'bride_age',
+            'bride_citizenship', 'bride_mother_first', 'bride_mother_last',
+            'bride_father_first', 'bride_father_last',
+        ]:
+            fields.setdefault(key, '')
+
+        return fields, confidence, '90'
+
+    raw_result = _pipeline.process_pdf(file_path, form_type=form_type)
+    actual_class = getattr(raw_result, 'form_class', None) or form_hint
+    class_map = {'form1a': '1A', 'form2a': '2A', 'form3a': '3A', 'form90': '90'}
+    form_class = class_map.get(str(actual_class).lower(), form_hint)
+
+    fields, confidence = _map_pipeline_output(raw_result, form_class)
+    return fields, confidence, form_class
+
+
+def _map_pipeline_output(raw: dict, form_hint: str):
+    confidence = {k: 0.90 for k in raw.keys()}
+
+    if form_hint == '1A':
+        fields = {
+            'registry_no': raw.get('registry_number') or raw.get('registry_no', ''),
+            'city_municipality': raw.get('city_municipality') or raw.get('city', ''),
+            'province': raw.get('province', ''),
+            'date_issuance': raw.get('date_issuance') or raw.get('date', ''),
+            'child_first': raw.get('child_first') or raw.get('name_of_child_first', ''),
+            'child_middle': raw.get('child_middle') or raw.get('name_of_child_middle', ''),
+            'child_last': raw.get('child_last') or raw.get('name_of_child_last', ''),
+            'sex': raw.get('sex', ''),
+            'dob_day': raw.get('dob_day') or raw.get('date_of_birth_day', ''),
+            'dob_month': raw.get('dob_month') or raw.get('date_of_birth_month', ''),
+            'dob_year': raw.get('dob_year') or raw.get('date_of_birth_year', ''),
+            'pob_hospital': raw.get('pob_hospital') or raw.get('place_of_birth_hospital', ''),
+            'pob_city': raw.get('pob_city') or raw.get('place_of_birth_city', ''),
+            'pob_province': raw.get('pob_province') or raw.get('place_of_birth_province', ''),
+            'mother_first': raw.get('mother_first') or raw.get('mother_name_first', ''),
+            'mother_middle': raw.get('mother_middle') or raw.get('mother_name_middle', ''),
+            'mother_last': raw.get('mother_last') or raw.get('mother_name_last', ''),
+            'mother_citizenship': raw.get('mother_citizenship') or raw.get('mother_nationality', ''),
+            'mother_age': raw.get('mother_age', ''),
+            'father_first': raw.get('father_first') or raw.get('father_name_first', ''),
+            'father_middle': raw.get('father_middle') or raw.get('father_name_middle', ''),
+            'father_last': raw.get('father_last') or raw.get('father_name_last', ''),
+            'father_citizenship': raw.get('father_citizenship') or raw.get('father_nationality', ''),
+            'parents_marriage_day': raw.get('parents_marriage_day', ''),
+            'parents_marriage_month': raw.get('parents_marriage_month', ''),
+            'parents_marriage_year': raw.get('parents_marriage_year', ''),
+            'parents_marriage_city': raw.get('parents_marriage_city', ''),
+            'parents_marriage_province': raw.get('parents_marriage_province', ''),
+            'date_submitted': raw.get('date_submitted') or raw.get('date_of_registration', ''),
+            'prepared_by': raw.get('prepared_by', ''),
+        }
+
+    elif form_hint == '2A':
+        fields = {
+            'registry_no': raw.get('registry_number') or raw.get('registry_no', ''),
+            'city_municipality': raw.get('city_municipality') or raw.get('city', ''),
+            'province': raw.get('province', ''),
+            'date_issuance': raw.get('date_issuance') or raw.get('date', ''),
+            'deceased_first': raw.get('deceased_first') or raw.get('name_of_deceased_first', ''),
+            'deceased_middle': raw.get('deceased_middle') or raw.get('name_of_deceased_middle', ''),
+            'deceased_last': raw.get('deceased_last') or raw.get('name_of_deceased_last', ''),
+            'sex': raw.get('sex', ''),
+            'age_years': raw.get('age_years') or raw.get('age', ''),
+            'civil_status': raw.get('civil_status', ''),
+            'citizenship': raw.get('citizenship') or raw.get('nationality', ''),
+            'dod_day': raw.get('dod_day') or raw.get('date_of_death_day', ''),
+            'dod_month': raw.get('dod_month') or raw.get('date_of_death_month', ''),
+            'dod_year': raw.get('dod_year') or raw.get('date_of_death_year', ''),
+            'pod_hospital': raw.get('pod_hospital') or raw.get('place_of_death_hospital', ''),
+            'pod_city': raw.get('pod_city') or raw.get('place_of_death_city', ''),
+            'pod_province': raw.get('pod_province') or raw.get('place_of_death_province', ''),
+            'cause_immediate': raw.get('cause_immediate') or raw.get('cause_of_death', ''),
+            'cause_antecedent': raw.get('cause_antecedent', ''),
+            'cause_underlying': raw.get('cause_underlying', ''),
+            'date_submitted': raw.get('date_submitted') or raw.get('date_of_registration', ''),
+        }
+
+    else:
+        fields = {
+            'registry_no': raw.get('registry_number') or raw.get('registry_no', ''),
+            'city_municipality': raw.get('city_municipality') or raw.get('city', ''),
+            'province': raw.get('province', ''),
+            'date_issuance': raw.get('date_issuance') or raw.get('date', ''),
+            'husband_first': raw.get('husband_first') or raw.get('husband_name_first', ''),
+            'husband_middle': raw.get('husband_middle') or raw.get('husband_name_middle', ''),
+            'husband_last': raw.get('husband_last') or raw.get('husband_name_last', ''),
+            'husband_age': raw.get('husband_age', ''),
+            'husband_citizenship': raw.get('husband_citizenship') or raw.get('husband_nationality', ''),
+            'husband_mother_first': raw.get('husband_mother_first', ''),
+            'husband_mother_last': raw.get('husband_mother_last', ''),
+            'husband_mother_citizenship': raw.get('husband_mother_citizenship', ''),
+            'husband_father_first': raw.get('husband_father_first', ''),
+            'husband_father_last': raw.get('husband_father_last', ''),
+            'husband_father_citizenship': raw.get('husband_father_citizenship', ''),
+            'wife_first': raw.get('wife_first') or raw.get('wife_name_first', ''),
+            'wife_middle': raw.get('wife_middle') or raw.get('wife_name_middle', ''),
+            'wife_last': raw.get('wife_last') or raw.get('wife_name_last', ''),
+            'wife_age': raw.get('wife_age', ''),
+            'wife_citizenship': raw.get('wife_citizenship') or raw.get('wife_nationality', ''),
+            'wife_mother_first': raw.get('wife_mother_first', ''),
+            'wife_mother_last': raw.get('wife_mother_last', ''),
+            'wife_mother_citizenship': raw.get('wife_mother_citizenship', ''),
+            'wife_father_first': raw.get('wife_father_first', ''),
+            'wife_father_last': raw.get('wife_father_last', ''),
+            'wife_father_citizenship': raw.get('wife_father_citizenship', ''),
+            'marriage_day': raw.get('marriage_day') or raw.get('date_of_marriage_day', ''),
+            'marriage_month': raw.get('marriage_month') or raw.get('date_of_marriage_month', ''),
+            'marriage_year': raw.get('marriage_year') or raw.get('date_of_marriage_year', ''),
+            'marriage_venue': raw.get('marriage_venue', ''),
+            'marriage_city': raw.get('marriage_city', ''),
+            'marriage_province': raw.get('marriage_province', ''),
+            'date_submitted': raw.get('date_submitted') or raw.get('date_of_registration', ''),
+        }
+
+    for k, v in raw.items():
+        if k not in fields and v:
+            fields[k] = v
+
+    return fields, confidence
+
+
+def _map_pipeline_output_form90(raw: dict, role: str):
+    confidence = {k: 0.90 for k in raw.keys()}
+
+    husband = raw.get('husband') or {}
+    wife = raw.get('wife') or {}
+    if not isinstance(husband, dict):
+        husband = {}
+    if not isinstance(wife, dict):
+        wife = {}
+
+    dom_raw = raw.get('date_of_marriage') or ''
+    dom_parts = [p.strip() for p in str(dom_raw).split(',') if p.strip()]
+    marriage_day = dom_parts[0] if len(dom_parts) > 0 else ''
+    marriage_month = dom_parts[1] if len(dom_parts) > 1 else ''
+    marriage_year = dom_parts[2] if len(dom_parts) > 2 else ''
+
+    pom_raw = raw.get('place_of_marriage') or ''
+    pom_parts = [p.strip() for p in str(pom_raw).split(',') if p.strip()]
+    marriage_venue = pom_parts[0] if len(pom_parts) > 0 else ''
+    marriage_city = pom_parts[1] if len(pom_parts) > 1 else ''
+
+    shared = {
+        'registry_no': str(raw.get('registry_number') or '').strip(),
+        'city_municipality': marriage_city,
+        'date_issuance': str(raw.get('date_of_registration') or '').strip(),
+        'license_no': str(raw.get('license_no') or raw.get('license_number') or '').strip(),
+        'marriage_day': marriage_day,
+        'marriage_month': marriage_month,
+        'marriage_year': marriage_year,
+        'marriage_venue': marriage_venue,
+        'marriage_city': marriage_city,
+        'marriage_province': str(raw.get('province') or '').strip(),
+    }
+
+    if role == 'groom':
+        person = husband
+        fields = {
+            **shared,
+            'groom_first': str(person.get('first_name') or person.get('first') or raw.get('groom_first') or '').strip(),
+            'groom_middle': str(person.get('middle_name') or person.get('middle') or raw.get('groom_middle') or '').strip(),
+            'groom_last': str(person.get('last_name') or person.get('last') or raw.get('groom_last') or '').strip(),
+            'groom_age': str(person.get('age') or raw.get('groom_age') or '').strip(),
+            'groom_citizenship': str(person.get('citizenship') or person.get('nationality') or raw.get('groom_citizenship') or '').strip(),
+            'groom_civil_status': str(person.get('civil_status') or '').strip(),
+            'groom_residence': str(person.get('residence') or person.get('address') or '').strip(),
+            'groom_mother_first': str(person.get('mother_first') or person.get('mother_name') or '').strip(),
+            'groom_mother_last': str(person.get('mother_last') or '').strip(),
+            'groom_father_first': str(person.get('father_first') or person.get('father_name') or '').strip(),
+            'groom_father_last': str(person.get('father_last') or '').strip(),
+        }
+    else:
+        person = wife
+        fields = {
+            **shared,
+            'bride_first': str(person.get('first_name') or person.get('first') or raw.get('bride_first') or '').strip(),
+            'bride_middle': str(person.get('middle_name') or person.get('middle') or raw.get('bride_middle') or '').strip(),
+            'bride_last': str(person.get('last_name') or person.get('last') or raw.get('bride_last') or '').strip(),
+            'bride_age': str(person.get('age') or raw.get('bride_age') or '').strip(),
+            'bride_citizenship': str(person.get('citizenship') or person.get('nationality') or raw.get('bride_citizenship') or '').strip(),
+            'bride_civil_status': str(person.get('civil_status') or '').strip(),
+            'bride_residence': str(person.get('residence') or person.get('address') or '').strip(),
+            'bride_mother_first': str(person.get('mother_first') or person.get('mother_name') or '').strip(),
+            'bride_mother_last': str(person.get('mother_last') or '').strip(),
+            'bride_father_first': str(person.get('father_first') or person.get('father_name') or '').strip(),
+            'bride_father_last': str(person.get('father_last') or '').strip(),
+        }
+
+    fields = {k: v for k, v in fields.items() if v}
+    return fields, confidence
+
+
+# ═════════════════════════════════════════════════════════════
+#  TEMPLATE MATCHING PIPELINE
+# ═════════════════════════════════════════════════════════════
+def _run_template_pipeline(file_path, form_hint, file2_path=None):
+    img_path = file_path
+    if file_path.lower().endswith('.pdf'):
+        img_path = pdf_to_image(file_path) or file_path
+
+    hint_to_source = {'1A': '102', '2A': '103', '3A': '97', '90': '90'}
+    source_type = hint_to_source.get(form_hint, '102')
+
+    detected_source = detect_form_type(img_path)
+    source_to_hint = {'102': '1A', '103': '2A', '97': '3A', '90': '90'}
+    detected_hint = source_to_hint.get(detected_source, '1A')
+
+    CERT_HINTS = {'1A', '2A', '3A'}
+    LICENSE_HINTS = {'90'}
+
+    if form_hint in CERT_HINTS and detected_hint in LICENSE_HINTS:
+        raise ValueError(
+            'Wrong form uploaded. This appears to be a Marriage License (Form 90). '
+            'Please upload it under Marriage License instead.'
+        )
+    if form_hint in LICENSE_HINTS and detected_hint in CERT_HINTS:
+        raise ValueError(
+            f'Wrong form uploaded. This appears to be a Form {detected_hint} (Certification). '
+            f'Please upload it under Certifications instead.'
+        )
+
+    if form_hint == '1A':
+        form_hint = detected_hint
+        source_type = detected_source
+
+    raw = extract_fields(img_path, source_type)
+
+    if isinstance(raw, dict) and raw.get('status') == 'error':
+        print('[app.py] Blank page detected — skipping extraction')
+        raise ValueError('Blank page detected. Please try another file.')
+
+    if form_hint == '90' and file2_path:
+        img_path2 = file2_path
+        if file2_path.lower().endswith('.pdf'):
+            img_path2 = pdf_to_image(file2_path) or file2_path
+        raw2 = extract_fields(img_path2, '90')
+        if isinstance(raw2, dict) and raw2.get('status') == 'error':
+            print(f'[app.py] Bride file aborted: {raw2["message"]}')
+        else:
+            raw = {**raw, **raw2}
+
+    fields = _map_template_output(raw, form_hint)
+    form_class = form_hint if form_hint in ('1A', '2A', '3A', '90') else '1A'
+
+    if _bridge is not None:
+        try:
+            ner_text = _raw_to_ner_text(raw, source_type)
+
+            if source_type == '90':
+                mnb_result = {
+                    'label': 'Form 90 - Application for Marriage License',
+                    'form_code': 'form90',
+                    'confidence': 1.0
+                }
+            else:
+                mnb_result = _bridge.mnb.classify_full(ner_text)
+
+            print(f'[app.py] MNB: {mnb_result["label"]} ({mnb_result["confidence"]:.1%})')
+
+            if source_type == '102':
+                ner_form = _bridge.filler.fill_form_1a(ner_text)
+            elif source_type == '103':
+                ner_form = _bridge.filler.fill_form_2a(ner_text)
+            elif source_type == '97':
+                ner_form = _bridge.filler.fill_form_3a(ner_text)
+            else:
+                ner_form = _bridge.filler.fill_form_90(ner_text, ner_text)
+
+            ner_fields = _ner_to_fields(ner_form, raw, form_hint)
+
+            TRUST_TEMPLATE_KEYS = {
+                'groom_name', 'bride_name',
+                'groom_age', 'bride_age', 'husband_age', 'wife_age',
+                'groom_dob', 'bride_dob', 'husband_dob', 'wife_dob',
+                'registry_no', 'license_no', 'date_issuance',
+            }
+
+            def _better(ner_val, tmpl_val, key):
+                if key in TRUST_TEMPLATE_KEYS:
+                    return tmpl_val if tmpl_val else ner_val
+                if not ner_val:
+                    return tmpl_val
+                if not tmpl_val:
+                    return ner_val
+                return ner_val if len(ner_val) >= len(tmpl_val) else tmpl_val
+
+            fields = {k: _better(ner_fields.get(k, ''), v, k) for k, v in fields.items()}
+            for k, v in ner_fields.items():
+                if k not in fields and v:
+                    fields[k] = v
+
+            ner_count = sum(1 for v in ner_fields.values() if v)
+            print(f'[app.py] NER enriched {ner_count} fields')
+            confidence = {k: mnb_result['confidence'] for k in fields}
+
+        except Exception as _ner_err:
+            print(f'[app.py] NER error (using template only): {_ner_err}')
+            confidence = {k: 0.85 for k in fields}
+    else:
+        confidence = {k: 0.85 for k in fields}
+
+    non_empty = {k: v for k, v in fields.items() if v}
+    print(f'[app.py] form_class={form_class}, {len(non_empty)}/{len(fields)} non-empty fields')
+    for k, v in non_empty.items():
+        print(f'  {k:<30} = {v}')
+
+    return fields, confidence, form_class
+
+
+def _clean_ocr(text: str) -> str:
+    import re
+    if not text:
+        return text
+    text = re.sub(r'\s+', ' ', text).strip()
+    text = text.strip('.,;:')
+    return text
+
+
+def _clean_age(text: str) -> str:
+    import re
+    nums = re.findall(r'\b\d+\b', text)
+    return nums[-1] if nums else _clean_ocr(text)
+
+
+def _clean_civil_status(text: str) -> str:
+    t = text.lower().replace(' ', '')
+    if any(x in t for x in ['singl', 'fngle', 'fingle', 'single']):
+        return 'Single'
+    if any(x in t for x in ['marr', 'maried', 'married']):
+        return 'Married'
+    if any(x in t for x in ['widow', 'widw']):
+        return 'Widowed'
+    if any(x in t for x in ['separ', 'annul']):
+        return 'Separated'
+    return _clean_ocr(text)
+
+
+def _map_template_output(raw: dict, form_hint: str) -> dict:
+    def g(key, *aliases):
+        for k in (key,) + aliases:
+            if raw.get(k):
+                return raw[k]
+        return ''
+
+    if form_hint == '1A':
+        return {
+            'registry_no': g('registry_no'),
+            'city_municipality': g('city_municipality'),
+            'province': g('province'),
+            'date_submitted': g('registration_date'),
+            'child_first': g('name_first'),
+            'child_middle': g('name_middle'),
+            'child_last': g('name_last'),
+            'sex': g('sex'),
+            'dob_day': g('dob_day'),
+            'dob_month': g('dob_month'),
+            'dob_year': g('dob_year'),
+            'pob_city': g('place_of_birth'),
+            'mother_first': g('mother_name'),
+            'mother_citizenship': g('mother_citizenship'),
+            'father_first': g('father_name'),
+            'father_citizenship': g('father_citizenship'),
+            'parents_marriage_month': g('marriage_date'),
+            'parents_marriage_city': g('marriage_place'),
+        }
+
+    elif form_hint == '2A':
+        cause = ' / '.join(filter(None, [
+            g('cause_immediate'), g('cause_antecedent'), g('cause_underlying')
+        ]))
+        return {
+            'registry_no': g('registry_no'),
+            'city_municipality': g('city_municipality'),
+            'province': g('province'),
+            'date_submitted': g('registration_date'),
+            'deceased_first': g('deceased_name'),
+            'sex': g('sex'),
+            'age_years': g('age'),
+            'civil_status': g('civil_status'),
+            'citizenship': g('citizenship'),
+            'dod_full': g('date_of_death'),
+            'pod_hospital': g('place_of_death'),
+            'cause_immediate': cause,
+        }
+
+    elif form_hint == '3A':
+        return {
+            'registry_no': g('registry_no'),
+            'city_municipality': g('city_municipality'),
+            'province': g('province'),
+            'date_submitted': g('registration_date'),
+            'husband_first': g('husband_name_first'),
+            'husband_middle': g('husband_name_middle'),
+            'husband_last': g('husband_name_last'),
+            'husband_age': g('husband_age'),
+            'husband_citizenship': g('husband_citizenship'),
+            'husband_mother_first': g('husband_mother_name'),
+            'husband_father_first': g('husband_father_name'),
+            'husband_mother_citizenship': g('husband_mother_citizenship'),
+            'husband_father_citizenship': g('husband_father_citizenship'),
+            'wife_first': g('wife_name_first'),
+            'wife_middle': g('wife_name_middle'),
+            'wife_last': g('wife_name_last'),
+            'wife_age': g('wife_age'),
+            'wife_citizenship': g('wife_citizenship'),
+            'wife_mother_first': g('wife_mother_name'),
+            'wife_father_first': g('wife_father_name'),
+            'wife_mother_citizenship': g('wife_mother_citizenship'),
+            'wife_father_citizenship': g('wife_father_citizenship'),
+            'marriage_venue': g('place_of_marriage'),
+            'marriage_city': g('city_municipality'),
+            'marriage_month': g('date_of_marriage'),
+        }
+
+    else:
+        def _join_name(*keys):
+            return ' '.join(raw.get(k, '') for k in keys).strip()
+
+        return {
+            'registry_no': g('registry_no'),
+            'city_municipality': g('city_municipality'),
+            'province': raw.get('province', '').strip(),
+            'license_no': g('marriage_license_no'),
+            'date_issuance': g('date_issued'),
+            'groom_name': _join_name('groom_name_first', 'groom_name_middle', 'groom_name_last'),
+            'groom_dob': g('groom_dob'),
+            'groom_age': g('groom_age'),
+            'groom_place_of_birth': g('groom_place_of_birth'),
+            'groom_sex': g('groom_sex'),
+            'groom_citizenship': g('groom_citizenship'),
+            'groom_civil_status': g('groom_civil_status'),
+            'groom_residence': g('groom_residence'),
+            'groom_religion': g('groom_religion'),
+            'groom_father_first': g('groom_father_name'),
+            'groom_father_citizenship': g('groom_father_citizenship'),
+            'groom_mother_first': g('groom_mother_name'),
+            'groom_mother_citizenship': g('groom_mother_citizenship'),
+            'bride_name': _join_name('bride_name_first', 'bride_name_middle', 'bride_name_last'),
+            'bride_dob': g('bride_dob'),
+            'bride_age': g('bride_age'),
+            'bride_place_of_birth': g('bride_place_of_birth'),
+            'bride_sex': g('bride_sex'),
+            'bride_citizenship': g('bride_citizenship'),
+            'bride_civil_status': g('bride_civil_status'),
+            'bride_residence': g('bride_residence'),
+            'bride_religion': g('bride_religion'),
+            'bride_father_first': g('bride_father_name'),
+            'bride_father_citizenship': g('bride_father_citizenship'),
+            'bride_mother_first': g('bride_mother_name'),
+            'bride_mother_citizenship': g('bride_mother_citizenship'),
+        }
+
+
+# ═════════════════════════════════════════════════════════════
+#  BRIDGE HELPERS
+# ═════════════════════════════════════════════════════════════
+def _raw_to_ner_text(raw: dict, source_type: str) -> str:
+    def g(*keys):
+        for k in keys:
+            v = raw.get(k, '')
+            if v:
+                return str(v)
+        return ''
+
+    if source_type == '102':
+        return (
+            f"Registry No.: {g('registry_no')}\n"
+            f"Date of Registration: {g('registration_date')}\n"
+            f"1. NAME: {g('name_first')} {g('name_middle')} {g('name_last')}\n"
+            f"2. SEX: {g('sex')}\n"
+            f"3. DATE OF BIRTH: {g('dob_month')} {g('dob_day')}, {g('dob_year')}\n"
+            f"4. PLACE OF BIRTH: {g('place_of_birth')}\n"
+            f"MOTHER:\n"
+            f"7. MAIDEN NAME: {g('mother_name')}\n"
+            f"8. CITIZENSHIP/NATIONALITY: {g('mother_citizenship')}\n"
+            f"FATHER:\n"
+            f"14. NAME: {g('father_name')}\n"
+            f"15. CITIZENSHIP/NATIONALITY: {g('father_citizenship')}\n"
+            f"MARRIAGE OF PARENTS:\n"
+            f"20a. DATE: {g('marriage_date')}\n"
+            f"20b. PLACE: {g('marriage_place')}\n"
+        )
+
+    elif source_type == '103':
+        return (
+            f"Registry No.: {g('registry_no')}\n"
+            f"Date of Registration: {g('registration_date')}\n"
+            f"1. NAME (First): {g('deceased_name')}\n"
+            f"2. SEX: {g('sex')}\n"
+            f"4. AGE: {g('age')}\n"
+            f"9. CIVIL STATUS: {g('civil_status')}\n"
+            f"7. CITIZENSHIP/NATIONALITY: {g('citizenship')}\n"
+            f"6. DATE OF DEATH: {g('date_of_death')}\n"
+            f"5. PLACE OF DEATH: {g('place_of_death')}\n"
+            f"17. CAUSE OF DEATH: {g('cause_immediate')}\n"
+            f"Antecedent cause: {g('cause_antecedent')}\n"
+            f"Underlying cause: {g('cause_underlying')}\n"
+        )
+
+    elif source_type == '97':
+        return (
+            f"Registry No.: {g('registry_no')}\n"
+            f"Date of Registration: {g('registration_date')}\n"
+            f"HUSBAND:\n"
+            f"1. NAME (First): {g('husband_name_first')}  (Middle): {g('husband_name_middle')}  (Last): {g('husband_name_last')}\n"
+            f"2b. AGE: {g('husband_age')}\n"
+            f"4b. CITIZENSHIP/NATIONALITY: {g('husband_citizenship')}\n"
+            f"8. NAME OF FATHER: {g('husband_father_name')}\n"
+            f"8b. FATHER CITIZENSHIP/NATIONALITY: {g('husband_father_citizenship')}\n"
+            f"10. NAME OF MOTHER: {g('husband_mother_name')}\n"
+            f"10b. MOTHER CITIZENSHIP/NATIONALITY: {g('husband_mother_citizenship')}\n"
+            f"WIFE:\n"
+            f"1. NAME (First): {g('wife_name_first')}  (Middle): {g('wife_name_middle')}  (Last): {g('wife_name_last')}\n"
+            f"2b. AGE: {g('wife_age')}\n"
+            f"4b. CITIZENSHIP/NATIONALITY: {g('wife_citizenship')}\n"
+            f"8. NAME OF FATHER: {g('wife_father_name')}\n"
+            f"8b. FATHER CITIZENSHIP/NATIONALITY: {g('wife_father_citizenship')}\n"
+            f"10. NAME OF MOTHER: {g('wife_mother_name')}\n"
+            f"10b. MOTHER CITIZENSHIP/NATIONALITY: {g('wife_mother_citizenship')}\n"
+            f"15. PLACE OF MARRIAGE: {g('place_of_marriage')}\n"
+            f"16. DATE OF MARRIAGE: {g('date_of_marriage')}\n"
+        )
+
+    else:
+        return (
+            f"GROOM:\n"
+            f"1. NAME (First): {g('groom_name_first')}  (Middle): {g('groom_name_middle')}  (Last): {g('groom_name_last')}\n"
+            f"2. DATE OF BIRTH: {g('groom_dob')}\n"
+            f"3. PLACE OF BIRTH: {g('groom_place_of_birth')}\n"
+            f"4. SEX: {g('groom_sex')}\n"
+            f"5. CITIZENSHIP/NATIONALITY: {g('groom_citizenship')}\n"
+            f"NAME OF FATHER: {g('groom_father_name')}\n"
+            f"FATHER CITIZENSHIP/NATIONALITY: {g('groom_father_citizenship')}\n"
+            f"NAME OF MOTHER: {g('groom_mother_name')}\n"
+            f"MOTHER CITIZENSHIP/NATIONALITY: {g('groom_mother_citizenship')}\n"
+            f"BRIDE:\n"
+            f"1. NAME (First): {g('bride_name_first')}  (Middle): {g('bride_name_middle')}  (Last): {g('bride_name_last')}\n"
+            f"2. DATE OF BIRTH: {g('bride_dob')}\n"
+            f"3. PLACE OF BIRTH: {g('bride_place_of_birth')}\n"
+            f"4. SEX: {g('bride_sex')}\n"
+            f"5. CITIZENSHIP/NATIONALITY: {g('bride_citizenship')}\n"
+            f"NAME OF FATHER: {g('bride_father_name')}\n"
+            f"FATHER CITIZENSHIP/NATIONALITY: {g('bride_father_citizenship')}\n"
+            f"NAME OF MOTHER: {g('bride_mother_name')}\n"
+            f"MOTHER CITIZENSHIP/NATIONALITY: {g('bride_mother_citizenship')}\n"
+        )
+
+
+def _split_name(full: str):
+    parts = (full or '').split()
+    if not parts:
+        return '', '', ''
+    first = parts[0]
+    last = parts[-1] if len(parts) > 1 else ''
+    mid = ' '.join(parts[1:-1]) if len(parts) > 2 else ''
+    return first, mid, last
+
+
+def _ner_to_fields(form, raw: dict, form_hint: str) -> dict:
+    def r(*keys):
+        for k in keys:
+            v = raw.get(k, '')
+            if v:
+                return v
+        return ''
+
+    def ga(attr, *fallback_keys):
+        v = getattr(form, attr, None) or ''
+        return v or r(*fallback_keys)
+
+    if form_hint == '1A':
+        cf, cm, cl = _split_name(getattr(form, 'name_of_child', '') or '')
+        return {
+            'registry_no': ga('registry_number', 'registry_no'),
+            'city_municipality': r('city_municipality'),
+            'province': r('province'),
+            'date_submitted': ga('date_of_registration', 'registration_date'),
+            'child_first': cf or r('name_first'),
+            'child_middle': cm or r('name_middle'),
+            'child_last': cl or r('name_last'),
+            'sex': ga('sex', 'sex'),
+            'dob_day': r('dob_day'),
+            'dob_month': r('dob_month'),
+            'dob_year': r('dob_year'),
+            'pob_city': ga('place_of_birth', 'place_of_birth'),
+            'mother_first': ga('name_of_mother', 'mother_name'),
+            'mother_citizenship': ga('nationality_of_mother', 'mother_citizenship'),
+            'father_first': ga('name_of_father', 'father_name'),
+            'father_citizenship': ga('nationality_of_father', 'father_citizenship'),
+            'parents_marriage_month': ga('date_of_marriage_of_parents', 'marriage_date'),
+            'parents_marriage_city': ga('place_of_marriage_of_parents', 'marriage_place'),
+        }
+
+    elif form_hint == '2A':
+        cause = ' / '.join(filter(None, [
+            getattr(form, 'cause_of_death', ''),
+            getattr(form, 'cause_antecedent', ''),
+            getattr(form, 'cause_underlying', ''),
+        ])) or ' / '.join(filter(None, [
+            r('cause_immediate'), r('cause_antecedent'), r('cause_underlying')
+        ]))
+        return {
+            'registry_no': ga('registry_number', 'registry_no'),
+            'city_municipality': r('city_municipality'),
+            'province': r('province'),
+            'date_submitted': ga('date_of_registration', 'registration_date'),
+            'deceased_first': ga('name_of_deceased', 'deceased_name'),
+            'sex': ga('sex', 'sex'),
+            'age_years': ga('age', 'age'),
+            'civil_status': ga('civil_status', 'civil_status'),
+            'citizenship': ga('nationality', 'citizenship'),
+            'dod_full': ga('date_of_death', 'date_of_death'),
+            'pod_hospital': ga('place_of_death', 'place_of_death'),
+            'cause_immediate': cause,
+        }
+
+    elif form_hint == '3A':
+        h = getattr(form, 'husband', None)
+        w = getattr(form, 'wife', None)
+        hd = h.to_dict() if h else {}
+        wd = w.to_dict() if w else {}
+        return {
+            'registry_no': ga('registry_number', 'registry_no'),
+            'city_municipality': r('city_municipality'),
+            'province': r('province'),
+            'date_submitted': ga('date_of_registration', 'registration_date'),
+            'husband_first': hd.get('name') or r('husband_name_first'),
+            'husband_middle': r('husband_name_middle'),
+            'husband_last': r('husband_name_last'),
+            'husband_age': hd.get('age') or r('husband_age'),
+            'husband_citizenship': hd.get('nationality') or r('husband_citizenship'),
+            'husband_mother_first': hd.get('name_of_mother') or r('husband_mother_name'),
+            'husband_mother_citizenship': hd.get('nationality_of_mother') or r('husband_mother_citizenship'),
+            'husband_father_first': hd.get('name_of_father') or r('husband_father_name'),
+            'husband_father_citizenship': hd.get('nationality_of_father') or r('husband_father_citizenship'),
+            'wife_first': wd.get('name') or r('wife_name_first'),
+            'wife_middle': r('wife_name_middle'),
+            'wife_last': r('wife_name_last'),
+            'wife_age': wd.get('age') or r('wife_age'),
+            'wife_citizenship': wd.get('nationality') or r('wife_citizenship'),
+            'wife_mother_first': wd.get('name_of_mother') or r('wife_mother_name'),
+            'wife_mother_citizenship': wd.get('nationality_of_mother') or r('wife_mother_citizenship'),
+            'wife_father_first': wd.get('name_of_father') or r('wife_father_name'),
+            'wife_father_citizenship': wd.get('nationality_of_father') or r('wife_father_citizenship'),
+            'marriage_venue': ga('place_of_marriage', 'place_of_marriage'),
+            'marriage_city': r('city_municipality'),
+            'marriage_month': ga('date_of_marriage', 'date_of_marriage'),
+        }
+
+    else:
+        groom = getattr(form, 'groom', None)
+        bride = getattr(form, 'bride', None)
+        gd = groom.to_dict() if groom else {}
+        bd = bride.to_dict() if bride else {}
+        return {
+            'registry_no': r('registry_no'),
+            'city_municipality': r('city_municipality'),
+            'province': r('province'),
+            'license_no': r('marriage_license_no'),
+            'date_issuance': r('date_issued'),
+            'groom_name': (gd.get('name_of_applicant') or ' '.join(filter(None, [
+                r('groom_name_first'), r('groom_name_middle'), r('groom_name_last')
+            ]))).strip(),
+            'groom_dob': gd.get('date_of_birth') or r('groom_dob'),
+            'groom_age': gd.get('age') or r('groom_age'),
+            'groom_place_of_birth': gd.get('place_of_birth') or r('groom_place_of_birth'),
+            'groom_sex': gd.get('sex') or r('groom_sex'),
+            'groom_citizenship': gd.get('citizenship') or r('groom_citizenship'),
+            'groom_civil_status': gd.get('civil_status', r('groom_civil_status')),
+            'groom_residence': gd.get('residence') or r('groom_residence'),
+            'groom_religion': gd.get('religion') or r('groom_religion'),
+            'groom_father_first': gd.get('name_of_father') or r('groom_father_name'),
+            'groom_father_citizenship': gd.get('father_citizenship') or r('groom_father_citizenship'),
+            'groom_mother_first': gd.get('maiden_name_of_mother') or r('groom_mother_name'),
+            'groom_mother_citizenship': gd.get('mother_citizenship') or r('groom_mother_citizenship'),
+            'bride_name': (bd.get('name_of_applicant') or ' '.join(filter(None, [
+                r('bride_name_first'), r('bride_name_middle'), r('bride_name_last')
+            ]))).strip(),
+            'bride_dob': bd.get('date_of_birth') or r('bride_dob'),
+            'bride_age': bd.get('age') or r('bride_age'),
+            'bride_place_of_birth': bd.get('place_of_birth') or r('bride_place_of_birth'),
+            'bride_sex': bd.get('sex') or r('bride_sex'),
+            'bride_citizenship': bd.get('citizenship') or r('bride_citizenship'),
+            'bride_civil_status': bd.get('civil_status', r('bride_civil_status')),
+            'bride_residence': bd.get('residence') or r('bride_residence'),
+            'bride_religion': bd.get('religion') or r('bride_religion'),
+            'bride_father_first': bd.get('name_of_father') or r('bride_father_name'),
+            'bride_father_citizenship': bd.get('father_citizenship') or r('bride_father_citizenship'),
+            'bride_mother_first': bd.get('maiden_name_of_mother') or r('bride_mother_name'),
+            'bride_mother_citizenship': bd.get('mother_citizenship') or r('bride_mother_citizenship'),
+        }
+
+
+# ═════════════════════════════════════════════════════════════
+#  FAKE PIPELINE
+# ═════════════════════════════════════════════════════════════
+def _run_fake_pipeline(form_hint):
+    if form_hint == '1A':
+        fields = {
+            'registry_no': '2026-BC-00123',
+            'city_municipality': 'Tarlac City',
+            'province': 'Tarlac',
+            'date_issuance': datetime.now().strftime('%B %d, %Y'),
+            'child_first': 'Maria Luisa',
+            'child_middle': 'Dela Cruz',
+            'child_last': 'Santos',
+            'sex': 'Female',
+            'dob_day': '10',
+            'dob_month': 'January',
+            'dob_year': '2026',
+            'pob_city': 'Tarlac City',
+            'pob_province': 'Tarlac',
+            'mother_first': 'Rosa',
+            'mother_middle': 'Reyes',
+            'mother_last': 'Dela Cruz',
+            'mother_citizenship': 'Filipino',
+            'mother_age': '28',
+            'father_first': 'Juan Pedro',
+            'father_middle': '',
+            'father_last': 'Santos',
+            'father_citizenship': 'Filipino',
+            'parents_marriage_day': '12',
+            'parents_marriage_month': 'June',
+            'parents_marriage_year': '2020',
+            'parents_marriage_city': 'Tarlac City',
+            'parents_marriage_province': 'Tarlac',
+            'date_submitted': 'January 15, 2026',
+            'processed_by': 'John Doe',
+            'verified_position': 'City Civil Registrar',
+            'issued_to': 'Rosa Reyes Dela Cruz',
+            'amount_paid': '75.00',
+            'or_number': 'OR-2026-00456',
+            'date_paid': datetime.now().strftime('%B %d, %Y'),
+        }
+
+    elif form_hint == '2A':
+        fields = {
+            'registry_no': '2026-DC-00045',
+            'city_municipality': 'Tarlac City',
+            'province': 'Tarlac',
+            'date_issuance': datetime.now().strftime('%B %d, %Y'),
+            'deceased_first': 'Roberto',
+            'deceased_middle': 'Cruz',
+            'deceased_last': 'Villanueva',
+            'sex': 'Male',
+            'age_years': '72',
+            'civil_status': 'Married',
+            'citizenship': 'Filipino',
+            'dod_day': '28',
+            'dod_month': 'January',
+            'dod_year': '2026',
+            'pod_hospital': 'Tarlac Provincial Hospital',
+            'pod_city': 'Tarlac City',
+            'pod_province': 'Tarlac',
+            'cause_immediate': 'Cardiopulmonary Arrest',
+            'date_submitted': 'February 1, 2026',
+            'processed_by': 'John Doe',
+        }
+
+    elif form_hint == '3A':
+        fields = {
+            'registry_no': '2026-MC-00112',
+            'city_municipality': 'Tarlac City',
+            'province': 'Tarlac',
+            'date_issuance': datetime.now().strftime('%B %d, %Y'),
+            'husband_first': 'Carlos',
+            'husband_middle': 'Mendoza',
+            'husband_last': 'Reyes',
+            'husband_age': '29',
+            'husband_citizenship': 'Filipino',
+            'husband_mother_first': 'Ana',
+            'husband_father_first': 'Pedro',
+            'husband_mother_citizenship': 'Filipino',
+            'husband_father_citizenship': 'Filipino',
+            'wife_first': 'Elena',
+            'wife_middle': 'Santos',
+            'wife_last': 'Garcia',
+            'wife_age': '27',
+            'wife_citizenship': 'Filipino',
+            'wife_mother_first': 'Luz',
+            'wife_father_first': 'Mario',
+            'wife_mother_citizenship': 'Filipino',
+            'wife_father_citizenship': 'Filipino',
+            'marriage_venue': 'St. Sebastian Cathedral',
+            'marriage_city': 'Tarlac City',
+            'marriage_month': 'January 25, 2026',
+            'date_submitted': 'January 26, 2026',
+        }
+
+    else:
+        fields = {
+            'registry_no': 'ML-2026-00788',
+            'city_municipality': 'Tarlac City',
+            'province': 'Tarlac',
+            'license_no': 'LIC-2026-0455',
+            'date_issuance': 'March 02, 2026',
+            'groom_name': 'Mark Anthony Dela Cruz',
+            'groom_dob': 'May 04, 1998',
+            'groom_age': '27',
+            'groom_place_of_birth': 'Tarlac City',
+            'groom_sex': 'MALE',
+            'groom_citizenship': 'Filipino',
+            'groom_civil_status': 'Single',
+            'groom_residence': 'San Roque, Tarlac',
+            'groom_religion': 'Catholic',
+            'groom_father_first': 'Jose Dela Cruz',
+            'groom_father_citizenship': 'Filipino',
+            'groom_mother_first': 'Maria Santos',
+            'groom_mother_citizenship': 'Filipino',
+            'bride_name': 'Jane Marie Garcia',
+            'bride_dob': 'August 11, 1999',
+            'bride_age': '26',
+            'bride_place_of_birth': 'Capas, Tarlac',
+            'bride_sex': 'FEMALE',
+            'bride_citizenship': 'Filipino',
+            'bride_civil_status': 'Single',
+            'bride_residence': 'Capas, Tarlac',
+            'bride_religion': 'Catholic',
+            'bride_father_first': 'Ramon Garcia',
+            'bride_father_citizenship': 'Filipino',
+            'bride_mother_first': 'Luisa Mendoza',
+            'bride_mother_citizenship': 'Filipino',
+        }
+
+    confidence = {k: 0.95 for k in fields}
+    form_class = form_hint if form_hint in ('1A', '2A', '3A', '90') else '1A'
+    return fields, confidence, form_class
+
+
+# ═════════════════════════════════════════════════════════════
+#  HTML PREVIEW
+# ═════════════════════════════════════════════��═══════════════
+def _build_preview_html(form_class: str, fields: dict) -> str:
+    def row(label, value):
+        return (
+            f'<tr><td class="lbl">{label}</td>'
+            f'<td class="val">{value or ""}</td></tr>'
+        )
+
+    rows = ""
+    title = f"Form {form_class}"
+
+    if form_class == '1A':
+        child = " ".join(filter(None, [
+            fields.get('child_first', ''),
+            fields.get('child_middle', ''),
+            fields.get('child_last', ''),
+        ])).strip()
+        rows = (
+            row('Registry No.', fields.get('registry_no', '')) +
+            row('City/Municipality', fields.get('city_municipality', '')) +
+            row('Province', fields.get('province', '')) +
+            row('Date Submitted', fields.get('date_submitted', '')) +
+            '<tr><td colspan="2">CHILD</td></tr>' +
+            row('Name', child) +
+            row('Sex', fields.get('sex', '')) +
+            row('Date of Birth', " ".join(filter(None, [
+                fields.get('dob_month', ''),
+                fields.get('dob_day', ''),
+                fields.get('dob_year', ''),
+            ])).strip()) +
+            row('Place of Birth', fields.get('pob_city', '')) +
+            '<tr><td colspan="2">MOTHER</td></tr>' +
+            row('Name', fields.get('mother_first', '')) +
+            row('Citizenship', fields.get('mother_citizenship', '')) +
+            '<tr><td colspan="2">FATHER</td></tr>' +
+            row('Name', fields.get('father_first', '')) +
+            row('Citizenship', fields.get('father_citizenship', ''))
+        )
+        title = child or 'Form 1A — Live Birth'
+
+    elif form_class == '2A':
+        deceased = " ".join(filter(None, [
+            fields.get('deceased_first', ''),
+            fields.get('deceased_middle', ''),
+            fields.get('deceased_last', ''),
+        ])).strip()
+        rows = (
+            row('Registry No.', fields.get('registry_no', '')) +
+            row('City/Municipality', fields.get('city_municipality', '')) +
+            row('Province', fields.get('province', '')) +
+            row('Date Submitted', fields.get('date_submitted', '')) +
+            row('Name of Deceased', deceased) +
+            row('Sex', fields.get('sex', '')) +
+            row('Age', fields.get('age_years', '')) +
+            row('Civil Status', fields.get('civil_status', '')) +
+            row('Citizenship', fields.get('citizenship', '')) +
+            row('Date of Death', fields.get('dod_full', '')) +
+            row('Place of Death', fields.get('pod_hospital', '')) +
+            row('Cause of Death', fields.get('cause_immediate', ''))
+        )
+        title = deceased or 'Form 2A — Death Certificate'
+
+    elif form_class == '3A':
+        husband = " ".join(filter(None, [
+            fields.get('husband_first', ''),
+            fields.get('husband_middle', ''),
+            fields.get('husband_last', ''),
+        ])).strip()
+        wife = " ".join(filter(None, [
+            fields.get('wife_first', ''),
+            fields.get('wife_middle', ''),
+            fields.get('wife_last', ''),
+        ])).strip()
+        rows = (
+            row('Registry No.', fields.get('registry_no', '')) +
+            row('City/Municipality', fields.get('city_municipality', '')) +
+            row('Province', fields.get('province', '')) +
+            row('Date Submitted', fields.get('date_submitted', '')) +
+            '<tr><td colspan="2">HUSBAND</td></tr>' +
+            row('Name', husband) +
+            row('Age', fields.get('husband_age', '')) +
+            row('Citizenship', fields.get('husband_citizenship', '')) +
+            row('Mother', fields.get('husband_mother_first', '')) +
+            row('Father', fields.get('husband_father_first', '')) +
+            '<tr><td colspan="2">WIFE</td></tr>' +
+            row('Name', wife) +
+            row('Age', fields.get('wife_age', '')) +
+            row('Citizenship', fields.get('wife_citizenship', '')) +
+            row('Mother', fields.get('wife_mother_first', '')) +
+            row('Father', fields.get('wife_father_first', '')) +
+            '<tr><td colspan="2">MARRIAGE</td></tr>' +
+            row('Venue', fields.get('marriage_venue', '')) +
+            row('City', fields.get('marriage_city', '')) +
+            row('Date', fields.get('marriage_month', ''))
+        )
+        title = f'Form 3A — {husband} & {wife}' if (husband or wife) else 'Form 3A — Marriage Certificate'
+
+    else:
+        g = fields.get('groom_name', '')
+        b = fields.get('bride_name', '')
+        dom = " ".join(filter(None, [
+            fields.get('marriage_month', ''),
+            fields.get('marriage_day', ''),
+            fields.get('marriage_year', ''),
+        ])).strip()
+        pom = " ".join(filter(None, [
+            fields.get('marriage_venue', ''),
+            fields.get('marriage_city', ''),
+        ])).strip()
+
+        rows = (
+            row('Registry No.', fields.get('registry_no', '')) +
+            row('City/Municipality', fields.get('city_municipality', '')) +
+            row('Province', fields.get('province', '')) +
+            row('License No.', fields.get('license_no', '')) +
+            row('Date Issuance', fields.get('date_issuance', '')) +
+            '<tr><td colspan="2" style="padding:8px 0;font-weight:bold;background:#f9f9f9;text-align:center;">GROOM</td></tr>' +
+            row('Name', g) +
+            row('Date of Birth', fields.get('groom_dob', '')) +
+            row('Age', fields.get('groom_age', '')) +
+            row('Place of Birth', fields.get('groom_place_of_birth', '')) +
+            row('Sex', fields.get('groom_sex', '')) +
+            row('Citizenship', fields.get('groom_citizenship', '')) +
+            row('Civil Status', fields.get('groom_civil_status', '')) +
+            row('Residence', fields.get('groom_residence', '')) +
+            row('Religion', fields.get('groom_religion', '')) +
+            row('Father', fields.get('groom_father_first', '')) +
+            row('Father Citizenship', fields.get('groom_father_citizenship', '')) +
+            row('Mother', fields.get('groom_mother_first', '')) +
+            row('Mother Citizenship', fields.get('groom_mother_citizenship', '')) +
+            '<tr><td colspan="2" style="padding:8px 0;font-weight:bold;background:#f9f9f9;text-align:center;">BRIDE</td></tr>' +
+            row('Name', b) +
+            row('Date of Birth', fields.get('bride_dob', '')) +
+            row('Age', fields.get('bride_age', '')) +
+            row('Place of Birth', fields.get('bride_place_of_birth', '')) +
+            row('Sex', fields.get('bride_sex', '')) +
+            row('Citizenship', fields.get('bride_citizenship', '')) +
+            row('Civil Status', fields.get('bride_civil_status', '')) +
+            row('Residence', fields.get('bride_residence', '')) +
+            row('Religion', fields.get('bride_religion', '')) +
+            row('Father', fields.get('bride_father_first', '')) +
+            row('Father Citizenship', fields.get('bride_father_citizenship', '')) +
+            row('Mother', fields.get('bride_mother_first', '')) +
+            row('Mother Citizenship', fields.get('bride_mother_citizenship', '')) +
+            '<tr><td colspan="2" style="padding:8px 0;font-weight:bold;background:#f9f9f9;text-align:center;">MARRIAGE</td></tr>' +
+            row('Date of Marriage', dom) +
+            row('Place of Marriage', pom)
+        )
+        title = f'Form 90 — {g} & {b}' if (g or b) else 'Form 90 — Marriage License'
+
+    mode = 'REAL PIPELINE' if (USE_REAL_PIPELINE and _pipeline) else 'FAKE DATA (dev mode)'
+
+    return f"""<!DOCTYPE html><html><head><meta charset="UTF-8"><title>{title}</title>
+<style>
+body{{font-family:Arial,sans-serif;font-size:13px;padding:40px 50px;color:#111;}}
+h2{{font-size:15px;border-bottom:2px solid #333;padding-bottom:8px;margin-bottom:16px;}}
+.mode{{font-size:11px;color:#888;margin-bottom:12px;}}
+table{{width:100%;border-collapse:collapse;}}
+td{{padding:6px 8px;border-bottom:1px dotted #ccc;vertical-align:top;}}
+td.lbl{{width:220px;color:#555;}}
+td.val{{font-weight:bold;background:#fffde7;border-bottom:1px solid #f0d000;}}
+tr td[colspan]{{background:#f5f5f5;font-weight:bold;text-align:center;color:#333;border-bottom:2px solid #ddd;}}
+</style></head><body>
+<h2>LCR Form No. {form_class} — {fields.get('city_municipality', '')}</h2>
+<div class="mode">Mode: {mode}</div>
+<table>{rows}</table>
+</body></html>"""
+
+
+if __name__ == '__main__':
+    port = int(os.environ.get('PORT', 7860))
+    app.run(host='0.0.0.0', port=port, debug=False)

bridge.py

@@ -0,0 +1,376 @@
+# bridge.py
+# ============================================================
+# BRIDGE — connects the three algorithms
+#
+#   CRNN+CTC  (Irish)     →  field dict from field_extractor.py
+#   MNB       (Princess)  →  classifies form type
+#   spacyNER  (Shane)     →  extracts + assembles fields
+#
+# DROP THIS FILE in the ROOT of your project:
+#
+#   LCR-Document-Digitization-System/
+#   ├── CRNN+CTC/
+#   ├── MNB/
+#   ├── spacyNER/
+#   ├── bridge.py          ← HERE
+#   └── pipeline.py
+#
+# NOTE: nationality = citizenship (same field, different label per form)
+#       The _get() helper handles both names automatically.
+# ============================================================
+
+import sys
+import os
+from pathlib import Path
+
+# ── Make all three algorithm folders importable ──────────────
+_ROOT = Path(__file__).resolve().parent
+
+for folder in ["CRNN+CTC", "MNB", "spacyNER"]:
+    p = str(_ROOT / folder)
+    if p not in sys.path:
+        sys.path.insert(0, p)
+
+if str(_ROOT) not in sys.path:
+    sys.path.insert(0, str(_ROOT))
+
+# ── Imports ──────────────────────────────────────────────────
+from spacyNER.extractor import CivilRegistryNER
+from spacyNER.autofill  import AutoFillEngine
+from MNB.classifier     import MNBClassifier
+
+# ── Default paths ────────────────────────────────────────────
+NER_MODEL_PATH = str(_ROOT / "spacyNER" / "models" / "civil_registry_model" / "model-best")
+MNB_MODEL_DIR  = str(_ROOT / "MNB" / "models")
+
+
+# ════════════════════════════════════════════════════════════
+# HELPER — nationality/citizenship alias
+# Tries multiple key names, returns first non-empty value.
+# nationality = citizenship — same field, different label per form.
+# ════════════════════════════════════════════════════════════
+
+def _get(f: dict, *keys, default='') -> str:
+    for k in keys:
+        v = f.get(k, '')
+        if v and str(v).strip():
+            return str(v).strip()
+    return default
+
+
+# ════════════════════════════════════════════════════════════
+# CRNN FIELD DICT → TEXT CONVERTERS
+# Turns Irish's field dict into readable text that NER can read.
+# Handles both old field names and new dynamic_field_extractor names.
+# ════════════════════════════════════════════════════════════
+
+def crnn_birth_to_text(f: dict) -> str:
+    """Form 102 → Form 1A text.
+    Fields needed:
+      Registry Number, Date of Registration,
+      Name of Child, Sex, Date of Birth, Place of Birth,
+      Name of Mother, Nationality/Citizenship of Mother,
+      Name of Father, Nationality/Citizenship of Father,
+      Date of Marriage of Parents, Place of Marriage of Parents
+    """
+    return (
+        f"Registry No.: {_get(f, 'registry_number', 'registry_no')}\n"
+        f"Date of Registration: {_get(f, 'date_of_registration')}\n"
+        f"1. NAME (First): {_get(f, 'child_first_name')}  "
+        f"(Middle): {_get(f, 'child_middle_name')}  "
+        f"(Last): {_get(f, 'child_last_name')}\n"
+        f"2. SEX: {_get(f, 'sex')}\n"
+        f"3. DATE OF BIRTH: {_get(f, 'dob_month')} {_get(f, 'dob_day')}, {_get(f, 'dob_year')}\n"
+        f"4. PLACE OF BIRTH: {_get(f, 'place_birth_hospital')} "
+        f"{_get(f, 'place_birth_city')} {_get(f, 'place_birth_province')}\n"
+        f"MOTHER:\n"
+        f"7. MAIDEN NAME (First): {_get(f, 'mother_first_name')}  "
+        f"(Middle): {_get(f, 'mother_middle_name')}  "
+        f"(Last): {_get(f, 'mother_last_name')}\n"
+        f"8. CITIZENSHIP/NATIONALITY: "
+        f"{_get(f, 'nationality_of_mother', 'mother_citizenship', 'mother_nationality')}\n"
+        f"FATHER:\n"
+        f"14. NAME (First): {_get(f, 'father_first_name')}  "
+        f"(Middle): {_get(f, 'father_middle_name')}  "
+        f"(Last): {_get(f, 'father_last_name')}\n"
+        f"15. CITIZENSHIP/NATIONALITY: "
+        f"{_get(f, 'nationality_of_father', 'father_citizenship', 'father_nationality')}\n"
+        f"MARRIAGE OF PARENTS:\n"
+        f"20a. DATE: {_get(f, 'parents_marriage_month')} "
+        f"{_get(f, 'parents_marriage_day')}, {_get(f, 'parents_marriage_year')}\n"
+        f"20b. PLACE: {_get(f, 'parents_marriage_city')} "
+        f"{_get(f, 'parents_marriage_province')}\n"
+    )
+
+
+def crnn_death_to_text(f: dict) -> str:
+    """Form 103 → Form 2A text.
+    Fields needed:
+      Registry Number, Date of Registration,
+      Name of Deceased, Sex, Age, Civil Status,
+      Nationality/Citizenship, Date of Death, Place of Death,
+      Cause of Death
+    """
+    return (
+        f"Registry No.: {_get(f, 'registry_number', 'registry_no')}\n"
+        f"Date of Registration: {_get(f, 'date_of_registration')}\n"
+        f"1. NAME (First): {_get(f, 'deceased_first_name')}  "
+        f"(Middle): {_get(f, 'deceased_middle_name')}  "
+        f"(Last): {_get(f, 'deceased_last_name')}\n"
+        f"2. SEX: {_get(f, 'sex')}\n"
+        f"4. AGE: {_get(f, 'age', 'age_years')}\n"
+        f"9. CIVIL STATUS: {_get(f, 'civil_status')}\n"
+        f"7. CITIZENSHIP/NATIONALITY: {_get(f, 'nationality', 'citizenship')}\n"
+        f"6. DATE OF DEATH: {_get(f, 'dod_month')} {_get(f, 'dod_day')}, {_get(f, 'dod_year')}\n"
+        f"5. PLACE OF DEATH: {_get(f, 'place_death_hospital')} "
+        f"{_get(f, 'place_death_city')} {_get(f, 'place_death_province')}\n"
+        f"17. CAUSE OF DEATH: {_get(f, 'cause_of_death', 'cause_immediate')}\n"
+        f"Antecedent cause: {_get(f, 'cause_antecedent')}\n"
+        f"Underlying cause: {_get(f, 'cause_underlying')}\n"
+    )
+
+
+def crnn_marriage_to_text(f: dict) -> str:
+    """Form 97 → Form 3A text.
+    Fields needed (both husband and wife):
+      Name, Age, Nationality/Citizenship,
+      Name of Mother, Nationality/Citizenship of Mother,
+      Name of Father, Nationality/Citizenship of Father,
+      Registry Number, Date of Registration,
+      Date of Marriage, Place of Marriage
+    """
+    return (
+        f"Registry No.: {_get(f, 'registry_number', 'registry_no')}\n"
+        f"Date of Registration: {_get(f, 'date_of_registration')}\n"
+        f"HUSBAND:\n"
+        f"1. NAME (First): {_get(f, 'husband_first_name')}  "
+        f"(Middle): {_get(f, 'husband_middle_name')}  "
+        f"(Last): {_get(f, 'husband_last_name')}\n"
+        f"2b. AGE: {_get(f, 'husband_age')}\n"
+        f"4b. CITIZENSHIP/NATIONALITY: "
+        f"{_get(f, 'husband_nationality', 'husband_citizenship')}\n"
+        f"8. NAME OF FATHER (First): {_get(f, 'husband_father_first')}  "
+        f"(Middle): {_get(f, 'husband_father_middle')}  "
+        f"(Last): {_get(f, 'husband_father_last')}\n"
+        f"8b. FATHER CITIZENSHIP/NATIONALITY: "
+        f"{_get(f, 'husband_father_nationality', 'husband_father_citizenship')}\n"
+        f"10. NAME OF MOTHER (First): {_get(f, 'husband_mother_first')}  "
+        f"(Middle): {_get(f, 'husband_mother_middle')}  "
+        f"(Last): {_get(f, 'husband_mother_last')}\n"
+        f"10b. MOTHER CITIZENSHIP/NATIONALITY: "
+        f"{_get(f, 'husband_mother_nationality', 'husband_mother_citizenship')}\n"
+        f"WIFE:\n"
+        f"1. NAME (First): {_get(f, 'wife_first_name')}  "
+        f"(Middle): {_get(f, 'wife_middle_name')}  "
+        f"(Last): {_get(f, 'wife_last_name')}\n"
+        f"2b. AGE: {_get(f, 'wife_age')}\n"
+        f"4b. CITIZENSHIP/NATIONALITY: "
+        f"{_get(f, 'wife_nationality', 'wife_citizenship')}\n"
+        f"8. NAME OF FATHER (First): {_get(f, 'wife_father_first')}  "
+        f"(Middle): {_get(f, 'wife_father_middle')}  "
+        f"(Last): {_get(f, 'wife_father_last')}\n"
+        f"8b. FATHER CITIZENSHIP/NATIONALITY: "
+        f"{_get(f, 'wife_father_nationality', 'wife_father_citizenship')}\n"
+        f"10. NAME OF MOTHER (First): {_get(f, 'wife_mother_first')}  "
+        f"(Middle): {_get(f, 'wife_mother_middle')}  "
+        f"(Last): {_get(f, 'wife_mother_last')}\n"
+        f"10b. MOTHER CITIZENSHIP/NATIONALITY: "
+        f"{_get(f, 'wife_mother_nationality', 'wife_mother_citizenship')}\n"
+        f"15. PLACE OF MARRIAGE: "
+        f"{_get(f, 'place_marriage_city')} {_get(f, 'place_marriage_province')}\n"
+        f"16. DATE OF MARRIAGE: {_get(f, 'date_marriage_month')} "
+        f"{_get(f, 'date_marriage_day')}, {_get(f, 'date_marriage_year')}\n"
+    )
+
+
+def crnn_birth_to_form90_text(f: dict, role: str = 'groom') -> str:
+    """Birth cert of groom or bride → Form 90 text.
+    Fields needed:
+      Name, Date of Birth, Place of Birth, Sex,
+      Citizenship/Nationality,
+      Name of Father, Citizenship of Father,
+      Name of Mother, Citizenship of Mother
+    role: 'groom' or 'bride'
+    """
+    return (
+        f"{role.upper()}:\n"
+        f"1. NAME (First): {_get(f, 'first_name', 'child_first_name')}  "
+        f"(Middle): {_get(f, 'middle_name', 'child_middle_name')}  "
+        f"(Last): {_get(f, 'last_name', 'child_last_name')}\n"
+        f"2. DATE OF BIRTH: {_get(f, 'dob_month')} {_get(f, 'dob_day')}, {_get(f, 'dob_year')}\n"
+        f"3. PLACE OF BIRTH: {_get(f, 'place_birth_hospital')} "
+        f"{_get(f, 'place_birth_city')} {_get(f, 'place_birth_province')}\n"
+        f"4. SEX: {_get(f, 'sex')}\n"
+        f"5. CITIZENSHIP/NATIONALITY: "
+        f"{_get(f, 'citizenship', 'nationality', 'nationality_of_mother', 'mother_citizenship')}\n"
+        f"NAME OF FATHER (First): {_get(f, 'father_first_name')}  "
+        f"(Middle): {_get(f, 'father_middle_name')}  "
+        f"(Last): {_get(f, 'father_last_name')}\n"
+        f"FATHER CITIZENSHIP/NATIONALITY: "
+        f"{_get(f, 'father_citizenship', 'father_nationality')}\n"
+        f"NAME OF MOTHER (First): {_get(f, 'mother_first_name')}  "
+        f"(Middle): {_get(f, 'mother_middle_name')}  "
+        f"(Last): {_get(f, 'mother_last_name')}\n"
+        f"MOTHER CITIZENSHIP/NATIONALITY: "
+        f"{_get(f, 'mother_citizenship', 'mother_nationality')}\n"
+    )
+
+
+# ── Auto-detect form type from CRNN field keys ───────────────
+_BIRTH_KEYS    = {'child_first_name', 'mother_first_name', 'dob_day',
+                  'registry_number', 'nationality_of_mother'}
+_DEATH_KEYS    = {'deceased_first_name', 'cause_of_death', 'dod_day',
+                  'cause_immediate', 'nationality'}
+_MARRIAGE_KEYS = {'husband_first_name', 'wife_first_name', 'date_marriage_day',
+                  'husband_nationality', 'wife_nationality'}
+
+_CONVERTERS = {
+    'birth':    crnn_birth_to_text,
+    'death':    crnn_death_to_text,
+    'marriage': crnn_marriage_to_text,
+}
+
+def _detect_form_type(fields: dict) -> str:
+    keys = set(fields.keys())
+    if keys & _BIRTH_KEYS:    return 'birth'
+    if keys & _DEATH_KEYS:    return 'death'
+    if keys & _MARRIAGE_KEYS: return 'marriage'
+    return 'birth'
+
+
+# ════════════════════════════════════════════════════════════
+# BRIDGE CLASS
+# ════════════════════════════════════════════════════════════
+
+class CivilRegistryBridge:
+    """
+    The single connection point between the three algorithms.
+
+    Usage:
+        from bridge import CivilRegistryBridge
+
+        bridge = CivilRegistryBridge()
+
+        # Path A — birth / death / marriage cert
+        form = bridge.process(crnn_fields, form_hint="birth")
+        print(form.to_dict())
+
+        # Path B — Form 90 (two birth certs)
+        form90 = bridge.process_marriage_license(
+            groom_crnn_fields,
+            bride_crnn_fields
+        )
+    """
+
+    def __init__(self,
+                 ner_model_path: str = NER_MODEL_PATH,
+                 mnb_model_dir:  str = MNB_MODEL_DIR):
+
+        # Princess's MNB classifier
+        self.mnb = MNBClassifier(model_dir=mnb_model_dir)
+
+        # Shane's NER extractor
+        self.extractor = CivilRegistryNER(model_path=ner_model_path)
+        self.filler    = AutoFillEngine(self.extractor)
+
+    # ── Path A — single cert (birth / death / marriage) ──────
+    def process(self, crnn_fields: dict, form_hint: str = None):
+        """
+        Parameters
+        ----------
+        crnn_fields : dict
+            Output from Irish's run_crnn_ocr() / dynamic_field_extractor
+
+        form_hint : str, optional
+            'birth' | 'death' | 'marriage'
+            Auto-detected from field keys if not given.
+
+        Returns
+        -------
+        Form1A | Form2A | Form3A  with all fields populated
+        """
+        form_type = form_hint or _detect_form_type(crnn_fields)
+        ocr_text  = _CONVERTERS.get(form_type, crnn_birth_to_text)(crnn_fields)
+        mnb_label = self.mnb.classify_form_type(ocr_text)
+        print(f"  [Bridge] hint={form_type!r}  MNB={mnb_label}  NER→running...")
+
+        # Use MNB classification result to pick the correct form filler
+        if mnb_label == 'form2a':
+            return self.filler.fill_form_2a(ocr_text)
+        elif mnb_label == 'form3a':
+            return self.filler.fill_form_3a(ocr_text)
+        elif mnb_label == 'form90':
+            return self.filler.fill_form_90(ocr_text, ocr_text)
+        else:
+            return self.filler.fill_form_1a(ocr_text)
+
+    # ── Path B — Form 90 (two birth certs) ───────────────────
+    def process_marriage_license(self,
+                                  groom_crnn_fields: dict,
+                                  bride_crnn_fields:  dict):
+        """
+        Parameters
+        ----------
+        groom_crnn_fields : dict   CRNN output for groom's birth cert
+        bride_crnn_fields : dict   CRNN output for bride's birth cert
+
+        Returns
+        -------
+        Form90  with groom.* and bride.* fields populated
+        """
+        groom_text = crnn_birth_to_form90_text(groom_crnn_fields, role='groom')
+        bride_text = crnn_birth_to_form90_text(bride_crnn_fields, role='bride')
+
+        groom_sex = self.mnb.classify_sex(groom_text)
+        bride_sex  = self.mnb.classify_sex(bride_text)
+        print(f"  [Bridge] Form90  groom_sex={groom_sex}  bride_sex={bride_sex}")
+
+        return self.filler.fill_form_90(groom_text, bride_text)
+
+
+# ── Quick test — run: python bridge.py ───────────────────────
+if __name__ == "__main__":
+
+    SAMPLE_BIRTH = {
+        "registry_number":           "2024-001",
+        "date_of_registration":      "June 12, 1998",
+        "child_first_name":          "TASLIAH",
+        "child_middle_name":         "ABOBACAR",
+        "child_last_name":           "GOMONSANG",
+        "sex":                       "FEMALE",
+        "dob_day":                   "12",
+        "dob_month":                 "JUNE",
+        "dob_year":                  "1998",
+        "place_birth_hospital":      "CAMP JAS BLISS",
+        "place_birth_city":          "MALABANG",
+        "place_birth_province":      "LANAO DEL SUR",
+        "mother_first_name":         "H. ASLIAH",
+        "mother_middle_name":        "SANTICAN",
+        "mother_last_name":          "ABOBACAR",
+        "nationality_of_mother":     "FILIPINO",   # nationality = citizenship
+        "father_first_name":         "H. NAEEF",
+        "father_middle_name":        "MUDAG",
+        "father_last_name":          "GOMONSANG",
+        "nationality_of_father":     "FILIPINO",   # nationality = citizenship
+        "parents_marriage_month":    "JANUARY",
+        "parents_marriage_day":      "5",
+        "parents_marriage_year":     "1990",
+        "parents_marriage_city":     "CAMP JAS BLISS MALABANG",
+        "parents_marriage_province": "LANAO DEL SUR",
+    }
+
+    print("=" * 55)
+    print("  BRIDGE TEST")
+    print("=" * 55)
+
+    bridge = CivilRegistryBridge()
+    form   = bridge.process(SAMPLE_BIRTH, form_hint="birth")
+
+    print(f"\n  name_of_child  → {form.name_of_child!r}")
+    print(f"  name_of_mother → {form.name_of_mother!r}")
+    print(f"  name_of_father → {form.name_of_father!r}")
+    print(f"  date_of_birth  → {form.date_of_birth!r}")
+    print("\n  Full result:")
+    for k, v in form.to_dict().items():
+        if v:
+            print(f"    {k:<35} {v}")

calibrate_fields.py

@@ -0,0 +1,196 @@
+"""
+calibrate_fields.py
+===================
+Click-to-measure tool for recalibrating field ratios in field_extractor.py.
+
+Usage:
+    python calibrate_fields.py --image your_scan.png --form birth
+
+Controls:
+    • Click and drag  → draw a field box
+    • After releasing → enter the field name in the terminal
+    • Press S         → save all measured ratios to calibrated_fields.py
+    • Press Z         → undo last box
+    • Press Q / ESC   → quit without saving
+
+Output:
+    calibrated_fields.py  — copy-paste the dict into field_extractor.py
+"""
+
+import argparse
+import json
+import cv2
+import numpy as np
+from pathlib import Path
+
+# ── state ─────────────────────────────────────────────────────────────────────
+drawing   = False
+ix, iy    = -1, -1
+ex, ey    = -1, -1
+boxes     = []        # list of (name, rx1, ry1, rx2, ry2)
+form_name = "birth"
+
+COLOURS = [
+    (0,200,0),(0,150,255),(200,0,200),(0,200,200),(200,200,0),(220,20,60),
+    (255,140,0),(150,50,200),(0,160,80),(30,144,255),(255,20,147),(100,200,100),
+]
+
+def draw_boxes(img, bounds):
+    left, top, right, bottom = bounds
+    fw = right - left
+    fh = bottom - top
+
+    vis = img.copy()
+    # form boundary
+    cv2.rectangle(vis, (left, top), (right, bottom), (0, 140, 255), 2)
+
+    for idx, (name, rx1, ry1, rx2, ry2) in enumerate(boxes):
+        x1 = int(left + rx1 * fw)
+        y1 = int(top  + ry1 * fh)
+        x2 = int(left + rx2 * fw)
+        y2 = int(top  + ry2 * fh)
+        c  = COLOURS[idx % len(COLOURS)]
+        cv2.rectangle(vis, (x1, y1), (x2, y2), c, 2)
+        cv2.putText(vis, name[:25], (x1 + 2, max(0, y1 - 3)),
+                    cv2.FONT_HERSHEY_SIMPLEX, 0.35, c, 1)
+
+    # live cursor box
+    if drawing and ix >= 0 and ex >= 0:
+        cv2.rectangle(vis, (ix, iy), (ex, ey), (255, 255, 255), 1)
+
+    # instructions
+    cv2.putText(vis, "Drag=draw box | S=save | Z=undo | Q=quit",
+                (10, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,0,255), 1)
+    cv2.putText(vis, f"Boxes: {len(boxes)}",
+                (10, 40), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,255,0), 1)
+    return vis
+
+
+def detect_bounds(image_bgr):
+    """Simple form boundary detection (reuses logic from FormBoundsDetector)."""
+    h, w  = image_bgr.shape[:2]
+    gray  = cv2.cvtColor(image_bgr, cv2.COLOR_BGR2GRAY)
+    try:
+        thresh  = cv2.adaptiveThreshold(
+            gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
+            cv2.THRESH_BINARY_INV, 11, 2)
+        hk      = cv2.getStructuringElement(cv2.MORPH_RECT, (max(w // 5, 10), 1))
+        h_lines = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, hk)
+        h_rows  = np.where(np.sum(h_lines, axis=1) > w * 0.15)[0]
+        vk      = cv2.getStructuringElement(cv2.MORPH_RECT, (1, max(h // 5, 10)))
+        v_lines = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, vk)
+        v_cols  = np.where(np.sum(v_lines, axis=0) > h * 0.08)[0]
+        if len(h_rows) == 0 or len(v_cols) == 0:
+            return (0, 0, w, h)
+        top_b, bottom_b = int(h_rows.min()), int(h_rows.max())
+        left_b, right_b = int(v_cols.min()), int(v_cols.max())
+        if (right_b - left_b) < w * 0.4 or (bottom_b - top_b) < h * 0.4:
+            return (0, 0, w, h)
+        return (left_b, top_b, right_b, bottom_b)
+    except Exception:
+        return (0, 0, w, h)
+
+
+def save_calibration(output_path, form):
+    dict_name = {
+        "birth":            "BIRTH_FIELDS",
+        "death":            "DEATH_FIELDS",
+        "marriage":         "MARRIAGE_FIELDS",
+        "marriage_license": "MARRIAGE_LICENSE_FIELDS",
+    }.get(form, "CALIBRATED_FIELDS")
+
+    lines = [f"# Auto-calibrated — copy-paste into field_extractor.py\n",
+             f"{dict_name} = {{\n"]
+    for name, rx1, ry1, rx2, ry2 in boxes:
+        lines.append(f'    "{name}":{" " * max(1, 34 - len(name))}'
+                     f'({rx1:.4f}, {ry1:.4f}, {rx2:.4f}, {ry2:.4f}),\n')
+    lines.append("}\n")
+
+    with open(output_path, "w") as f:
+        f.writelines(lines)
+    print(f"\n  Saved {len(boxes)} fields → {output_path}")
+
+
+def main():
+    global drawing, ix, iy, ex, ey, form_name
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--image", required=True)
+    parser.add_argument("--form",  default="birth",
+                        choices=["birth","death","marriage","marriage_license"])
+    parser.add_argument("--output", default="calibrated_fields.py")
+    parser.add_argument("--scale",  type=float, default=1.0,
+                        help="Scale factor to fit image on screen (e.g. 0.5)")
+    args = parser.parse_args()
+    form_name = args.form
+
+    img_orig = cv2.imread(args.image)
+    if img_orig is None:
+        print(f"ERROR: Cannot load {args.image}")
+        return
+
+    scale = args.scale
+    if scale != 1.0:
+        img_orig = cv2.resize(img_orig, None, fx=scale, fy=scale)
+
+    bounds = detect_bounds(img_orig)
+    left, top, right, bottom = bounds
+    fw = right - left
+    fh = bottom - top
+    print(f"  Form boundary detected: {bounds}  ({fw}×{fh} px)")
+    print(f"  Scale: {scale}")
+    print("\n  Instructions:")
+    print("    Drag  → draw a field box")
+    print("    After releasing → type field name in terminal, press Enter")
+    print("    S     → save all boxes")
+    print("    Z     → undo last box")
+    print("    Q/ESC → quit\n")
+
+    win = "Calibrate Fields"
+    cv2.namedWindow(win, cv2.WINDOW_NORMAL)
+
+    def mouse(event, x, y, flags, param):
+        global drawing, ix, iy, ex, ey
+        if event == cv2.EVENT_LBUTTONDOWN:
+            drawing = True
+            ix, iy  = x, y
+            ex, ey  = x, y
+        elif event == cv2.EVENT_MOUSEMOVE and drawing:
+            ex, ey  = x, y
+        elif event == cv2.EVENT_LBUTTONUP:
+            drawing = False
+            ex, ey  = x, y
+            x1r = (min(ix, ex) - left) / fw
+            y1r = (min(iy, ey) - top)  / fh
+            x2r = (max(ix, ex) - left) / fw
+            y2r = (max(iy, ey) - top)  / fh
+            x1r, y1r = max(0.0, x1r), max(0.0, y1r)
+            x2r, y2r = min(1.0, x2r), min(1.0, y2r)
+            if (x2r - x1r) > 0.005 and (y2r - y1r) > 0.003:
+                name = input(f"  Field name for ({x1r:.3f},{y1r:.3f},{x2r:.3f},{y2r:.3f}): ").strip()
+                if name:
+                    boxes.append((name, x1r, y1r, x2r, y2r))
+                    print(f"  ✓  '{name}' added  (total: {len(boxes)})")
+
+    cv2.setMouseCallback(win, mouse)
+
+    while True:
+        vis = draw_boxes(img_orig, bounds)
+        cv2.imshow(win, vis)
+        key = cv2.waitKey(20) & 0xFF
+
+        if key in (ord('q'), 27):
+            print("  Quit — no file saved.")
+            break
+        elif key == ord('s'):
+            save_calibration(args.output, form_name)
+            break
+        elif key == ord('z') and boxes:
+            removed = boxes.pop()
+            print(f"  Undone: '{removed[0]}'")
+
+    cv2.destroyAllWindows()
+
+
+if __name__ == "__main__":
+    main()

calibrated_fields.py

@@ -0,0 +1,7 @@
+# Auto-calibrated � copy-paste into field_extractor.py
+BIRTH_FIELDS = {
+    "Province":                          (0.0941, 0.0701, 0.6361, 0.0848),
+    "City/Municipality":                 (0.1621, 0.0880, 0.6429, 0.1086),
+    "first_name":                        (0.0465, 0.1183, 0.3265, 0.1375),
+    "middle_name":                       (0.3469, 0.1189, 0.6916, 0.1375),
+}

check_cer.py

@@ -0,0 +1,331 @@
+"""
+check_cer.py
+============
+Measures TRUE CER by actually running the model on images.
+
+Usage:
+    python check_cer.py                        # live CER on val set
+    python check_cer.py --saved                # old behavior (fast, unreliable)
+    python check_cer.py --images test_images/  # run on any image folder
+"""
+
+import os
+import sys
+import json
+import random
+import cv2
+import numpy as np
+import editdistance
+from pathlib import Path
+
+try:
+    import torch
+except ImportError:
+    print("ERROR: torch not installed. Run: pip install torch")
+    exit(1)
+
+USE_SAVED = '--saved' in sys.argv
+IMAGE_DIR = None
+for i, arg in enumerate(sys.argv[1:], 1):
+    if arg == '--images' and i < len(sys.argv) - 1:
+        IMAGE_DIR = sys.argv[i + 1]
+    elif arg.startswith('--images='):
+        IMAGE_DIR = arg.split('=', 1)[1]
+
+CHECKPOINTS = [
+    'checkpoint_epoch_50.pth',
+    'checkpoint_epoch_60.pth',
+    'checkpoint_epoch_70.pth',
+    'checkpoint_epoch_80.pth',
+    'checkpoint_epoch_90.pth',
+    'checkpoint_epoch_100.pth',
+]
+CHECKPOINT_DIR = 'checkpoints'
+VAL_DATA_DIR   = 'data/val'
+VAL_ANN_FILE   = 'data/val_annotations.json'
+
+
+class AdaptiveImageNormalizer:
+    def __init__(self, target_height=64, target_width=512):
+        self.H = target_height
+        self.W = target_width
+
+    def _crop_to_text(self, gray):
+        inv = cv2.bitwise_not(gray)
+        _, thresh = cv2.threshold(inv, 20, 255, cv2.THRESH_BINARY)
+        coords = np.column_stack(np.where(thresh > 0))
+        if len(coords) == 0:
+            return gray
+        y_min, x_min = coords.min(axis=0)
+        y_max, x_max = coords.max(axis=0)
+        pad   = max(4, int((y_max - y_min) * 0.15))
+        y_min = max(0, y_min - pad)
+        x_min = max(0, x_min - pad)
+        y_max = min(gray.shape[0] - 1, y_max + pad)
+        x_max = min(gray.shape[1] - 1, x_max + pad)
+        return gray[y_min:y_max + 1, x_min:x_max + 1]
+
+    def _smart_resize_gray(self, gray):
+        h, w = gray.shape
+        if h == 0 or w == 0:
+            return np.ones((self.H, self.W), dtype=np.uint8) * 255
+        scale = self.H / h
+        new_w = int(w * scale)
+        new_h = self.H
+        if new_w > self.W:
+            scale = self.W / w
+            new_h = int(h * scale)
+            new_w = self.W
+        resized = cv2.resize(gray, (new_w, new_h), interpolation=cv2.INTER_LANCZOS4)
+        canvas  = np.ones((self.H, self.W), dtype=np.uint8) * 255
+        y_off   = (self.H - new_h) // 2
+        x_off   = (self.W - new_w) // 2
+        canvas[y_off:y_off + new_h, x_off:x_off + new_w] = resized
+        return canvas
+
+    def _binarize(self, img):
+        _, otsu = cv2.threshold(img, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
+        white_ratio = np.mean(otsu == 255)
+        if white_ratio < 0.30 or white_ratio > 0.97:
+            return cv2.adaptiveThreshold(
+                img, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
+                cv2.THRESH_BINARY, 11, 2)
+        return otsu
+
+    def normalize(self, img):
+        if len(img.shape) == 3:
+            gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+        else:
+            gray = img.copy()
+        gray = cv2.fastNlMeansDenoising(gray, None, 10, 7, 21)
+        gray = self._crop_to_text(gray)
+        gray = self._smart_resize_gray(gray)
+        return self._binarize(gray)
+
+    def to_tensor(self, img):
+        return torch.FloatTensor(
+            img.astype(np.float32) / 255.0
+        ).unsqueeze(0).unsqueeze(0)
+
+
+def greedy_decode(outputs, idx_to_char):
+    pred_indices = torch.argmax(outputs, dim=2).permute(1, 0)
+    results = []
+    for seq in pred_indices:
+        chars, prev = [], -1
+        for idx in seq:
+            idx = idx.item()
+            if idx != 0 and idx != prev and idx in idx_to_char:
+                chars.append(idx_to_char[idx])
+            prev = idx
+        results.append(''.join(chars))
+    return results
+
+
+def measure_live_cer(model, idx_to_char, img_h, img_w,
+                     ann_file, data_dir, device, max_samples=200):
+    if not os.path.exists(ann_file):
+        return None, 0, f"Annotation file not found: {ann_file}"
+
+    with open(ann_file, 'r', encoding='utf-8') as f:
+        annotations = json.load(f)
+
+    if len(annotations) > max_samples:
+        random.seed(42)
+        annotations = random.sample(annotations, max_samples)
+
+    normalizer = AdaptiveImageNormalizer(img_h, img_w)
+    model.eval()
+
+    total_char_dist = 0
+    total_chars     = 0
+    total_word_dist = 0
+    total_words     = 0
+    n_exact         = 0
+    n_evaluated     = 0
+    worst_errors    = []
+
+    with torch.no_grad():
+        for ann in annotations:
+            img_path = os.path.join(data_dir, ann['image_path'])
+            gt       = ann['text']
+            if not os.path.exists(img_path):
+                continue
+            try:
+                raw = cv2.imread(img_path)
+                if raw is None:
+                    continue
+                norm   = normalizer.normalize(raw)
+                tensor = normalizer.to_tensor(norm).to(device)
+                out    = model(tensor)
+                pred   = greedy_decode(out.cpu(), idx_to_char)[0]
+
+                cd = editdistance.eval(pred, gt)
+                wd = editdistance.eval(pred.split(), gt.split())
+
+                total_char_dist += cd
+                total_chars     += len(gt)
+                total_word_dist += wd
+                total_words     += len(gt.split())
+                if pred == gt:
+                    n_exact += 1
+                if cd > 0:
+                    worst_errors.append((gt, pred, cd))
+                n_evaluated += 1
+            except Exception:
+                continue
+
+    if n_evaluated == 0:
+        return None, 0, "No images could be evaluated"
+
+    cer = (total_char_dist / total_chars * 100) if total_chars > 0 else 0
+    wer = (total_word_dist / total_words * 100) if total_words > 0 else 0
+    acc = (n_exact / n_evaluated * 100)
+
+    return {
+        'cer': cer, 'wer': wer, 'exact_match': acc,
+        'n_evaluated': n_evaluated,
+        'errors': sorted(worst_errors, key=lambda x: x[2], reverse=True)[:5]
+    }, n_evaluated, None
+
+
+def run_on_folder(model, idx_to_char, img_h, img_w, folder, device):
+    normalizer = AdaptiveImageNormalizer(img_h, img_w)
+    model.eval()
+    exts  = {'.jpg', '.jpeg', '.png', '.bmp'}
+    paths = sorted(p for p in Path(folder).rglob('*') if p.suffix.lower() in exts)
+    results = []
+    with torch.no_grad():
+        for p in paths:
+            try:
+                raw    = cv2.imread(str(p))
+                norm   = normalizer.normalize(raw)
+                tensor = normalizer.to_tensor(norm).to(device)
+                pred   = greedy_decode(model(tensor).cpu(), idx_to_char)[0]
+                results.append((p.name, pred))
+            except Exception as e:
+                results.append((p.name, f'ERROR: {e}'))
+    return results
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# MAIN
+# ─────────────────────────────────────────────────────────────────────────────
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+if USE_SAVED:
+    print("=" * 65)
+    print("  SAVED CER  (training-time value — may not reflect real accuracy)")
+    print("  Run without --saved for true live CER.")
+    print("=" * 65)
+    print("{:<8} {:<12} {:<12} {}".format("Epoch", "CER(%)", "WER(%)", "File"))
+    print("-" * 65)
+    best_cer, best_cp = float('inf'), None
+    for cp in CHECKPOINTS:
+        path = os.path.join(CHECKPOINT_DIR, cp)
+        if not os.path.exists(path):
+            continue
+        try:
+            c        = torch.load(path, weights_only=False)
+            cer      = c.get('val_cer', c.get('val_loss', 0))
+            epoch    = c['epoch']
+            history  = c.get('history', {})
+            wer_list = history.get('val_wer', [])
+            wer      = wer_list[epoch - 1] if wer_list and epoch <= len(wer_list) else None
+            wer_s    = f"{wer:.4f}%" if wer else 'N/A'
+            marker   = ' <-- BEST' if cer < best_cer else ''
+            print("{:<8} {:<12} {:<12} {}{}".format(
+                epoch, f"{cer:.4f}%", wer_s, cp, marker))
+            if cer < best_cer:
+                best_cer, best_cp = cer, cp
+        except Exception as e:
+            print(f"  Could not load {cp}: {e}")
+    print("=" * 65)
+    print(f"\nBEST: {best_cp}  CER={best_cer:.4f}%")
+
+else:
+    print("=" * 78)
+    print("  LIVE CER  —  model actually runs on images  (true accuracy)")
+    print("=" * 78)
+    print("{:<8} {:<10} {:<10} {:<12} {:<8} {}".format(
+        "Epoch", "CER(%)", "WER(%)", "ExactMatch", "N", "File"))
+    print("-" * 78)
+
+    best_cer, best_cp, best_metrics = float('inf'), None, None
+
+    for cp in CHECKPOINTS:
+        cp_path = os.path.join(CHECKPOINT_DIR, cp)
+        if not os.path.exists(cp_path):
+            print(f"  (skipping {cp} — not found)")
+            continue
+        try:
+            from crnn_model import get_crnn_model
+            c           = torch.load(cp_path, map_location=device, weights_only=False)
+            epoch       = c['epoch']
+            idx_to_char = c['idx_to_char']
+            config      = c.get('config', {})
+            img_h       = config.get('img_height', 64)
+            img_w       = config.get('img_width', 512)
+            saved_cer   = c.get('val_cer', c.get('val_loss', None))
+
+            model = get_crnn_model(
+                model_type=config.get('model_type', 'standard'),
+                img_height=img_h,
+                num_chars=c['model_state_dict']['fc.weight'].shape[0],
+                hidden_size=config.get('hidden_size', 128),       # FIXED: was 256
+                num_lstm_layers=config.get('num_lstm_layers', 1)  # FIXED: was 2
+            ).to(device)
+            model.load_state_dict(c['model_state_dict'])
+
+            if IMAGE_DIR:
+                print(f"\nPredictions from {cp}:")
+                for fname, pred in run_on_folder(
+                        model, idx_to_char, img_h, img_w, IMAGE_DIR, device):
+                    print(f"  {fname:<35} ->  {pred}")
+                continue
+
+            metrics, n, err = measure_live_cer(
+                model, idx_to_char, img_h, img_w,
+                VAL_ANN_FILE, VAL_DATA_DIR, device)
+
+            if metrics is None:
+                print(f"  Epoch {epoch}  SKIP: {err}")
+                continue
+
+            cer    = metrics['cer']
+            marker = ' <-- BEST' if cer < best_cer else ''
+            print("{:<8} {:<10} {:<10} {:<12} {:<8} {}{}".format(
+                epoch,
+                f"{cer:.2f}%",
+                f"{metrics['wer']:.2f}%",
+                f"{metrics['exact_match']:.1f}%",
+                n, cp, marker))
+
+            if saved_cer and abs(cer - saved_cer) > 2.0:
+                print(f"          ^ MISMATCH: saved={saved_cer:.2f}%  live={cer:.2f}%"
+                      f"  diff={abs(cer - saved_cer):.2f}%")
+                print(f"            Cause: model trained on clean synthetic only.")
+                print(f"            Fix:   regenerate data with fix_data.py + retrain.")
+
+            if cer < best_cer:
+                best_cer, best_cp, best_metrics = cer, cp, metrics
+
+        except Exception as e:
+            print(f"  Could not evaluate {cp}: {e}")
+
+    if not IMAGE_DIR:
+        print("=" * 78)
+        print(f"\nBEST CHECKPOINT : {best_cp}")
+        print(f"BEST LIVE CER   : {best_cer:.4f}%")
+
+        if best_metrics and best_metrics['errors']:
+            print(f"\nWorst predictions (GT -> Predicted):")
+            for gt, pred, dist in best_metrics['errors']:
+                print(f"  [{dist:2d}]  '{gt}'")
+                print(f"        '{pred}'")
+
+        print(f"\nTo use best model:")
+        print(f"  import shutil")
+        print(f"  shutil.copy('checkpoints/{best_cp}', 'checkpoints/best_model.pth')")
+        print("=" * 78)

compare_checkpoints.py

@@ -0,0 +1,34 @@
+import torch
+import sys
+sys.path.append('.')
+from crnn_model import get_crnn_model
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+def test_model(path, label):
+    c      = torch.load(path, map_location=device, weights_only=False)
+    config = c.get('config', {})
+    model  = get_crnn_model(
+        model_type      = config.get('model_type', 'standard'),
+        img_height      = config.get('img_height', 64),
+        num_chars       = c['model_state_dict']['fc.weight'].shape[0],
+        hidden_size     = config.get('hidden_size', 128),
+        num_lstm_layers = config.get('num_lstm_layers', 1),
+    ).to(device)
+    model.load_state_dict(c['model_state_dict'], strict=False)
+    epoch   = c.get('epoch', 'N/A')
+    val_loss = c.get('val_loss', None)   # fine-tuned checkpoints (EMNIST, IAM)
+    val_cer  = c.get('val_cer',  None)   # synthetic baseline checkpoint
+    if val_loss is not None:
+        metric_str = f"val_loss={val_loss:.4f}"
+    elif val_cer is not None:
+        metric_str = f"val_cer={val_cer:.4f}%"
+    else:
+        metric_str = "no metric saved"
+    print(f"{label}: epoch={epoch}  {metric_str}")
+
+print("=" * 55)
+test_model('checkpoints/best_model.pth',       'Synthetic  ')
+test_model('checkpoints/best_model_emnist.pth', 'EMNIST     ')
+test_model('checkpoints/best_model_iam.pth',    'IAM        ')
+print("=" * 55)

compare_live_cer.py

@@ -0,0 +1,158 @@
+"""
+compare_live_cer.py
+===================
+Runs live CER on all three checkpoints to find the best one.
+Usage: python compare_live_cer.py
+"""
+
+import os
+import sys
+import json
+import random
+import cv2
+import numpy as np
+import editdistance
+import torch
+import torch.nn.functional as F
+sys.path.append('.')
+from crnn_model import get_crnn_model
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+VAL_ANN  = 'data/val_annotations.json'
+VAL_DIR  = 'data/val'
+MAX_SAMPLES = 200
+
+CHECKPOINTS = {
+    'Synthetic' : 'checkpoints/best_model.pth',
+    'EMNIST'    : 'checkpoints/best_model_emnist.pth',
+    'IAM'       : 'checkpoints/best_model_iam.pth',
+}
+
+
+def normalize(img, H=64, W=512):
+    if len(img.shape) == 3:
+        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+    else:
+        gray = img.copy()
+    gray = cv2.fastNlMeansDenoising(gray, None, 10, 7, 21)
+    inv  = cv2.bitwise_not(gray)
+    _, thresh = cv2.threshold(inv, 20, 255, cv2.THRESH_BINARY)
+    coords = np.column_stack(np.where(thresh > 0))
+    if len(coords) > 0:
+        y_min, x_min = coords.min(axis=0)
+        y_max, x_max = coords.max(axis=0)
+        pad   = max(4, int((y_max - y_min) * 0.15))
+        y_min = max(0, y_min - pad)
+        x_min = max(0, x_min - pad)
+        y_max = min(gray.shape[0]-1, y_max + pad)
+        x_max = min(gray.shape[1]-1, x_max + pad)
+        gray  = gray[y_min:y_max+1, x_min:x_max+1]
+    h, w = gray.shape
+    if h == 0 or w == 0:
+        return np.ones((H, W), dtype=np.uint8) * 255
+    scale = H / h
+    new_w = int(w * scale)
+    if new_w > W:
+        scale = W / w
+        new_w = W
+        new_h = int(h * scale)
+    else:
+        new_h = H
+    resized = cv2.resize(gray, (new_w, new_h), interpolation=cv2.INTER_LANCZOS4)
+    canvas  = np.ones((H, W), dtype=np.uint8) * 255
+    canvas[(H-new_h)//2:(H-new_h)//2+new_h,
+           (W-new_w)//2:(W-new_w)//2+new_w] = resized
+    _, otsu = cv2.threshold(canvas, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
+    return otsu
+
+
+def greedy_decode(outputs, idx_to_char):
+    pred_indices = torch.argmax(outputs, dim=2).permute(1, 0)
+    results = []
+    for seq in pred_indices:
+        chars, prev = [], -1
+        for idx in seq:
+            idx = idx.item()
+            if idx != 0 and idx != prev and idx in idx_to_char:
+                chars.append(idx_to_char[idx])
+            prev = idx
+        results.append(''.join(chars))
+    return results
+
+
+def evaluate(checkpoint_path, label):
+    if not os.path.exists(checkpoint_path):
+        print(f"  {label:<12}: FILE NOT FOUND — skipping")
+        return
+
+    c      = torch.load(checkpoint_path, map_location=device, weights_only=False)
+    config = c.get('config', {})
+
+    # Load idx_to_char from checkpoint if available
+    idx_to_char = c.get('idx_to_char', None)
+    if idx_to_char is None:
+        from dataset import build_char_maps
+        _, idx_to_char, _ = build_char_maps()
+
+    model = get_crnn_model(
+        model_type      = config.get('model_type', 'standard'),
+        img_height      = config.get('img_height', 64),
+        num_chars       = c['model_state_dict']['fc.weight'].shape[0],
+        hidden_size     = config.get('hidden_size', 128),
+        num_lstm_layers = config.get('num_lstm_layers', 1),
+    ).to(device)
+    model.load_state_dict(c['model_state_dict'], strict=False)
+    model.eval()
+
+    with open(VAL_ANN, 'r', encoding='utf-8') as f:
+        anns = json.load(f)
+    random.seed(42)
+    if len(anns) > MAX_SAMPLES:
+        anns = random.sample(anns, MAX_SAMPLES)
+
+    total_cd, total_c = 0, 0
+    exact, n = 0, 0
+    worst = []
+
+    with torch.no_grad():
+        for ann in anns:
+            img_path = os.path.join(VAL_DIR, ann['image_path'])
+            gt       = ann['text']
+            if not os.path.exists(img_path):
+                continue
+            raw = cv2.imread(img_path)
+            if raw is None:
+                continue
+            norm   = normalize(raw)
+            tensor = torch.FloatTensor(
+                norm.astype(np.float32) / 255.0
+            ).unsqueeze(0).unsqueeze(0).to(device)
+            out  = model(tensor)
+            pred = greedy_decode(out.cpu(), idx_to_char)[0]
+            cd   = editdistance.eval(pred, gt)
+            total_cd += cd
+            total_c  += len(gt)
+            if pred == gt:
+                exact += 1
+            if cd > 0:
+                worst.append((gt, pred, cd))
+            n += 1
+
+    cer = (total_cd / total_c * 100) if total_c > 0 else 0
+    acc = (exact / n * 100) if n > 0 else 0
+    print(f"  {label:<12}: CER={cer:.2f}%  ExactMatch={acc:.1f}%  (n={n})")
+
+    if worst:
+        worst = sorted(worst, key=lambda x: x[2], reverse=True)[:2]
+        for gt, pred, d in worst:
+            print(f"             [{d}] '{gt}' -> '{pred}'")
+
+
+print("=" * 60)
+print("  LIVE CER COMPARISON — all checkpoints")
+print("=" * 60)
+for label, path in CHECKPOINTS.items():
+    evaluate(path, label)
+print("=" * 60)
+print("Use the checkpoint with the lowest CER for IAM/physical fine-tuning.")

create_test_images.py

@@ -0,0 +1,50 @@
+import os
+from PIL import Image, ImageDraw, ImageFont
+
+os.makedirs('test_images', exist_ok=True)
+
+def load_font(size=22):   # FIXED: was 20 — must match fix_data.py FONT_SIZE=22
+    """Same font loader as fix_data.py — tries multiple paths."""
+    for fp in [
+        'arial.ttf', 'Arial.ttf',
+        '/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf',
+        '/System/Library/Fonts/Helvetica.ttc',
+        'C:/Windows/Fonts/arial.ttf',
+    ]:
+        try:
+            return ImageFont.truetype(fp, size)
+        except Exception:
+            continue
+    print("WARNING: Could not load Arial/DejaVu font. Using default — predictions may be inaccurate.")
+    return ImageFont.load_default()
+
+def create_image(text, filename):
+    """Render text exactly the same way as fix_data.py training images."""
+    img  = Image.new('RGB', (512, 64), color=(255, 255, 255))
+    draw = ImageDraw.Draw(img)
+    font = load_font(22)
+
+    bbox = draw.textbbox((0, 0), text, font=font)
+    x = max((512 - (bbox[2] - bbox[0])) // 2, 2)
+    y = max((64  - (bbox[3] - bbox[1])) // 2, 2)
+    draw.text((x, y), text, fill=(0, 0, 0), font=font)
+    img.save(filename)
+    print(f'Created: {filename}')
+
+# ── Test samples ──────────────────────────────────────────────
+create_image('Juan Dela Cruz',                  'test_images/demo.jpg')
+create_image('Juan Dela Cruz',                  'test_images/name1.jpg')
+create_image('01/15/1990',                      'test_images/date1.jpg')
+create_image('Tarlac City',                     'test_images/place1.jpg')
+create_image('Maria Santos',                    'test_images/form1a_sample.jpg')
+
+# ── Extra test cases (names, dates, addresses) ────────────────
+create_image('Jose Dela Cruz Jr.',              'test_images/name2.jpg')
+create_image('Ana Marie Reyes',                 'test_images/name3.jpg')
+create_image('03/22/1985',                      'test_images/date2.jpg')
+create_image('07/04/2000',                      'test_images/date3.jpg')
+create_image('Brgy. San Jose, Capas, Tarlac',   'test_images/place2.jpg')
+create_image('78 MacArthur Hwy., Tarlac City',  'test_images/place3.jpg')
+
+print('\nAll test images created!')
+print('Font used matches training data — predictions should be accurate.')

crnn_model.py

@@ -0,0 +1,119 @@
+"""
+CRNN+CTC Model — simplified for small datasets (~5000-10000 samples)
+~700K parameters, converges reliably without CTC blank collapse.
+"""
+import torch
+import torch.nn as nn
+
+
+class CRNN_CivilRegistry(nn.Module):
+
+    def __init__(self, img_height=64, num_chars=96, hidden_size=128, num_lstm_layers=1,
+                 dropout=0.3):
+        super().__init__()
+
+        # CNN — width reductions for 512px input:
+        # MaxPool(2,2): 512→256, MaxPool(2,2): 256→128
+        # MaxPool(2,1): 128 (height only), MaxPool(2,1): 128 (height only)
+        # Conv(k=2,p=0): 127  →  seq_len=127, fits labels up to 64 chars
+        self.cnn = nn.Sequential(
+            nn.Conv2d(1, 32, 3, padding=1), nn.BatchNorm2d(32), nn.ReLU(inplace=True),
+            nn.MaxPool2d(2, 2),
+
+            nn.Conv2d(32, 64, 3, padding=1), nn.BatchNorm2d(64), nn.ReLU(inplace=True),
+            nn.MaxPool2d(2, 2),
+
+            nn.Conv2d(64, 128, 3, padding=1), nn.BatchNorm2d(128), nn.ReLU(inplace=True),
+            nn.MaxPool2d((2, 1)),
+
+            nn.Conv2d(128, 256, 3, padding=1), nn.BatchNorm2d(256), nn.ReLU(inplace=True),
+            nn.MaxPool2d((2, 1)),
+
+            nn.Conv2d(256, 256, kernel_size=2, padding=0),
+            nn.BatchNorm2d(256), nn.ReLU(inplace=True),
+        )
+
+        # FIXED Bug 4: derive cnn_out_h from a real forward pass instead of
+        # a hardcoded formula — safer if architecture or img_height ever changes.
+        with torch.no_grad():
+            _dummy = torch.zeros(1, 1, img_height, 32)
+            _out   = self.cnn(_dummy)
+            cnn_out_h = _out.shape[2]   # actual height after all CNN layers
+        rnn_input = 256 * cnn_out_h
+
+        self.rnn = nn.LSTM(
+            input_size=rnn_input,
+            hidden_size=hidden_size,
+            num_layers=num_lstm_layers,
+            bidirectional=True,
+            batch_first=False,
+        )
+        # Dropout before FC — prevents overfitting on small datasets.
+        # Applied after BiLSTM output, before character projection.
+        # p=0.3 is standard for CRNN OCR models (disabled at inference via model.eval()).
+        self.dropout = nn.Dropout(p=dropout)
+        self.fc = nn.Linear(hidden_size * 2, num_chars)
+
+    def forward(self, x):
+        f = self.cnn(x)
+        B, C, h, w = f.size()
+        f = f.permute(3, 0, 1, 2).reshape(w, B, C * h)
+        f, _ = self.rnn(f)
+        return self.fc(self.dropout(f))
+
+
+class CRNN_Ensemble(nn.Module):
+    def __init__(self, num_models=3, **kwargs):
+        super().__init__()
+        self.models = nn.ModuleList([CRNN_CivilRegistry(**kwargs) for _ in range(num_models)])
+
+    def forward(self, x):
+        # FIXED Rec 3: average softmax probabilities across models (correct ensemble),
+        # then return log of the average so CTCLoss receives log-probabilities —
+        # the same contract as CRNN_CivilRegistry (raw logits + log_softmax in trainer).
+        # Returning raw averaged probabilities caused CTCLoss to receive un-logged values.
+        probs = [torch.nn.functional.softmax(m(x), dim=2) for m in self.models]
+        avg_probs = torch.mean(torch.stack(probs), dim=0)
+        return torch.log(avg_probs.clamp(min=1e-9))  # log-probs, safe clamp avoids log(0)
+
+
+def get_crnn_model(model_type='standard', **kwargs):
+    if model_type == 'ensemble':
+        return CRNN_Ensemble(**kwargs)
+    return CRNN_CivilRegistry(**kwargs)
+
+
+def initialize_weights(model):
+    for m in model.modules():
+        if isinstance(m, nn.Conv2d):
+            nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
+            if m.bias is not None:
+                nn.init.constant_(m.bias, 0)
+        elif isinstance(m, nn.BatchNorm2d):
+            nn.init.constant_(m.weight, 1)
+            nn.init.constant_(m.bias, 0)
+        elif isinstance(m, nn.Linear):
+            nn.init.normal_(m.weight, 0, 0.01)
+            nn.init.constant_(m.bias, 0)
+        elif isinstance(m, nn.LSTM):
+            for name, param in m.named_parameters():
+                if 'weight' in name:
+                    nn.init.orthogonal_(param)
+                elif 'bias' in name:
+                    nn.init.constant_(param, 0)
+                    # Rec 1: set forget gate bias to 1.0 — helps the model
+                    # remember across long sequences at the start of training.
+                    # LSTM gate order: [input | forget | cell | output]
+                    n = param.size(0)
+                    param.data[n // 4 : n // 2].fill_(1.0)
+
+
+if __name__ == "__main__":
+    model = get_crnn_model('standard', img_height=64, num_chars=96, hidden_size=128, num_lstm_layers=1)
+    initialize_weights(model)
+    x = torch.randn(2, 1, 64, 512)
+    out = model(x)
+    params = sum(p.numel() for p in model.parameters())
+    print(f"Output: {out.shape}  seq_len={out.shape[0]}")
+    print(f"Params: {params:,}  (unchanged — dropout adds no parameters)")
+    print(f"Dropout p=0.3 active during training, disabled during model.eval()")

dataset.py

@@ -0,0 +1,401 @@
+"""
+dataset.py
+==========
+PyTorch Dataset and DataLoader utilities for the Civil Registry OCR system.
+"""
+
+import os
+import json
+import random
+from pathlib import Path
+from typing import List, Tuple, Dict, Optional
+
+import cv2
+import numpy as np
+import torch
+from torch.utils.data import Dataset
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+#  CHARACTER SET
+# ─────────────────────────────────────────────────────────────────────────────
+
+PRINTABLE_CHARS = [chr(i) for i in range(32, 127)]  # space (32) to ~ (126)
+
+
+def build_char_maps(extra_chars: Optional[List[str]] = None):
+    chars = PRINTABLE_CHARS.copy()
+    if extra_chars:
+        for c in extra_chars:
+            if c not in chars:
+                chars.append(c)
+    char_to_idx = {c: i + 1 for i, c in enumerate(chars)}
+    idx_to_char = {i + 1: c for i, c in enumerate(chars)}
+    num_chars   = len(chars) + 1  # +1 for blank=0
+    return char_to_idx, idx_to_char, num_chars
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+#  IMAGE NORMALIZER
+# ─────────────────────────────────────────────────────────────────────────────
+
+class ImageNormalizer:
+
+    def __init__(self, target_height: int = 64, target_width: int = 512):
+        self.H = target_height
+        self.W = target_width
+
+    def _to_gray(self, img):
+        if len(img.shape) == 3:
+            return cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+        return img.copy()
+
+    def _crop_to_text(self, gray):
+        inv = cv2.bitwise_not(gray)
+        _, thresh = cv2.threshold(inv, 20, 255, cv2.THRESH_BINARY)
+        coords = np.column_stack(np.where(thresh > 0))
+        if len(coords) == 0:
+            return gray
+        y_min, x_min = coords.min(axis=0)
+        y_max, x_max = coords.max(axis=0)
+        pad   = max(4, int((y_max - y_min) * 0.15))
+        y_min = max(0, y_min - pad)
+        x_min = max(0, x_min - pad)
+        y_max = min(gray.shape[0] - 1, y_max + pad)
+        x_max = min(gray.shape[1] - 1, x_max + pad)
+        return gray[y_min:y_max + 1, x_min:x_max + 1]
+
+    def _aspect_resize(self, gray):
+        h, w = gray.shape
+        if h == 0 or w == 0:
+            return np.ones((self.H, self.W), dtype=np.uint8) * 255
+        scale = self.H / h
+        new_w = int(w * scale)
+        new_h = self.H
+        if new_w > self.W:
+            scale = self.W / w
+            new_h = int(h * scale)
+            new_w = self.W
+        resized = cv2.resize(gray, (new_w, new_h), interpolation=cv2.INTER_LANCZOS4)
+        canvas  = np.ones((self.H, self.W), dtype=np.uint8) * 255
+        y_off   = (self.H - new_h) // 2
+        x_off   = (self.W - new_w) // 2
+        canvas[y_off:y_off + new_h, x_off:x_off + new_w] = resized
+        return canvas
+
+    def _binarize(self, img):
+        _, otsu = cv2.threshold(img, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
+        white_ratio = np.mean(otsu == 255)
+        if white_ratio < 0.30 or white_ratio > 0.97:
+            return cv2.adaptiveThreshold(
+                img, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
+                cv2.THRESH_BINARY, 11, 2)
+        return otsu
+
+    def normalize(self, img: np.ndarray, augmenter=None) -> np.ndarray:
+        gray = self._to_gray(img)
+        # NOTE: fastNlMeansDenoising intentionally removed from training pipeline.
+        # It is slow (~200ms/image) and pointless on clean synthetic images.
+        # Denoising is only applied in check_cer.py / inference.py (AdaptiveNormalizer)
+        # which runs on real scanned documents where denoising actually helps.
+        gray = self._crop_to_text(gray)
+        gray = self._aspect_resize(gray)
+        # FIXED Bug 3: augment on grayscale BEFORE binarize.
+        # Brightness/contrast augmentation has zero effect on binary (0/255) pixels.
+        if augmenter is not None:
+            gray = augmenter(gray)
+        return self._binarize(gray)
+
+    def to_tensor(self, img: np.ndarray) -> torch.Tensor:
+        return torch.FloatTensor(
+            img.astype(np.float32) / 255.0
+        ).unsqueeze(0)  # [1, H, W]
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+#  AUGMENTATION
+# ─────────────────────────────────────────────────────────────────────────────
+
+class Augmenter:
+
+    def __call__(self, img: np.ndarray) -> np.ndarray:
+        img = img.copy()
+
+        # Random slight rotation (±3°)
+        if random.random() < 0.3:
+            angle = random.uniform(-3, 3)
+            h, w  = img.shape
+            M     = cv2.getRotationMatrix2D((w // 2, h // 2), angle, 1.0)
+            img   = cv2.warpAffine(img, M, (w, h),
+                                   borderMode=cv2.BORDER_CONSTANT,
+                                   borderValue=255)
+
+        # Random brightness/contrast
+        if random.random() < 0.4:
+            alpha = random.uniform(0.8, 1.2)
+            beta  = random.randint(-20, 20)
+            img   = np.clip(alpha * img.astype(np.float32) + beta,
+                            0, 255).astype(np.uint8)
+
+        # Gaussian blur
+        if random.random() < 0.3:
+            ksize = random.choice([3, 5])
+            img   = cv2.GaussianBlur(img, (ksize, ksize), 0)
+
+        # Salt-and-pepper noise
+        if random.random() < 0.2:
+            noise = np.random.randint(0, 100, img.shape)
+            img[noise < 2]  = 0
+            img[noise > 97] = 255
+
+        # Random small horizontal shift
+        if random.random() < 0.2:
+            h, w  = img.shape
+            shift = random.randint(-int(w * 0.05), int(w * 0.05))
+            M     = np.float32([[1, 0, shift], [0, 1, 0]])
+            img   = cv2.warpAffine(img, M, (w, h),
+                                   borderMode=cv2.BORDER_CONSTANT,
+                                   borderValue=255)
+
+        # ── NEW: Horizontal line noise ────────────────────────────────────────
+        # Simulates ruled form lines bleeding through behind the text.
+        # Civil registry forms have printed horizontal grid lines — scanners
+        # often pick these up as faint grey stripes across text fields.
+        if random.random() < 0.3:
+            h, w    = img.shape
+            n_lines = random.randint(1, 3)
+            for _ in range(n_lines):
+                y         = random.randint(0, h - 1)
+                thickness = random.choice([1, 1, 1, 2])   # mostly 1px
+                intensity = random.randint(160, 220)       # light grey, not black
+                cv2.line(img, (0, y), (w, y),
+                         color=intensity, thickness=thickness)
+
+        # ── NEW: Perspective warp ─────────────────────────────────────────────
+        # Simulates documents scanned or photographed at a slight angle.
+        # Keystone distortion is common when forms are placed unevenly on
+        # a flatbed scanner or photographed with a phone camera.
+        if random.random() < 0.25:
+            h, w  = img.shape
+            d     = 0.03
+            dx    = int(w * d)
+            dy    = int(h * d)
+            src   = np.float32([[0, 0], [w, 0], [w, h], [0, h]])
+            dst   = np.float32([
+                [random.randint(0, dx),     random.randint(0, dy)],
+                [w - random.randint(0, dx), random.randint(0, dy)],
+                [w - random.randint(0, dx), h - random.randint(0, dy)],
+                [random.randint(0, dx),     h - random.randint(0, dy)],
+            ])
+            M   = cv2.getPerspectiveTransform(src, dst)
+            img = cv2.warpPerspective(img, M, (w, h),
+                                      borderMode=cv2.BORDER_CONSTANT,
+                                      borderValue=255)
+
+        return img
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+#  DATASET
+# ─────────────────────────────────────────────────────────────────────────────
+
+class CivilRegistryDataset(Dataset):
+    """
+    Args:
+        data_dir         : root folder containing image subfolders (e.g. 'data/train')
+        annotations_file : path to JSON file with image_path + text pairs
+        img_height       : target image height (default 64)
+        img_width        : target image width  (default 512)
+        augment          : True = apply augmentation (training only)
+        form_type        : 'all' or filter by form e.g. 'form1a'
+
+    Properties used by train.py:
+        .num_chars    → passed to CRNN model
+        .char_to_idx  → saved in checkpoint
+        .idx_to_char  → used for decoding predictions
+
+    __getitem__ returns:
+        image_tensor   FloatTensor [1, H, W]
+        target         LongTensor  [label_length]
+        target_length  int
+        text           str  (original ground truth)
+    """
+
+    def __init__(
+        self,
+        data_dir:         str,
+        annotations_file: str,
+        img_height:       int  = 64,
+        img_width:        int  = 512,
+        augment:          bool = False,
+        form_type:        str  = 'all',
+        seed:             Optional[int] = None,   # Rec 2: reproducible augmentation
+    ):
+        self.data_dir   = Path(data_dir)
+        self.augment    = augment
+        self.normalizer = ImageNormalizer(img_height, img_width)
+        self.augmenter  = Augmenter()
+        if seed is not None:                       # Rec 2: seed random for reproducibility
+            random.seed(seed)
+            np.random.seed(seed)
+
+        self.char_to_idx, self.idx_to_char, self.num_chars = build_char_maps()
+
+        with open(annotations_file, 'r', encoding='utf-8') as f:
+            all_annotations = json.load(f)
+
+        if form_type != 'all':
+            all_annotations = [
+                a for a in all_annotations
+                if form_type in a.get('image_path', '')
+            ]
+
+        self.samples: List[Dict] = []
+        missing = 0
+        for ann in all_annotations:
+            img_path = self.data_dir / ann['image_path']
+            if img_path.exists():
+                text = ann['text'].strip()
+                if text:
+                    self.samples.append({
+                        'image_path': str(img_path),
+                        'text':       text,
+                    })
+            else:
+                missing += 1
+
+        if missing > 0:
+            print(f"  [Dataset] WARNING: {missing} image(s) not found and skipped.")
+
+        print(f"  [Dataset] Loaded {len(self.samples)} samples "
+              f"from {annotations_file}  (augment={augment})")
+
+    def __len__(self) -> int:
+        return len(self.samples)
+
+    def __getitem__(self, idx: int):
+        sample = self.samples[idx]
+        text   = sample['text']
+
+        img = cv2.imread(sample['image_path'])
+        if img is None:
+            img = np.ones((64, 512, 3), dtype=np.uint8) * 255
+
+        # FIXED Bug 3: pass augmenter into normalize() so it runs on grayscale
+        # (before binarization), not on the binary output where it has no effect.
+        aug = self.augmenter if self.augment else None
+        normalized = self.normalizer.normalize(img, augmenter=aug)
+
+        image_tensor = self.normalizer.to_tensor(normalized)  # [1, H, W]
+
+        encoded = [
+            self.char_to_idx[c]
+            for c in text
+            if c in self.char_to_idx
+        ]
+        if len(encoded) == 0:
+            encoded = [self.char_to_idx.get(' ', 1)]
+
+        target        = torch.LongTensor(encoded)
+        target_length = len(encoded)
+
+        return image_tensor, target, target_length, text
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+#  COLLATE FUNCTION
+# ─────────────────────────────────────────────────────────────────────────────
+
+def collate_fn(batch):
+    """
+    CTC loss needs all labels packed into one flat 1D tensor.
+    PyTorch's default collator can't handle variable-length labels,
+    so this custom function packs them correctly.
+
+    Returns:
+        images         FloatTensor [B, 1, H, W]
+        targets        LongTensor  [sum of all label lengths]
+        target_lengths LongTensor  [B]
+        texts          List[str]
+    """
+    images, targets, target_lengths, texts = zip(*batch)
+
+    images         = torch.stack(images, dim=0)
+    targets        = torch.cat([t for t in targets])
+    target_lengths = torch.LongTensor(target_lengths)
+
+    return images, targets, target_lengths, list(texts)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+#  HELPER: CREATE ANNOTATION FILE  (run once to build your JSON)
+# ─────────────────────────────────────────────────────────────────────────────
+
+def create_annotation_file(data_dir: str, output_file: str,
+                            extensions=('.jpg', '.jpeg', '.png')):
+    """
+    Auto-generate annotations JSON by scanning data_dir.
+    For each image, looks for a sidecar .txt file with the same name.
+    If not found, uses the filename stem (underscores → spaces) as label.
+
+    Usage:
+        from dataset import create_annotation_file
+        create_annotation_file('data/train', 'data/train_annotations.json')
+        create_annotation_file('data/val',   'data/val_annotations.json')
+    """
+    data_path   = Path(data_dir)
+    annotations = []
+
+    for img_path in sorted(data_path.rglob('*')):
+        if img_path.suffix.lower() not in extensions:
+            continue
+        txt_path = img_path.with_suffix('.txt')
+        if txt_path.exists():
+            label = txt_path.read_text(encoding='utf-8').strip()
+        else:
+            label = img_path.stem.replace('_', ' ')
+        if not label:
+            continue
+        rel_path = img_path.relative_to(data_path)
+        annotations.append({
+            'image_path': str(rel_path).replace('\\', '/'),
+            'text':       label,
+        })
+
+    os.makedirs(Path(output_file).parent, exist_ok=True)
+    with open(output_file, 'w', encoding='utf-8') as f:
+        json.dump(annotations, f, indent=2, ensure_ascii=False)
+
+    print(f"✓ Saved {len(annotations)} entries → {output_file}")
+    return annotations
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+#  SELF-TEST  (python dataset.py)
+# ─────────────────────────────────────────────────────────────────────────────
+
+if __name__ == '__main__':
+    print("=" * 55)
+    print("  dataset.py self-test")
+    print("=" * 55)
+
+    c2i, i2c, n = build_char_maps()
+    print(f"\n  Vocab size : {n}  (including blank=0)")
+    print(f"  'A'={c2i['A']}  '0'={c2i['0']}  ' '={c2i[' ']}  '.'={c2i['.']}")
+
+    dummy = np.ones((80, 300, 3), dtype=np.uint8) * 200
+    norm  = ImageNormalizer(64, 512)
+    out   = norm.normalize(dummy)
+    t     = norm.to_tensor(out)
+    print(f"\n  Normalizer : {dummy.shape} → {out.shape} → tensor {t.shape}")
+
+    fake = [
+        (torch.zeros(1, 64, 512), torch.LongTensor([1, 2, 3]),    3, "ABC"),
+        (torch.zeros(1, 64, 512), torch.LongTensor([4, 5]),        2, "DE"),
+        (torch.zeros(1, 64, 512), torch.LongTensor([6, 7, 8, 9]), 4, "FGHI"),
+    ]
+    imgs, tgts, tlens, txts = collate_fn(fake)
+    print(f"\n  collate_fn : images={imgs.shape}  "
+          f"targets={tgts.shape}  lengths={tlens.tolist()}")
+
+    print("\n  ✓ All checks passed.\n")

debug_and_retrain.py

@@ -0,0 +1,20 @@
+import cv2
+import matplotlib.pyplot as plt
+
+# Load and show the image
+img = cv2.imread('your_image.png')
+plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
+plt.title('Original Image')
+plt.show()
+
+# Preprocess and show
+gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+thresh = cv2.threshold(gray, 128, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU)[1]
+plt.imshow(thresh, cmap='gray')
+plt.title('Thresholded Image')
+plt.show()
+
+# Run OCR and print output
+import pytesseract
+text = pytesseract.image_to_string(thresh)
+print("OCR Output:", text)

extract_actual_data.py

@@ -0,0 +1,203 @@
+"""
+extract_actual_data.py
+======================
+Extract field crops from actual scanned civil registry forms and
+auto-label them with EasyOCR as a starting point for CRNN fine-tuning.
+
+Reads images from:
+    actual_images/{form_type}/*.{png,jpg,jpeg}
+
+For each image:
+  1. Aligns to reference using ORB + ECC + corner fallback
+  2. Crops every field defined in TEMPLATES
+  3. Applies CLAHE per-crop before auto-labeling
+  4. Saves crop to data/actual_crops/
+  5. Auto-labels with EasyOCR + field-type post-processing
+
+Output:
+    data/actual_crops/             -- field crop images
+    data/actual_annotations.json   -- labels for fine-tuning
+
+After running:
+  - Open actual_annotations.json
+  - Fix any wrong 'text' values
+  - Run finetune.py to train
+
+Usage:
+    cd python/CRNN+CTC
+    python extract_actual_data.py
+
+    # or point to a different images folder:
+    python extract_actual_data.py --images /path/to/actual_images
+"""
+
+import os
+import sys
+import json
+import argparse
+import numpy as np
+from PIL import Image
+
+# ── Paths ─────────────────────────────────────────────────────
+THIS_DIR   = os.path.dirname(os.path.abspath(__file__))
+ROOT_DIR   = os.path.dirname(os.path.dirname(THIS_DIR))   # project root
+PYTHON_DIR = os.path.dirname(THIS_DIR)                    # python/
+
+sys.path.insert(0, PYTHON_DIR)
+
+from template_matcher import (
+    TEMPLATES, REFERENCE_IMAGES,
+    align_to_reference, _preprocess, _crop_field,
+    _get_easyocr, _postprocess,
+)
+
+try:
+    import cv2 as _cv2
+    _CV2_OK = True
+except ImportError:
+    _CV2_OK = False
+
+CROPS_DIR  = os.path.join(THIS_DIR, 'data', 'actual_crops')
+ANN_PATH   = os.path.join(THIS_DIR, 'data', 'actual_annotations.json')
+MIN_CROP_W = 10
+MIN_CROP_H = 6
+
+# Substrings that indicate a file is a debug/test output, not a real scan
+_SKIP_SUBSTRINGS = ('debug', 'aligned', 'crops_aligned')
+_SKIP_PREFIXES   = ('test_', 'father_', 'father2_')
+
+
+def _is_scan(fname: str) -> bool:
+    base = fname.lower()
+    ext  = os.path.splitext(base)[1]
+    if ext not in ('.png', '.jpg', '.jpeg', '.tiff', '.bmp'):
+        return False
+    if any(s in base for s in _SKIP_SUBSTRINGS):
+        return False
+    if any(base.startswith(p) for p in _SKIP_PREFIXES):
+        return False
+    return True
+
+
+def _ocr_crop(arr: np.ndarray, reader) -> str:
+    """Run EasyOCR on a uint8 RGB numpy array."""
+    try:
+        results = reader.readtext(arr, detail=0, paragraph=True)
+        return ' '.join(results).strip()
+    except Exception as e:
+        return ''
+
+
+def process_image(img_path: str, form_type: str, reader, crops_dir: str) -> list:
+    """
+    Align one scan, crop every template field, save crops, auto-label.
+    Returns list of annotation dicts for this image.
+    """
+    template = TEMPLATES[form_type]
+    fname    = os.path.basename(img_path)
+    stem     = os.path.splitext(fname)[0]
+
+    try:
+        img = Image.open(img_path).convert('RGB')
+    except Exception as e:
+        print(f'    [skip] Cannot open: {e}')
+        return []
+
+    w, h = img.size
+    print(f'  Processing {fname} ({w}x{h})...')
+
+    # Align (ORB → ECC → corner → resize)
+    img, orb_inliers = align_to_reference(img, form_type)
+    print(f'    ORB inliers: {orb_inliers}')
+
+    # Grayscale + deskew
+    processed = _preprocess(img)
+
+    annotations = []
+
+    for field_name, coords in template.items():
+        x1r, y1r, x2r, y2r, _ = coords
+        crop = _crop_field(processed, x1r, y1r, x2r, y2r)
+
+        if crop is None or crop.size[0] < MIN_CROP_W or crop.size[1] < MIN_CROP_H:
+            continue
+
+        # CLAHE per-crop before OCR (same as extract_fields in template_matcher)
+        gray = np.array(crop.convert('L'))
+        if _CV2_OK:
+            clahe = _cv2.createCLAHE(clipLimit=1.5, tileGridSize=(2, 2))
+            gray  = clahe.apply(gray)
+        arr = np.stack([gray, gray, gray], axis=-1)
+
+        raw   = _ocr_crop(arr, reader)
+        label = _postprocess(raw, field_name)
+
+        crop_fname = f'{form_type}_{stem}_{field_name}.png'
+        crop.save(os.path.join(crops_dir, crop_fname))
+
+        annotations.append({
+            'image_path': os.path.join('data', 'actual_crops', crop_fname),
+            'text':       label,
+            'form_type':  form_type,
+            'field':      field_name,
+            'source_img': fname,
+        })
+
+    print(f'    Saved {len(annotations)} crops')
+    return annotations
+
+
+def main(images_root: str):
+    os.makedirs(CROPS_DIR, exist_ok=True)
+
+    print('[extract] Loading EasyOCR...')
+    reader = _get_easyocr()
+    if reader is None:
+        print('[extract] ERROR: EasyOCR failed to load.')
+        sys.exit(1)
+    print('[extract] EasyOCR ready.')
+
+    all_annotations = []
+
+    for form_type in sorted(TEMPLATES.keys()):
+        folder = os.path.join(images_root, form_type)
+        if not os.path.isdir(folder):
+            print(f'\n[extract] No images in {folder}, skipping.')
+            continue
+
+        scans = sorted(f for f in os.listdir(folder) if _is_scan(f))
+        if not scans:
+            print(f'\n[extract] No scan images in {folder}, skipping.')
+            continue
+
+        ref = REFERENCE_IMAGES.get(form_type, '')
+        if not os.path.exists(ref):
+            print(f'\n[extract] WARNING: No reference image for form {form_type} — alignment will be resize-only')
+
+        print(f'\n[extract] Form {form_type} — {len(scans)} image(s)')
+
+        for fname in scans:
+            anns = process_image(os.path.join(folder, fname), form_type, reader, CROPS_DIR)
+            all_annotations.extend(anns)
+
+    with open(ANN_PATH, 'w', encoding='utf-8') as f:
+        json.dump(all_annotations, f, indent=2, ensure_ascii=False)
+
+    total = len(all_annotations)
+    print(f'\n[extract] Done.')
+    print(f'  Crops saved : {total}')
+    print(f'  Annotations : {ANN_PATH}')
+    print()
+    print('Review actual_annotations.json and correct any wrong labels,')
+    print('then run finetune.py to train on this data.')
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--images',
+        default=os.path.join(ROOT_DIR, 'actual_images'),
+        help='Path to actual_images/ folder (default: <project_root>/actual_images)',
+    )
+    args = parser.parse_args()
+    main(args.images)

field_extractor.py

@@ -0,0 +1,735 @@
+"""
+Philippine Civil Registry — Field Extractor (Dynamic)
+======================================================
+Automatically detects form borders on ANY scan/photo and aligns field
+extraction to the detected boundary — no hardcoded pixel positions.
+
+Field coordinates calibrated directly from official PDF renders at 200 DPI:
+  Form 102 (Birth):    1700 x 2800 px
+  Form 103 (Death):    1700 x 2878 px
+  Form 97  (Marriage): 1700 x 2600 px
+  Form 90  (License):  1700 x 2600 px
+
+Usage:
+    python field_extractor.py --pdf  FORM_102.pdf --form birth
+    python field_extractor.py --pdf  FORM_97.pdf  --form marriage --visualize
+    python field_extractor.py --pdf  FORM_103.pdf --form death    --output results.json
+    python field_extractor.py --image form102.png --form birth    --visualize
+    python field_extractor.py --pdf  FORM_102.pdf --form birth    --checkpoint checkpoints/best_model_emnist.pth
+
+.env file (project root) — each team member sets their own:
+    POPPLER_PATH=C:\\your\\path\\to\\poppler\\Library\\bin
+"""
+
+import argparse
+import os
+import sys
+import json
+import cv2
+import numpy as np
+from pathlib import Path
+
+import torch
+from dotenv import load_dotenv
+
+# Load .env from same folder as this script (works regardless of cwd)
+_script_dir = Path(__file__).parent.resolve()
+load_dotenv(dotenv_path=_script_dir / ".env")
+
+# Poppler path — from .env or None (Linux/Mac auto-detects)
+POPPLER_PATH = os.environ.get("POPPLER_PATH", None)
+DEFAULT_CHECKPOINT = "checkpoints/best_model.pth"
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+#  FIELD RATIO MAPS
+#  Format: field_name: (x1, y1, x2, y2) — ratios 0.0–1.0
+#  Coordinates are relative to the DETECTED FORM BOUNDARY (not full image).
+#  x = left→right,  y = top→bottom
+# ══════════════════════════════════════════════════════════════════════════════
+
+# Form 102 → Certificate of Live Birth (Form 1A)
+BIRTH_FIELDS = {
+    # Header
+    "province":                   (0.02, 0.068, 0.30, 0.088),
+    "registry_number":            (0.66, 0.068, 0.99, 0.108),
+    "city_municipality":          (0.02, 0.090, 0.65, 0.108),
+
+    # Item 1 — Child Name
+    "child_first_name":           (0.03, 0.109, 0.40, 0.141),
+    "child_middle_name":          (0.40, 0.109, 0.64, 0.141),
+    "child_last_name":            (0.64, 0.109, 0.99, 0.141),
+
+    # Items 2-3 — Sex / Date of Birth
+    "sex":                        (0.03, 0.142, 0.30, 0.167),
+    "dob_day":                    (0.40, 0.142, 0.80, 0.167),
+    "dob_month":                  (0.80, 0.142, 0.60, 0.167),
+    "dob_year":                   (0.80, 0.142, 0.99, 0.167),
+
+    # Item 4 — Place of Birth
+    "place_birth_hospital":       (0.03, 0.169, 0.46, 0.197),
+    "place_birth_city":           (0.47, 0.169, 0.70, 0.199),
+    "place_birth_province":       (0.71, 0.169, 0.99, 0.199),
+
+  
+
+    # Mother section
+    "mother_first_name":          (0.03, 0.248, 0.40, 0.276),
+    "mother_middle_name":         (0.40, 0.248, 0.64, 0.276),
+    "mother_last_name":           (0.64, 0.248, 0.99, 0.276),
+    "mother_citizenship":         (0.03, 0.277, 0.50, 0.305),
+    
+
+    # Father section
+    "father_first_name":          (0.03, 0.380, 0.40, 0.410),
+    "father_middle_name":         (0.40, 0.380, 0.64, 0.410),
+    "father_last_name":           (0.64, 0.380, 0.99, 0.410),
+    "father_citizenship":         (0.03, 0.411, 0.28, 0.445),
+   
+
+    # Item 20 — Marriage of Parents
+    "parents_marriage_month":     (0.03, 0.496, 0.19, 0.526),
+    "parents_marriage_day":       (0.19, 0.496, 0.27, 0.526),
+    "parents_marriage_year":      (0.27, 0.496, 0.38, 0.526),
+    
+    "parents_marriage_city":      (0.41, 0.496, 0.68, 0.526),
+    "parents_marriage_province":  (0.68, 0.496, 0.84, 0.526),
+
+   
+}
+
+# Form 103 → Certificate of Death (Form 2A)
+DEATH_FIELDS = {
+    # Header
+    "province":                   (0.04, 0.128, 0.40, 0.144),
+    "registry_number":            (0.52, 0.128, 0.75, 0.144),
+    "city_municipality":          (0.04, 0.145, 0.45, 0.160),
+
+    # Item 1 — Name
+    "deceased_first_name":        (0.10, 0.162, 0.34, 0.178),
+    "deceased_middle_name":       (0.34, 0.162, 0.56, 0.178),
+    "deceased_last_name":         (0.56, 0.162, 0.75, 0.178),
+
+    # Items 2-4 — Sex / Religion / Age
+    "sex":                        (0.04, 0.182, 0.13, 0.220),
+    "age_years":                  (0.28, 0.182, 0.38, 0.202),
+
+    # Item 5 — Place of Death
+    "place_death_hospital":       (0.13, 0.224, 0.42, 0.242),
+    "place_death_city":           (0.42, 0.224, 0.58, 0.242),
+    "place_death_province":       (0.58, 0.224, 0.75, 0.242),
+
+    # Items 6-7 — Date of Death / Citizenship
+    "dod_day":                    (0.10, 0.252, 0.22, 0.268),
+    "dod_month":                  (0.22, 0.252, 0.38, 0.268),
+    "dod_year":                   (0.38, 0.252, 0.52, 0.268),
+    "citizenship":                (0.52, 0.252, 0.75, 0.268),
+
+    # Item 8 — Residence
+    "residence_house":            (0.13, 0.278, 0.40, 0.294),
+    "residence_city":             (0.40, 0.278, 0.56, 0.294),
+    "residence_province":         (0.56, 0.278, 0.75, 0.294),
+
+    # Items 9-10 — Civil Status / Occupation
+    "civil_status":               (0.04, 0.302, 0.38, 0.360),
+    "occupation":                 (0.44, 0.302, 0.75, 0.360),
+
+    # Item 17 — Causes of Death
+    "cause_immediate":            (0.18, 0.402, 0.58, 0.418),
+    "cause_antecedent":           (0.18, 0.424, 0.58, 0.440),
+    "cause_underlying":           (0.18, 0.446, 0.58, 0.462),
+    "cause_other":                (0.18, 0.468, 0.58, 0.484),
+
+    # Item 25 — Informant
+    "informant_name":             (0.04, 0.808, 0.35, 0.822),
+    "informant_address":          (0.04, 0.822, 0.35, 0.836),
+    "informant_date":             (0.35, 0.836, 0.58, 0.850),
+}
+
+# Form 97 → Certificate of Marriage (Form 3A)
+# Only the fields that flow through bridge.py → spaCy NER → SpouseOutput/Form3A.
+# Removed: province, city_municipality, dob_day/month/year (×2),
+#   place_birth_city/prov/country (×2), sex (×2), residence (×2),
+#   religion (×2), civil_status (×2).
+MARRIAGE_FIELDS = {
+    # ── Header ───────────────────────────────────────────────────────────────
+    "registry_number":            (0.62, 0.088, 0.97, 0.104),  # → Form3A.registry_number
+    
+    #"registry_number":            (0.62, 0.088, 0.97, 0.104),  # → Form3A.registry_number
+
+    # ── Item 1 — Name (HUSBAND left / WIFE right) ────────────────────────────
+    "husband_first_name":         (0.23, 0.121, 0.56, 0.139),
+    "husband_middle_name":        (0.23, 0.141, 0.56, 0.159),
+    "husband_last_name":          (0.23, 0.160, 0.56, 0.178),
+    "wife_first_name":            (0.65, 0.121, 0.98, 0.139),
+    "wife_middle_name":           (0.65, 0.141, 0.98, 0.159),
+    "wife_last_name":             (0.65, 0.160, 0.98, 0.178),
+
+   # "husband_first_name":         (0.14, 0.138, 0.47, 0.156),
+   # "husband_middle_name":        (0.14, 0.156, 0.47, 0.174),
+   # "husband_last_name":          (0.14, 0.174, 0.47, 0.192),
+   # "wife_first_name":            (0.53, 0.138, 0.86, 0.156),
+   # "wife_middle_name":           (0.53, 0.156, 0.86, 0.174),
+   # "wife_last_name":             (0.53, 0.174, 0.86, 0.192),
+
+    # ── Item 2b — Age ────────────────────────────────────────────────────────
+    "husband_age":                (0.40, 0.198, 0.47, 0.216),  # → husband.age
+    "wife_age":                   (0.78, 0.198, 0.86, 0.216),  # → wife.age
+
+    # ── Item 4b — Citizenship ────────────────────────────────────────────────
+    "husband_citizenship":        (0.22, 0.252, 0.47, 0.270),  # → husband.nationality
+    "wife_citizenship":           (0.62, 0.252, 0.86, 0.270),  # → wife.nationality
+
+    # ── Item 8 — Name of Father ──────────────────────────────────────────────
+    "husband_father_first":       (0.14, 0.396, 0.24, 0.414),
+    "husband_father_middle":      (0.24, 0.396, 0.34, 0.414),
+    "husband_father_last":        (0.34, 0.396, 0.47, 0.414),
+    "wife_father_first":          (0.53, 0.396, 0.63, 0.414),
+    "wife_father_middle":         (0.63, 0.396, 0.73, 0.414),
+    "wife_father_last":           (0.73, 0.396, 0.86, 0.414),
+
+    # ── Item 9 — Citizenship of Father ──────────────────────────────────────
+    "husband_father_citizenship": (0.14, 0.420, 0.47, 0.436),  # → husband.nationality_of_father
+    "wife_father_citizenship":    (0.53, 0.420, 0.86, 0.436),  # → wife.nationality_of_father
+
+    # ── Item 10 — Name of Mother ─────────────────────────────────────────────
+    "husband_mother_first":       (0.14, 0.444, 0.24, 0.462),
+    "husband_mother_middle":      (0.24, 0.444, 0.34, 0.462),
+    "husband_mother_last":        (0.34, 0.444, 0.47, 0.462),
+    "wife_mother_first":          (0.53, 0.444, 0.63, 0.462),
+    "wife_mother_middle":         (0.63, 0.444, 0.73, 0.462),
+    "wife_mother_last":           (0.73, 0.444, 0.86, 0.462),
+
+    # ── Item 11 — Citizenship of Mother ─────────────────────────────────────
+    "husband_mother_citizenship": (0.14, 0.468, 0.47, 0.484),  # → husband.nationality_of_mother
+    "wife_mother_citizenship":    (0.53, 0.468, 0.86, 0.484),  # → wife.nationality_of_mother
+
+    # ── Items 15–16 — Place / Date of Marriage ───────────────────────────────
+    "place_marriage_office":      (0.14, 0.596, 0.44, 0.614),
+    "place_marriage_city":        (0.44, 0.596, 0.68, 0.614),
+    "place_marriage_province":    (0.68, 0.596, 0.88, 0.614),
+    "date_marriage_day":          (0.14, 0.630, 0.24, 0.648),
+    "date_marriage_month":        (0.24, 0.630, 0.38, 0.648),
+    "date_marriage_year":         (0.38, 0.630, 0.48, 0.648),
+}
+
+# Form 90 → Application for Marriage License
+MARRIAGE_LICENSE_FIELDS = {
+    # Header
+    "province":                   (0.12, 0.092, 0.48, 0.108),
+    "registry_number":            (0.56, 0.092, 0.97, 0.108),
+    "city_municipality":          (0.12, 0.108, 0.48, 0.124),
+    "received_by":                (0.12, 0.124, 0.48, 0.140),
+    "date_of_receipt":            (0.12, 0.140, 0.48, 0.156),
+    "marriage_license_number":    (0.56, 0.124, 0.97, 0.140),
+    "date_of_issuance":           (0.56, 0.140, 0.97, 0.156),
+
+    # Item 1 — Name of Applicant (GROOM left / BRIDE right)
+    "groom_first_name":           (0.02, 0.278, 0.46, 0.294),
+    "bride_first_name":           (0.54, 0.278, 0.97, 0.294),
+    "groom_middle_name":          (0.02, 0.296, 0.46, 0.312),
+    "bride_middle_name":          (0.54, 0.296, 0.97, 0.312),
+    "groom_last_name":            (0.02, 0.314, 0.46, 0.330),
+    "bride_last_name":            (0.54, 0.314, 0.97, 0.330),
+
+    # Item 2 — Date of Birth / Age
+    "groom_dob_day":              (0.02, 0.334, 0.12, 0.350),
+    "groom_dob_month":            (0.12, 0.334, 0.24, 0.350),
+    "groom_dob_year":             (0.24, 0.334, 0.34, 0.350),
+    "groom_age":                  (0.34, 0.334, 0.46, 0.350),
+    "bride_dob_day":              (0.54, 0.334, 0.62, 0.350),
+    "bride_dob_month":            (0.62, 0.334, 0.74, 0.350),
+    "bride_dob_year":             (0.74, 0.334, 0.84, 0.350),
+    "bride_age":                  (0.84, 0.334, 0.97, 0.350),
+
+    # Item 3 — Place of Birth
+    "groom_place_birth_city":     (0.02, 0.354, 0.18, 0.370),
+    "groom_place_birth_province": (0.18, 0.354, 0.32, 0.370),
+    "groom_place_birth_country":  (0.32, 0.354, 0.46, 0.370),
+    "bride_place_birth_city":     (0.54, 0.354, 0.70, 0.370),
+    "bride_place_birth_province": (0.70, 0.354, 0.84, 0.370),
+    "bride_place_birth_country":  (0.84, 0.354, 0.97, 0.370),
+
+    # Item 4 — Sex / Citizenship
+    "groom_sex":                  (0.02, 0.374, 0.16, 0.390),
+    "groom_citizenship":          (0.16, 0.374, 0.46, 0.390),
+    "bride_sex":                  (0.54, 0.374, 0.68, 0.390),
+    "bride_citizenship":          (0.68, 0.374, 0.97, 0.390),
+
+    # Item 5 — Residence
+    "groom_residence":            (0.02, 0.394, 0.46, 0.412),
+    "bride_residence":            (0.54, 0.394, 0.97, 0.412),
+
+    # Item 6 — Religion
+    "groom_religion":             (0.02, 0.424, 0.46, 0.440),
+    "bride_religion":             (0.54, 0.424, 0.97, 0.440),
+
+    # Item 7 — Civil Status
+    "groom_civil_status":         (0.02, 0.452, 0.46, 0.468),
+    "bride_civil_status":         (0.54, 0.452, 0.97, 0.468),
+
+    # Item 9 — Place where dissolved
+    "groom_dissolution_city":     (0.02, 0.496, 0.16, 0.512),
+    "groom_dissolution_province": (0.16, 0.496, 0.30, 0.512),
+    "groom_dissolution_country":  (0.30, 0.496, 0.46, 0.512),
+    "bride_dissolution_city":     (0.54, 0.496, 0.68, 0.512),
+    "bride_dissolution_province": (0.68, 0.496, 0.82, 0.512),
+    "bride_dissolution_country":  (0.82, 0.496, 0.97, 0.512),
+
+    # Item 10 — Date when dissolved
+    "groom_dissolution_day":      (0.02, 0.520, 0.12, 0.536),
+    "groom_dissolution_month":    (0.12, 0.520, 0.24, 0.536),
+    "groom_dissolution_year":     (0.24, 0.520, 0.34, 0.536),
+    "bride_dissolution_day":      (0.54, 0.520, 0.62, 0.536),
+    "bride_dissolution_month":    (0.62, 0.520, 0.74, 0.536),
+    "bride_dissolution_year":     (0.74, 0.520, 0.84, 0.536),
+
+    # Item 12 — Father Name
+    "groom_father_first":         (0.02, 0.594, 0.16, 0.610),
+    "groom_father_middle":        (0.16, 0.594, 0.28, 0.610),
+    "groom_father_last":          (0.28, 0.594, 0.46, 0.610),
+    "bride_father_first":         (0.54, 0.594, 0.66, 0.610),
+    "bride_father_middle":        (0.66, 0.594, 0.78, 0.610),
+    "bride_father_last":          (0.78, 0.594, 0.97, 0.610),
+
+    # Item 13 — Father Citizenship
+    "groom_father_citizenship":   (0.02, 0.620, 0.46, 0.636),
+    "bride_father_citizenship":   (0.54, 0.620, 0.97, 0.636),
+
+    # Item 14 — Father Residence
+    "groom_father_residence":     (0.02, 0.644, 0.46, 0.660),
+    "bride_father_residence":     (0.54, 0.644, 0.97, 0.660),
+
+    # Item 15 — Mother Name
+    "groom_mother_first":         (0.02, 0.674, 0.16, 0.690),
+    "groom_mother_middle":        (0.16, 0.674, 0.28, 0.690),
+    "groom_mother_last":          (0.28, 0.674, 0.46, 0.690),
+    "bride_mother_first":         (0.54, 0.674, 0.66, 0.690),
+    "bride_mother_middle":        (0.66, 0.674, 0.78, 0.690),
+    "bride_mother_last":          (0.78, 0.674, 0.97, 0.690),
+
+    # Item 16 — Mother Citizenship
+    "groom_mother_citizenship":   (0.02, 0.696, 0.46, 0.712),
+    "bride_mother_citizenship":   (0.54, 0.696, 0.97, 0.712),
+
+    # Item 17 — Mother Residence
+    "groom_mother_residence":     (0.02, 0.720, 0.46, 0.736),
+    "bride_mother_residence":     (0.54, 0.720, 0.97, 0.736),
+}
+
+FORM_FIELDS = {
+    "birth":            BIRTH_FIELDS,
+    "death":            DEATH_FIELDS,
+    "marriage":         MARRIAGE_FIELDS,
+    "marriage_license": MARRIAGE_LICENSE_FIELDS,
+}
+
+COLOURS = [
+    (0,200,0),(0,150,255),(200,0,200),(0,200,200),(200,200,0),(220,20,60),
+    (255,140,0),(150,50,200),(0,160,80),(30,144,255),(255,20,147),(100,200,100),
+]
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+#  FORM BOUNDS DETECTOR
+#  Finds the outer border of a civil registry form using line detection.
+#  Falls back to full image if detection fails.
+# ══════════════════════════════════════════════════════════════════════════════
+
+class FormBoundsDetector:
+    def __init__(self, verbose=False):
+        self.verbose = verbose
+
+    def detect(self, image_bgr):
+        h, w   = image_bgr.shape[:2]
+        gray   = cv2.cvtColor(image_bgr, cv2.COLOR_BGR2GRAY)
+        bounds = self._detect_by_lines(gray, w, h)
+        if bounds is None:
+            if self.verbose:
+                print("  [Bounds] Line detection failed — using full image")
+            return (0, 0, w, h)
+        if self.verbose:
+            print(f"  [Bounds] Detected: {bounds}")
+        return bounds
+
+    def _detect_by_lines(self, gray, w, h):
+        try:
+            thresh  = cv2.adaptiveThreshold(
+                gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
+                cv2.THRESH_BINARY_INV, 11, 2)
+            hk      = cv2.getStructuringElement(cv2.MORPH_RECT, (max(w // 5, 10), 1))
+            h_lines = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, hk)
+            h_rows  = np.where(np.sum(h_lines, axis=1) > w * 0.15)[0]
+            vk      = cv2.getStructuringElement(cv2.MORPH_RECT, (1, max(h // 5, 10)))
+            v_lines = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, vk)
+            v_cols  = np.where(np.sum(v_lines, axis=0) > h * 0.08)[0]
+            if len(h_rows) == 0 or len(v_cols) == 0:
+                return None
+            top, bottom = int(h_rows.min()), int(h_rows.max())
+            left, right = int(v_cols.min()), int(v_cols.max())
+            if (right - left) < w * 0.4 or (bottom - top) < h * 0.4:
+                return None
+            return (left, top, right, bottom)
+        except Exception as e:
+            if self.verbose:
+                print(f"  [Bounds error] {e}")
+            return None
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+#  DYNAMIC FIELD EXTRACTOR
+#  Crops each field region relative to the detected form boundary.
+#  Works on any image size, DPI, scan margin, or slight rotation.
+# ══════════════════════════════════════════════════════════════════════════════
+
+class DynamicFieldExtractor:
+    def __init__(self, form_type="birth", verbose=False):
+        self.form_type    = form_type.lower()
+        self.field_map    = FORM_FIELDS.get(self.form_type, BIRTH_FIELDS)
+        self.detector     = FormBoundsDetector(verbose=verbose)
+        self.verbose      = verbose
+        self._last_bounds = None
+
+    def _to_bgr(self, image):
+        try:
+            from PIL import Image as PILImage
+            if isinstance(image, PILImage.Image):
+                arr = np.array(image.convert("RGB"))
+                return cv2.cvtColor(arr, cv2.COLOR_RGB2BGR)
+        except ImportError:
+            pass
+        if isinstance(image, np.ndarray):
+            if len(image.shape) == 2:
+                return cv2.cvtColor(image, cv2.COLOR_GRAY2BGR)
+            if image.shape[2] == 4:
+                return cv2.cvtColor(image, cv2.COLOR_BGRA2BGR)
+            return image
+        raise TypeError(f"Unsupported image type: {type(image)}")
+
+    def extract(self, image):
+        """Returns {field_name: BGR numpy array}."""
+        image = self._to_bgr(image)
+        h, w  = image.shape[:2]
+        left, top, right, bottom = self.detector.detect(image)
+        self._last_bounds = (left, top, right, bottom)
+        form_w = right - left
+        form_h = bottom - top
+        if self.verbose:
+            print(f"  [Extract] Image={w}x{h} "
+                  f" Form={form_w}x{form_h} @ ({left},{top})-({right},{bottom})")
+        crops = {}
+        for name, (rx1, ry1, rx2, ry2) in self.field_map.items():
+            x1 = max(0, min(int(left + rx1 * form_w), w - 1))
+            y1 = max(0, min(int(top  + ry1 * form_h), h - 1))
+            x2 = max(0, min(int(left + rx2 * form_w), w - 1))
+            y2 = max(0, min(int(top  + ry2 * form_h), h - 1))
+            if x2 > x1 and y2 > y1:
+                crops[name] = image[y1:y2, x1:x2]
+        return crops
+
+    def visualize(self, image, output_path=None):
+        """Draw detected boundary + field boxes. Returns annotated BGR image."""
+        image = self._to_bgr(image)
+        vis   = image.copy()
+        h, w  = vis.shape[:2]
+        self.extract(image)
+        left, top, right, bottom = self._last_bounds
+        form_w = right - left
+        form_h = bottom - top
+        cv2.rectangle(vis, (left, top), (right, bottom), (0, 140, 255), 3)
+        cv2.putText(vis, "DETECTED FORM BOUNDARY",
+                    (left, max(0, top - 8)),
+                    cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 140, 255), 1)
+        for idx, (name, (rx1, ry1, rx2, ry2)) in enumerate(self.field_map.items()):
+            x1 = max(0, min(int(left + rx1 * form_w), w - 1))
+            y1 = max(0, min(int(top  + ry1 * form_h), h - 1))
+            x2 = max(0, min(int(left + rx2 * form_w), w - 1))
+            y2 = max(0, min(int(top  + ry2 * form_h), h - 1))
+            c  = COLOURS[idx % len(COLOURS)]
+            cv2.rectangle(vis, (x1, y1), (x2, y2), c, 2)
+            cv2.putText(vis, name[:22], (x1 + 2, max(0, y1 - 2)),
+                        cv2.FONT_HERSHEY_SIMPLEX, 0.28, c, 1)
+        if output_path:
+            cv2.imwrite(str(output_path), vis)
+            print(f"  Field map saved -> {output_path}")
+        return vis
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+#  FIELD NORMALIZER — prepares a BGR crop for CRNN inference
+# ══════════════════════════════════════════════════════════════════════════════
+
+class FieldNormalizer:
+    def __init__(self, target_height=64, target_width=512):
+        self.H = target_height
+        self.W = target_width
+
+    def _crop_to_text(self, gray):
+        inv = cv2.bitwise_not(gray)
+        _, thresh = cv2.threshold(inv, 20, 255, cv2.THRESH_BINARY)
+        coords = np.column_stack(np.where(thresh > 0))
+        if len(coords) == 0:
+            return gray
+        y_min, x_min = coords.min(axis=0)
+        y_max, x_max = coords.max(axis=0)
+        pad   = max(4, int((y_max - y_min) * 0.15))
+        y_min = max(0, y_min - pad)
+        x_min = max(0, x_min - pad)
+        y_max = min(gray.shape[0] - 1, y_max + pad)
+        x_max = min(gray.shape[1] - 1, x_max + pad)
+        return gray[y_min:y_max + 1, x_min:x_max + 1]
+
+    def _smart_resize(self, gray):
+        h, w = gray.shape
+        if h == 0 or w == 0:
+            return np.ones((self.H, self.W), dtype=np.uint8) * 255
+        scale = self.H / h
+        new_w = int(w * scale)
+        new_h = self.H
+        if new_w > self.W:
+            scale = self.W / w
+            new_h = int(h * scale)
+            new_w = self.W
+        resized = cv2.resize(gray, (new_w, new_h), interpolation=cv2.INTER_LANCZOS4)
+        canvas  = np.ones((self.H, self.W), dtype=np.uint8) * 255
+        y_off   = (self.H - new_h) // 2
+        x_off   = (self.W - new_w) // 2
+        canvas[y_off:y_off + new_h, x_off:x_off + new_w] = resized
+        return canvas
+
+    def _binarize(self, img):
+        _, otsu = cv2.threshold(img, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
+        white_ratio = np.mean(otsu == 255)
+        if white_ratio < 0.30 or white_ratio > 0.97:
+            return cv2.adaptiveThreshold(
+                img, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
+                cv2.THRESH_BINARY, 11, 2)
+        return otsu
+
+    def normalize(self, crop) -> np.ndarray:
+        """Accept BGR numpy array or PIL image, return normalized binary array."""
+        try:
+            from PIL import Image as PILImage
+            if isinstance(crop, PILImage.Image):
+                crop = cv2.cvtColor(np.array(crop.convert("RGB")), cv2.COLOR_RGB2BGR)
+        except ImportError:
+            pass
+        gray = cv2.cvtColor(crop, cv2.COLOR_BGR2GRAY) if len(crop.shape) == 3 else crop.copy()
+        gray = cv2.fastNlMeansDenoising(gray, None, 10, 7, 21)
+        gray = self._crop_to_text(gray)
+        gray = self._smart_resize(gray)
+        return self._binarize(gray)
+
+    def to_tensor(self, img: np.ndarray) -> torch.Tensor:
+        return torch.FloatTensor(
+            img.astype(np.float32) / 255.0
+        ).unsqueeze(0).unsqueeze(0)
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+#  CRNN MODEL LOADER
+# ══════════════════════════════════════════════════════════════════════════════
+
+def load_crnn_model(checkpoint_path: str, device: torch.device):
+    sys.path.insert(0, str(Path(__file__).parent))
+    from crnn_model import get_crnn_model
+
+    print(f"  Loading CRNN model from: {checkpoint_path}")
+    c           = torch.load(checkpoint_path, map_location=device, weights_only=False)
+    config      = c.get("config", {})
+    idx_to_char = c["idx_to_char"]
+    num_chars   = c["model_state_dict"]["fc.weight"].shape[0]
+
+    model = get_crnn_model(
+        model_type=config.get("model_type", "standard"),
+        img_height=config.get("img_height", 64),
+        num_chars=num_chars,
+        hidden_size=config.get("hidden_size", 128),
+        num_lstm_layers=config.get("num_lstm_layers", 1),
+    ).to(device)
+    model.load_state_dict(c["model_state_dict"])
+    model.eval()
+
+    val_cer  = c.get("val_cer",  None)
+    val_loss = c.get("val_loss", None)
+    metric   = f"val_cer={val_cer:.2f}%" if val_cer else \
+               f"val_loss={val_loss:.4f}" if val_loss else "no metric"
+    print(f"  Model loaded  |  {metric}  |  chars={num_chars}")
+    return model, idx_to_char, config.get("img_height", 64), config.get("img_width", 512)
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+#  GREEDY CTC DECODE
+# ══════════════════════════════════════════════════════════════════════════════
+
+def greedy_decode(outputs: torch.Tensor, idx_to_char: dict) -> str:
+    pred_indices = torch.argmax(outputs, dim=2).permute(1, 0)
+    chars, prev  = [], -1
+    for idx in pred_indices[0]:
+        idx = idx.item()
+        if idx != 0 and idx != prev and idx in idx_to_char:
+            chars.append(idx_to_char[idx])
+        prev = idx
+    return "".join(chars)
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+#  PDF → PIL IMAGE
+# ══════════════════════════════════════════════════════════════════════════════
+
+def pdf_to_image(pdf_path: str, dpi: int = 200):
+    from pdf2image import convert_from_path
+    # Resolve to absolute path — fixes "Unable to get page count" on Windows
+    pdf_path = str(Path(pdf_path).resolve())
+    kwargs = {"dpi": dpi, "first_page": 1, "last_page": 1}
+    if POPPLER_PATH:
+        kwargs["poppler_path"] = str(Path(POPPLER_PATH).resolve())
+    return convert_from_path(pdf_path, **kwargs)[0]
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+#  CRNN OCR — runs on extracted field crops
+# ══════════════════════════════════════════════════════════════════════════════
+
+def run_crnn_ocr(crops: dict, model, idx_to_char: dict,
+                 img_h: int, img_w: int, device: torch.device) -> dict:
+    normalizer = FieldNormalizer(target_height=img_h, target_width=img_w)
+    results    = {}
+    with torch.no_grad():
+        for name, crop in crops.items():
+            try:
+                norm   = normalizer.normalize(crop)
+                tensor = normalizer.to_tensor(norm).to(device)
+                text   = greedy_decode(model(tensor).cpu(), idx_to_char)
+                results[name] = text
+            except Exception as e:
+                results[name] = f"[ERROR: {e}]"
+    return results
+
+
+# ════════════════════════════════════���═════════════════════════════════════════
+#  CONVENIENCE WRAPPER — for other scripts that import this module
+# ══════════════════════════════════════════════════════════════════════════════
+
+def extract_field_images(image, form_type="birth", verbose=False):
+    """Extract field crops using dynamic boundary detection.
+
+    Parameters
+    ----------
+    image     : PIL Image or BGR numpy array
+    form_type : str  'birth' | 'death' | 'marriage' | 'marriage_license'
+    verbose   : bool
+
+    Returns
+    -------
+    dict  {field_name: BGR numpy array}
+    """
+    return DynamicFieldExtractor(form_type=form_type, verbose=verbose).extract(image)
+
+
+# Keep old name as alias so any existing code doesn't break
+extract_field_images_dynamic = extract_field_images
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+#  MAIN
+# ══════════════════════════════════════════════════════════════════════════════
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="PH Civil Registry Field Extractor — Dynamic CRNN OCR")
+    group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument("--pdf",   help="Path to scanned PDF")
+    group.add_argument("--image", help="Path to scanned image (JPG/PNG)")
+    parser.add_argument("--form",       required=True,
+                        choices=["birth", "death", "marriage", "marriage_license"])
+    parser.add_argument("--checkpoint", default=DEFAULT_CHECKPOINT)
+    parser.add_argument("--visualize",  action="store_true",
+                        help="Save annotated field-map image")
+    parser.add_argument("--output",     default=None,
+                        help="Save extracted fields to JSON")
+    parser.add_argument("--poppler",    default=None,
+                        help="Override Poppler bin path (overrides .env)")
+    parser.add_argument("--dpi",        type=int, default=200)
+    parser.add_argument("--verbose",    action="store_true")
+    args = parser.parse_args()
+
+    global POPPLER_PATH
+    if args.poppler:
+        POPPLER_PATH = args.poppler
+
+    form_labels = {
+        "birth":            "Form 102 — Certificate of Live Birth",
+        "death":            "Form 103 — Certificate of Death",
+        "marriage":         "Form 97  — Certificate of Marriage",
+        "marriage_license": "Form 90  — Application for Marriage License",
+    }
+    input_file = args.pdf or args.image
+
+    print("\nPhilippine Civil Registry OCR — Dynamic Field Extractor")
+    print("=" * 65)
+    print(f"  Form       : {form_labels[args.form]}")
+    print(f"  File       : {input_file}")
+    print(f"  Checkpoint : {args.checkpoint}")
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    print(f"  Device     : {device}\n")
+
+    if not os.path.exists(args.checkpoint):
+        print(f"ERROR: Checkpoint not found: {args.checkpoint}")
+        sys.exit(1)
+
+    model, idx_to_char, img_h, img_w = load_crnn_model(args.checkpoint, device)
+
+    # Load image
+    if args.pdf:
+        print(f"  Converting PDF to image at {args.dpi} DPI...")
+        try:
+            pil_img    = pdf_to_image(args.pdf, dpi=args.dpi)
+            page_image = cv2.cvtColor(np.array(pil_img.convert("RGB")), cv2.COLOR_RGB2BGR)
+        except Exception as e:
+            print(f"\nERROR converting PDF: {e}")
+            print("Fix: add POPPLER_PATH=C:\\...\\poppler\\Library\\bin to your .env file")
+            sys.exit(1)
+    else:
+        page_image = cv2.imread(args.image)
+        if page_image is None:
+            print(f"ERROR: Could not load image: {args.image}")
+            sys.exit(1)
+
+    h, w = page_image.shape[:2]
+    print(f"  Page size  : {w} x {h} px")
+
+    extractor = DynamicFieldExtractor(form_type=args.form, verbose=args.verbose)
+
+    if args.visualize:
+        stem     = Path(input_file).stem
+        out_path = stem + "_field_map.jpg"
+        extractor.visualize(page_image, output_path=out_path)
+        print(f"  Field map saved -> {out_path}")
+
+    print(f"\n  Detecting form boundary and extracting fields...")
+    crops = extractor.extract(page_image)
+    print(f"  {len(crops)} field crops extracted")
+
+    print(f"\n  Running CRNN OCR on {len(crops)} fields...")
+    results = run_crnn_ocr(crops, model, idx_to_char, img_h, img_w, device)
+
+    print(f"\n{'─'*65}")
+    print(f"  {'FIELD':<42} TEXT")
+    print(f"{'─'*65}")
+    for name, text in results.items():
+        print(f"  {name:<42} {text if text.strip() else '(empty)'}")
+    print(f"{'─'*65}")
+    print(f"\n  Fields recognized : {sum(1 for t in results.values() if t.strip())} / {len(results)}")
+
+    if args.output:
+        with open(args.output, "w", encoding="utf-8") as f:
+            json.dump({"form": form_labels[args.form], "file": input_file,
+                       "fields": results}, f, ensure_ascii=False, indent=2)
+        print(f"\n  Results saved -> {args.output}")
+    print()
+
+
+if __name__ == "__main__":
+    main()

finetune.py

@@ -0,0 +1,233 @@
+"""
+finetune.py
+===========
+Fine-tune CRNN+CTC on generated civil registry form crops.
+
+Continues from best_model_v2.pth, trains on actual_annotations.json
++ train_annotations.json, saves to best_model_v4.pth.
+
+Usage:
+    python finetune.py
+
+Output:
+    checkpoints/best_model_v4.pth
+"""
+
+import os
+import sys
+import json
+import shutil
+import torch
+import torch.nn.functional as F
+import torch.optim as optim
+from torch.utils.data import DataLoader, ConcatDataset
+
+sys.path.append('.')
+from crnn_model import get_crnn_model
+from dataset import CivilRegistryDataset, collate_fn
+
+# ── Config ────────────────────────────────────────────────────
+CHECKPOINT_IN  = "checkpoints/best_model_v3.pth"
+CHECKPOINT_OUT = "checkpoints/best_model_v4.pth"
+
+ACTUAL_ANN = "data/actual_annotations.json"  # real scanned forms
+SYNTH_ANN  = "data/train_annotations.json"   # synthetic / train split
+VAL_ANN    = "data/val_annotations.json"     # validation set
+
+DRIVE_BACKUP = "/content/drive/MyDrive/crnn_finetune/CRNN+CTC/checkpoints/best_model_v4.pth"
+
+IMG_HEIGHT = 64
+IMG_WIDTH  = 512
+BATCH_SIZE = 32
+
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# ── Phase settings ────────────────────────────────────────────
+PHASES = [
+    # (name, epochs, lr, freeze_cnn, patience)
+    ("Phase 1 — CNN frozen,   warm up on actual crops",  20, 1e-4, True,  5),
+    ("Phase 2 — Full model,   main training",            30, 1e-5, False, 6),
+    ("Phase 3 — Full model,   slow burn",                30, 5e-6, False, 6),
+    ("Phase 4 — Full model,   final polish",             20, 1e-6, False, 5),
+]
+
+# ── Fix Windows backslash paths ───────────────────────────────
+def fix_paths(json_path):
+    with open(json_path) as f:
+        ann = json.load(f)
+    changed = False
+    for a in ann:
+        if 'image_path' in a and '\\' in a['image_path']:
+            a['image_path'] = a['image_path'].replace('\\', '/')
+            changed = True
+    if changed:
+        with open(json_path, 'w') as f:
+            json.dump(ann, f)
+        print(f"  Fixed backslash paths in {json_path}")
+
+# ── Main ──────────────────────────────────────────────────────
+def main():
+    print("=" * 60)
+    print("  Fine-tuning CRNN+CTC on civil registry form crops")
+    print("=" * 60)
+    print(f"  Device      : {DEVICE}")
+    print(f"  Checkpoint  : {CHECKPOINT_IN}")
+
+    # ── Check required files ──────────────────────────────────
+    for f in [CHECKPOINT_IN, VAL_ANN]:
+        if not os.path.exists(f):
+            print(f"ERROR: {f} not found.")
+            sys.exit(1)
+
+    # ── Fix backslash paths ───────────────────────────────────
+    for ann_file in [ACTUAL_ANN, SYNTH_ANN, VAL_ANN]:
+        if os.path.exists(ann_file):
+            fix_paths(ann_file)
+
+    # ── Datasets ──────────────────────────────────────────────
+    datasets_to_merge = []
+
+    # 1. Actual scanned forms (highest priority — real data)
+    if os.path.exists(ACTUAL_ANN):
+        actual_dataset = CivilRegistryDataset(
+            data_dir=".", annotations_file=ACTUAL_ANN,
+            img_height=IMG_HEIGHT, img_width=IMG_WIDTH, augment=True
+        )
+        datasets_to_merge.append(actual_dataset)
+        print(f"  Actual crops: {len(actual_dataset)}  (real scanned forms)")
+    else:
+        print(f"  [!] {ACTUAL_ANN} not found")
+
+    # 2. Fully synthetic — keep so model doesn't forget basic characters
+    if os.path.exists(SYNTH_ANN):
+        synth_dataset = CivilRegistryDataset(
+            data_dir=".", annotations_file=SYNTH_ANN,
+            img_height=IMG_HEIGHT, img_width=IMG_WIDTH, augment=True
+        )
+        datasets_to_merge.append(synth_dataset)
+        print(f"  Synth crops : {len(synth_dataset)}  (fully synthetic)")
+
+    if not datasets_to_merge:
+        print("ERROR: No training data found.")
+        sys.exit(1)
+
+    val_dataset = CivilRegistryDataset(
+        data_dir=".", annotations_file=VAL_ANN,
+        img_height=IMG_HEIGHT, img_width=IMG_WIDTH, augment=False
+    )
+
+    train_dataset = ConcatDataset(datasets_to_merge) if len(datasets_to_merge) > 1 else datasets_to_merge[0]
+    print(f"  Total train : {len(train_dataset)}")
+    print(f"  Val         : {len(val_dataset)}")
+
+    train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE,
+                              shuffle=True,  num_workers=0, collate_fn=collate_fn)
+    val_loader   = DataLoader(val_dataset,   batch_size=BATCH_SIZE,
+                              shuffle=False, num_workers=0, collate_fn=collate_fn)
+
+    # ── Load checkpoint ───────────────────────────────────────
+    print(f"\n  Loading {CHECKPOINT_IN}...")
+    ckpt   = torch.load(CHECKPOINT_IN, map_location=DEVICE, weights_only=False)
+    config = ckpt.get('config', {})
+
+    ref_dataset = datasets_to_merge[0]
+    model = get_crnn_model(
+        model_type      = config.get('model_type', 'standard'),
+        img_height      = config.get('img_height', 64),
+        num_chars       = ref_dataset.num_chars,
+        hidden_size     = config.get('hidden_size', 128),
+        num_lstm_layers = config.get('num_lstm_layers', 1),
+    ).to(DEVICE)
+
+    missing, _ = model.load_state_dict(ckpt['model_state_dict'], strict=False)
+    if missing:
+        print(f"  Note: {len(missing)} layers re-initialized (expected if vocab size changed)")
+    print(f"  Loaded epoch {ckpt.get('epoch','?')}  "
+          f"val_loss={ckpt.get('val_loss', ckpt.get('val_cer', 0)):.4f}")
+
+    criterion = torch.nn.CTCLoss(blank=0, reduction='mean', zero_infinity=True)
+    os.makedirs("checkpoints", exist_ok=True)
+
+    # ── Train/val loop ────────────────────────────────────────
+    def run_epoch(loader, training, optimizer=None):
+        model.train() if training else model.eval()
+        total, n = 0, 0
+        ctx = torch.enable_grad() if training else torch.no_grad()
+        with ctx:
+            for images, targets, target_lengths, _ in loader:
+                images     = images.to(DEVICE)
+                batch_size = images.size(0)
+                if training:
+                    optimizer.zero_grad()
+                outputs       = F.log_softmax(model(images), dim=2)
+                seq_len       = outputs.size(0)
+                input_lengths = torch.full((batch_size,), seq_len, dtype=torch.long)
+                loss = criterion(outputs, targets, input_lengths, target_lengths)
+                if not torch.isnan(loss) and not torch.isinf(loss):
+                    if training:
+                        loss.backward()
+                        torch.nn.utils.clip_grad_norm_(model.parameters(), 5)
+                        optimizer.step()
+                    total += loss.item()
+                    n     += 1
+        return total / max(n, 1)
+
+    best_overall = float('inf')
+
+    for phase_name, epochs, lr, freeze_cnn, patience in PHASES:
+        print(f"\n{'='*60}")
+        print(f"  {phase_name}   LR={lr}")
+        print(f"{'='*60}")
+
+        for name, param in model.named_parameters():
+            param.requires_grad = not (freeze_cnn and 'cnn' in name)
+
+        trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
+        print(f"  Trainable params : {trainable:,}")
+
+        opt   = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=lr)
+        sched = optim.lr_scheduler.ReduceLROnPlateau(opt, patience=2, factor=0.5)
+        best  = float('inf')
+        wait  = 0
+
+        for epoch in range(1, epochs + 1):
+            tr = run_epoch(train_loader, True,  opt)
+            vl = run_epoch(val_loader,   False, None)
+            sched.step(vl)
+
+            if vl < best:
+                best = vl
+                wait = 0
+                if vl < best_overall:
+                    best_overall = vl
+                    torch.save({
+                        **ckpt,
+                        'model_state_dict': model.state_dict(),
+                        'config':           config,
+                        'char_to_idx':      ref_dataset.char_to_idx,
+                        'idx_to_char':      ref_dataset.idx_to_char,
+                        'epoch':            epoch,
+                        'val_loss':         vl,
+                    }, CHECKPOINT_OUT)
+                print(f"  Epoch {epoch:02d}/{epochs}  Train={tr:.4f}  Val={vl:.4f}  <- saved")
+            else:
+                wait += 1
+                print(f"  Epoch {epoch:02d}/{epochs}  Train={tr:.4f}  Val={vl:.4f}  (patience {wait}/{patience})")
+                if wait >= patience:
+                    print(f"  Early stopping.")
+                    break
+
+    # ── Drive backup ──────────────────────────────────────────
+    if os.path.exists(CHECKPOINT_OUT) and os.path.exists(os.path.dirname(DRIVE_BACKUP)):
+        shutil.copy(CHECKPOINT_OUT, DRIVE_BACKUP)
+        print(f"\n  Backed up to Drive: {DRIVE_BACKUP}")
+
+    print(f"\n{'='*60}")
+    print(f"  Fine-tuning complete!")
+    print(f"  Best val loss : {best_overall:.4f}")
+    print(f"  Saved         : {CHECKPOINT_OUT}")
+    print(f"{'='*60}")
+
+
+if __name__ == '__main__':
+    main()

fix_annotations.py

@@ -0,0 +1,40 @@
+import json, os
+
+# Maps any image path to its correct form subfolder.
+# FIXED: was only handling form1a/form2a — missed form3a and form90.
+def detect_folder(image_path):
+    for form in ['form1a', 'form2a', 'form3a', 'form90']:
+        if form in image_path:
+            return form
+    return 'form1a'   # safe fallback
+
+for split in ['train', 'val']:
+    ann_file = f'data/{split}_annotations.json'
+    if not os.path.exists(ann_file):
+        print(f'SKIP: {ann_file} not found')
+        continue
+
+    with open(ann_file) as f:
+        data = json.load(f)
+
+    fixed = []
+    skipped = 0
+    for d in data:
+        # Support both old key names ('image'/'label') and new ('image_path'/'text')
+        image_val = d.get('image') or d.get('image_path', '')
+        text_val  = d.get('label') or d.get('text', '')
+
+        if not image_val or not text_val:
+            skipped += 1
+            continue
+
+        filename = os.path.basename(image_val)
+        folder   = detect_folder(image_val)
+        fixed.append({'image_path': f'{folder}/{filename}', 'text': text_val})
+
+    with open(ann_file, 'w') as f:
+        json.dump(fixed, f, indent=2)
+
+    print(f'{split}: {len(fixed)} fixed, {skipped} skipped')
+
+print('Done!')

fix_data.py

@@ -0,0 +1,770 @@
+"""
+fix_data.py
+===========
+Generates synthetic training images for the Civil Registry OCR system.
+
+Run this ONCE before training to create your dataset.
+
+STEP ORDER:
+  1. python generate_ph_names.py   <- generates data/ph_names.json
+  2. python fix_data.py            <- generates all training images  (THIS FILE)
+  3. python train.py               <- trains the CRNN model
+
+WHAT IT GENERATES:
+  - Printed text images of names, dates, places, and other form fields
+  - Covers all 4 form types: birth, death, marriage, marriage license
+  - Splits into train (90%) and val (10%)
+  - Writes data/train_annotations.json and data/val_annotations.json
+
+OUTPUT STRUCTURE:
+  data/
+    train/
+      form1a/   <- birth certificate fields
+      form2a/   <- death certificate fields
+      form3a/   <- marriage certificate fields
+      form90/   <- marriage license fields
+    val/
+      form1a/
+      form2a/
+      form3a/
+      form90/
+    train_annotations.json
+    val_annotations.json
+"""
+
+import os
+import json
+import random
+import numpy as np
+from pathlib import Path
+from PIL import Image, ImageDraw, ImageFont, ImageFilter
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+#  CONFIG
+# ─────────────────────────────────────────────────────────────────────────────
+
+IMG_WIDTH   = 512
+IMG_HEIGHT  = 64
+FONT_SIZE   = 22
+VAL_SPLIT   = 0.10
+RANDOM_SEED = 42
+
+SAMPLES_PER_FORM = {
+    'form1a': 6000,
+    'form2a': 4000,
+    'form3a': 4000,
+    'form90': 2000,
+}
+
+PH_NAMES_FILE = 'data/ph_names.json'
+
+random.seed(RANDOM_SEED)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+#  FONT LOADER
+# ─────────────────────────────────────────────────────────────────────────────
+
+def load_font(size=FONT_SIZE):
+    """Load a single font — used as fallback. Prefer load_font_pool()."""
+    for fp in [
+        'arial.ttf', 'Arial.ttf',
+        '/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf',
+        '/usr/share/fonts/truetype/liberation/LiberationSans-Regular.ttf',
+        '/System/Library/Fonts/Helvetica.ttc',
+        'C:/Windows/Fonts/arial.ttf',
+        'C:/Windows/Fonts/calibri.ttf',
+    ]:
+        try:
+            return ImageFont.truetype(fp, size)
+        except Exception:
+            continue
+    print("WARNING: Could not load a TrueType font. Using default bitmap font.")
+    print("         Prediction accuracy may be lower.")
+    return ImageFont.load_default()
+
+
+def load_font_pool(size=FONT_SIZE):
+    """
+    Load a pool of diverse fonts so the model trains on varied typefaces.
+    Using only one font causes the model to overfit to that font's style and
+    fail on real civil registry documents which use mixed fonts.
+    Returns a list of at least 1 font; caller picks randomly per image.
+    """
+    candidates = [
+        # Sans-serif (most common in PH civil registry printed forms)
+        'arial.ttf', 'Arial.ttf',
+        '/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf',
+        '/usr/share/fonts/truetype/liberation/LiberationSans-Regular.ttf',
+        '/usr/share/fonts/truetype/liberation/LiberationSans-Bold.ttf',
+        'C:/Windows/Fonts/arial.ttf',
+        'C:/Windows/Fonts/arialbd.ttf',
+        'C:/Windows/Fonts/calibri.ttf',
+        'C:/Windows/Fonts/calibrib.ttf',
+        # Serif (used in older typewriter-style registry entries)
+        '/usr/share/fonts/truetype/dejavu/DejaVuSerif.ttf',
+        '/usr/share/fonts/truetype/liberation/LiberationSerif-Regular.ttf',
+        'C:/Windows/Fonts/times.ttf',
+        'C:/Windows/Fonts/Georgia.ttf',
+        '/System/Library/Fonts/Times.ttc',
+        # Mono (typewriter — common in pre-2000 civil registry forms)
+        '/usr/share/fonts/truetype/dejavu/DejaVuSansMono.ttf',
+        '/usr/share/fonts/truetype/liberation/LiberationMono-Regular.ttf',
+        'C:/Windows/Fonts/cour.ttf',
+        # Condensed / narrow (space-saving fonts used in registry tables)
+        'C:/Windows/Fonts/arialn.ttf',
+        '/usr/share/fonts/truetype/ubuntu/UbuntuCondensed-Regular.ttf',
+    ]
+    pool = []
+    for fp in candidates:
+        try:
+            pool.append(ImageFont.truetype(fp, size))
+        except Exception:
+            continue
+    if not pool:
+        print("WARNING: No TrueType fonts found. Using default bitmap font.")
+        pool.append(ImageFont.load_default())
+    else:
+        print(f"  ✓ Font pool loaded: {len(pool)} font(s) available")
+    return pool
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+#  IMAGE RENDERER
+# ─────────────────────────────────────────────────────────────────────────────
+
+def render_text_image(text: str, font, width=IMG_WIDTH, height=IMG_HEIGHT,
+                      handwriting=False) -> Image.Image:
+    """
+    Render text on a white background, centered.
+    handwriting=True applies handwriting-style augmentations.
+    """
+    img  = Image.new('RGB', (width, height), color=(255, 255, 255))
+    draw = ImageDraw.Draw(img)
+
+    bbox = draw.textbbox((0, 0), text, font=font)
+    tw   = bbox[2] - bbox[0]
+    th   = bbox[3] - bbox[1]
+    x    = max(4, (width  - tw) // 2)
+    y    = max(4, (height - th) // 2)
+
+    if not handwriting:
+        # ── PRINTED mode ──────────────────────────────────────
+        shade = random.randint(0, 40)
+        draw.text((x, y), text, fill=(shade, shade, shade), font=font)
+
+    else:
+        # ── HANDWRITING simulation mode ───────────────────────
+        # 1. Pen color — blue-black ballpen
+        r = random.randint(0, 60)
+        g = random.randint(0, 60)
+        b = random.randint(0, 120)
+        ink_color = (r, g, b)
+
+        # 2. Per-character y-wobble (unsteady hand)
+        if random.choice([True, False]) and len(text) > 1:
+            char_x = x
+            for ch in text:
+                y_offset = random.randint(-2, 2)
+                draw.text((char_x, y + y_offset), ch, fill=ink_color, font=font)
+                ch_bbox  = draw.textbbox((0, 0), ch, font=font)
+                char_x  += (ch_bbox[2] - ch_bbox[0]) + random.randint(-1, 1)
+        else:
+            draw.text((x, y), text, fill=ink_color, font=font)
+
+        # 3. Pixel-level augmentation
+        arr = np.array(img).astype(np.float32)
+
+        # 4. Ink bleed
+        if random.random() < 0.5:
+            img_pil = Image.fromarray(arr.astype(np.uint8))
+            img_pil = img_pil.filter(
+                ImageFilter.GaussianBlur(radius=random.uniform(0.3, 0.7)))
+            arr = np.array(img_pil).astype(np.float32)
+
+        # 5. Paper texture noise
+        noise_map = np.random.normal(0, random.uniform(3, 10), arr.shape)
+        arr = np.clip(arr + noise_map, 0, 255)
+
+        # 6. Scan shadow patch
+        if random.random() < 0.3:
+            patch_x = random.randint(0, width - 20)
+            patch_w = random.randint(10, 60)
+            arr[:, patch_x:patch_x + patch_w] *= random.uniform(0.88, 0.97)
+            arr = np.clip(arr, 0, 255)
+
+        img = Image.fromarray(arr.astype(np.uint8))
+
+        # 7. Pen tilt rotation (+-3 degrees)
+        if random.random() < 0.6:
+            angle = random.uniform(-3, 3)
+            img   = img.rotate(angle, fillcolor=(255, 255, 255), expand=False)
+
+    return img
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+#  NAME / DATA POOLS
+# ─────────────────────────────────────────────────────────────────────────────
+
+# Populated at runtime from ph_names.json via load_ph_names()
+MIDDLE_NAMES = []
+
+SUFFIXES = ['Jr.', 'Sr.', 'II', 'III', '']
+
+MONTHS = [
+    'January', 'February', 'March', 'April', 'May', 'June',
+    'July', 'August', 'September', 'October', 'November', 'December',
+]
+
+CITIES = [
+    # NCR
+    'Manila', 'Quezon City', 'Caloocan', 'Pasig', 'Makati',
+    'Taguig', 'Paranaque', 'Pasay', 'Las Pinas', 'Muntinlupa',
+    'Marikina', 'Valenzuela', 'Malabon', 'Navotas', 'Mandaluyong',
+    'San Juan', 'Pateros',
+    # Luzon
+    'Tarlac City', 'Angeles City', 'San Fernando', 'Olongapo',
+    'Cabanatuan', 'San Jose del Monte', 'Bacoor', 'Imus', 'Dasmarinas',
+    'Antipolo', 'Binangonan', 'Taytay', 'Santa Rosa', 'Calamba',
+    'San Pablo', 'Lucena', 'Batangas City', 'Lipa', 'Naga City',
+    'Legazpi', 'Sorsogon City', 'Tuguegarao', 'Ilagan', 'Santiago City',
+    'Cauayan', 'San Fernando (La Union)', 'Vigan', 'Laoag',
+    'Dagupan', 'San Carlos', 'Urdaneta', 'Baguio City',
+    # Visayas
+    'Cebu City', 'Mandaue', 'Lapu-Lapu', 'Talisay', 'Danao',
+    'Toledo', 'Carcar', 'Bacolod', 'Bago', 'Sagay', 'Victorias',
+    'Iloilo City', 'Passi', 'Roxas City', 'Kalibo',
+    'Tacloban', 'Ormoc', 'Palo', 'Catbalogan', 'Calbayog',
+    'Tagbilaran', 'Dumaguete', 'Tanjay', 'Bayawan', 'Kabankalan',
+    # Mindanao
+    'Davao City', 'Tagum', 'Panabo', 'Digos', 'Mati',
+    'General Santos', 'Koronadal', 'Kidapawan', 'Cotabato City',
+    'Cagayan de Oro', 'Iligan', 'Ozamiz', 'Oroquieta', 'Tangub',
+    'Butuan', 'Cabadbaran', 'Surigao City', 'Bislig', 'Bayugan',
+    'Zamboanga City', 'Pagadian', 'Dipolog', 'Dapitan',
+    'Marawi', 'Malaybalay', 'Valencia',
+]
+
+PROVINCES = [
+    # Luzon
+    'Tarlac', 'Pampanga', 'Bulacan', 'Nueva Ecija', 'Bataan',
+    'Zambales', 'Aurora', 'Rizal', 'Cavite', 'Laguna',
+    'Batangas', 'Quezon', 'Marinduque', 'Occidental Mindoro',
+    'Oriental Mindoro', 'Palawan', 'Romblon',
+    'Camarines Norte', 'Camarines Sur', 'Albay', 'Sorsogon',
+    'Catanduanes', 'Masbate',
+    'Pangasinan', 'La Union', 'Benguet', 'Ifugao', 'Mountain Province',
+    'Kalinga', 'Apayao', 'Abra', 'Ilocos Norte', 'Ilocos Sur',
+    'Cagayan', 'Isabela', 'Nueva Vizcaya', 'Quirino',
+    'Metro Manila',
+    # Visayas
+    'Cebu', 'Bohol', 'Negros Oriental', 'Siquijor',
+    'Negros Occidental', 'Iloilo', 'Capiz', 'Aklan', 'Antique',
+    'Guimaras', 'Leyte', 'Southern Leyte', 'Samar', 'Eastern Samar',
+    'Northern Samar', 'Biliran',
+    # Mindanao
+    'Davao del Sur', 'Davao del Norte', 'Davao Oriental',
+    'Davao Occidental', 'Davao de Oro',
+    'South Cotabato', 'Sarangani', 'Sultan Kudarat', 'North Cotabato',
+    'Misamis Oriental', 'Misamis Occidental', 'Camiguin',
+    'Bukidnon', 'Lanao del Norte', 'Lanao del Sur',
+    'Maguindanao', 'Basilan', 'Sulu', 'Tawi-Tawi',
+    'Zamboanga del Sur', 'Zamboanga del Norte', 'Zamboanga Sibugay',
+    'Agusan del Norte', 'Agusan del Sur', 'Surigao del Norte',
+    'Surigao del Sur', 'Dinagat Islands',
+]
+
+BARANGAYS = [
+    'Brgy. San Jose', 'Brgy. Sta. Maria', 'Brgy. San Antonio',
+    'Brgy. Santo Nino', 'Brgy. Poblacion', 'Brgy. San Isidro',
+    'Brgy. San Pedro', 'Brgy. San Miguel', 'Brgy. Mabini',
+    'Brgy. Rizal', 'Brgy. Magsaysay', 'Brgy. Quezon',
+    'Brgy. Bagong Silang', 'Brgy. Bagumbayan', 'Brgy. Batasan Hills',
+    'Brgy. Commonwealth', 'Brgy. Culiat', 'Brgy. Fairview',
+    'Brgy. Holy Spirit', 'Brgy. Kamuning', 'Brgy. Laging Handa',
+    'Brgy. Malaya', 'Brgy. Masagana', 'Brgy. Pinyahan',
+    'Brgy. Roxas', 'Brgy. Sacred Heart', 'Brgy. San Roque',
+    'Brgy. Santa Cruz', 'Brgy. Santa Teresita', 'Brgy. Santo Domingo',
+    'Brgy. Silangan', 'Brgy. South Triangle', 'Brgy. Tagumpay',
+    'Brgy. Tandang Sora', 'Brgy. Vasra', 'Brgy. White Plains',
+]
+
+STREETS = [
+    'Mabini St.', 'Rizal Ave.', 'MacArthur Hwy.', 'Quezon Blvd.',
+    'Gen. Luna St.', 'Bonifacio St.', 'Aguinaldo St.', 'Burgos St.',
+    'Del Pilar St.', 'Gomez St.', 'Jacinto St.', 'Lapu-Lapu St.',
+    'Lopez Jaena St.', 'Luna St.', 'Osmena Blvd.', 'Padre Faura St.',
+    'Palma St.', 'Plaridel St.', 'Recto Ave.', 'Roxas Blvd.',
+    'San Andres St.', 'Shaw Blvd.', 'Taft Ave.', 'Tandang Sora Ave.',
+    'Timog Ave.', 'Tuazon Blvd.', 'Visayas Ave.', 'Aurora Blvd.',
+    'EDSA', 'Espana Blvd.', 'Katipunan Ave.', 'Marcos Hwy.',
+    'Ortigas Ave.', 'Quirino Ave.',
+]
+
+RELIGIONS = [
+    'Roman Catholic', 'Catholic', 'Islam', 'Muslim',
+    'Iglesia ni Cristo', 'INC', 'Baptist', 'Methodist',
+    'Seventh Day Adventist', 'Born Again Christian', 'Aglipayan',
+]
+
+OCCUPATIONS = [
+    'Farmer', 'Teacher', 'Engineer', 'Nurse', 'Doctor',
+    'Laborer', 'Housewife', 'Driver', 'Carpenter', 'Vendor',
+    'Student', 'OFW', 'Fisherman', 'Mechanic', 'Electrician',
+    'Police Officer', 'Military', 'Government Employee',
+    'Business Owner', 'Retired',
+]
+
+CIVIL_STATUSES = ['Single', 'Married', 'Widowed', 'Legally Separated']
+
+CITIZENSHIPS = ['Filipino', 'Filipino', 'Filipino', 'American',
+                'Chinese', 'Japanese', 'Korean']
+
+DEATH_CAUSES = [
+    'Cardio-Respiratory Arrest', 'Hypertensive Cardiovascular Disease',
+    'Acute Myocardial Infarction', 'Cerebrovascular Accident',
+    'Pneumonia', 'Septicemia', 'Renal Failure', 'Diabetes Mellitus',
+    'Pulmonary Tuberculosis', 'Cancer of the Lung',
+    'Chronic Obstructive Pulmonary Disease', 'Liver Cirrhosis',
+    'Dengue Hemorrhagic Fever', 'Acute Gastroenteritis',
+    'Congestive Heart Failure',
+]
+
+ATTENDANT_TYPES = [
+    'Private Physician', 'Public Health Officer',
+    'Hospital Authority', 'Hilot', 'None',
+]
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+#  NAME LOADER
+# ─────────────────────────────────────────────────────────────────────────────
+
+def load_ph_names():
+    """
+    Load Filipino names from ph_names.json.
+    Returns (first_names, last_names, middle_names).
+    Falls back to built-in lists if JSON not found.
+    """
+    if os.path.exists(PH_NAMES_FILE):
+        with open(PH_NAMES_FILE, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+        first_names  = data['first_names']['all']
+        last_names   = data['last_names']
+        # Load middle_names from JSON (added by updated generate_ph_names.py)
+        # Falls back to last_names if key missing (older ph_names.json)
+        middle_names = data.get('middle_names', last_names)
+        print(f"  Loaded ph_names.json: "
+              f"{len(first_names)} first, "
+              f"{len(last_names)} last, "
+              f"{len(middle_names)} middle names")
+    else:
+        print(f"  WARNING: {PH_NAMES_FILE} not found.")
+        print(f"  Using built-in fallback names.")
+        print(f"  For better results run: python generate_ph_names.py first.")
+        first_names = [
+            'Juan', 'Maria', 'Jose', 'Ana', 'Pedro', 'Rosa', 'Carlos',
+            'Elena', 'Ramon', 'Lucia', 'Eduardo', 'Carmen', 'Antonio',
+            'Isabel', 'Francisco', 'Gloria', 'Roberto', 'Corazon',
+            'Ricardo', 'Remedios', 'Manuel', 'Teresita', 'Andres',
+            'Lourdes', 'Fernando', 'Maricel', 'Rolando', 'Rowena',
+            'Danilo', 'Cristina', 'Ernesto', 'Marilou', 'Renato',
+            'Felicidad', 'Alfredo', 'Natividad', 'Domingo', 'Milagros',
+        ]
+        last_names = [
+            'Santos', 'Reyes', 'Cruz', 'Bautista', 'Ocampo', 'Garcia',
+            'Mendoza', 'Torres', 'Flores', 'Aquino', 'Dela Cruz',
+            'Del Rosario', 'San Jose', 'De Guzman', 'Villanueva',
+            'Gonzales', 'Ramos', 'Diaz', 'Castro', 'Morales',
+            'Lim', 'Tan', 'Go', 'Chua', 'Sy', 'Ong',
+            'Macaraeg', 'Pascual', 'Buenaventura', 'Concepcion',
+            'Manalo', 'Soriano', 'Evangelista', 'Salazar', 'Tolentino',
+        ]
+        middle_names = last_names
+    return first_names, last_names, middle_names
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+#  TEXT GENERATORS
+# ─────────────────────────────────────────────────────────────────────────────
+
+def gen_full_name(first_names, last_names, with_suffix=True):
+    first  = random.choice(first_names)
+    middle = random.choice(MIDDLE_NAMES) if MIDDLE_NAMES else random.choice(last_names)
+    last   = random.choice(last_names)
+    suffix = random.choice(SUFFIXES) if with_suffix else ''
+    name   = f"{first} {middle} {last}"
+    if suffix:
+        name += f" {suffix}"
+    return name
+
+
+def gen_first_name(first_names):
+    return random.choice(first_names)
+
+
+def gen_last_name(last_names):
+    return random.choice(last_names)
+
+
+def gen_middle_name(last_names):
+    # Always draw from MIDDLE_NAMES (700+ entries from ph_names.json)
+    pool = MIDDLE_NAMES if MIDDLE_NAMES else last_names
+    return random.choice(pool)
+
+
+def gen_date_slash():
+    month = random.randint(1, 12)
+    day   = random.randint(1, 28)
+    year  = random.randint(1930, 2024)
+    return f"{month:02d}/{day:02d}/{year}"
+
+
+def gen_date_long():
+    month = random.choice(MONTHS)
+    day   = random.randint(1, 28)
+    year  = random.randint(1930, 2024)
+    return f"{month} {day}, {year}"
+
+
+def gen_date_day():
+    return str(random.randint(1, 28))
+
+
+def gen_date_month():
+    return random.choice(MONTHS)
+
+
+def gen_date_year():
+    return str(random.randint(1930, 2024))
+
+
+def gen_age():
+    return str(random.randint(1, 95))
+
+
+def gen_place_full():
+    return (f"{random.choice(BARANGAYS)}, "
+            f"{random.choice(CITIES)}, "
+            f"{random.choice(PROVINCES)}")
+
+
+def gen_place_city():
+    return random.choice(CITIES)
+
+
+def gen_place_province():
+    return random.choice(PROVINCES)
+
+
+def gen_address():
+    num = random.randint(1, 999)
+    st  = random.choice(STREETS)
+    return f"{num} {st}, {random.choice(CITIES)}"
+
+
+def gen_registry_no():
+    year = random.randint(2000, 2024)
+    seq  = random.randint(1, 9999)
+    return f"{year}-{seq:04d}"
+
+
+def gen_sex():
+    return random.choice(['Male', 'Female'])
+
+
+def gen_religion():
+    return random.choice(RELIGIONS)
+
+
+def gen_occupation():
+    return random.choice(OCCUPATIONS)
+
+
+def gen_civil_status():
+    return random.choice(CIVIL_STATUSES)
+
+
+def gen_citizenship():
+    return random.choice(CITIZENSHIPS)
+
+
+def gen_weight():
+    return f"{random.randint(1500, 4500)} grams"
+
+
+def gen_death_cause():
+    return random.choice(DEATH_CAUSES)
+
+
+def gen_attendant():
+    return random.choice(ATTENDANT_TYPES)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+#  FORM FIELD DEFINITIONS
+# ─────────────���───────────────────────────────────────────────────────────────
+
+def get_form_fields(form_type, first_names, last_names):
+    fn = first_names
+    ln = last_names
+
+    if form_type == 'form1a':   # Birth Certificate
+        return [
+            ('province',                  lambda: gen_place_province()),
+            ('registry_no',               lambda: gen_registry_no()),
+            ('city_municipality',         lambda: gen_place_city()),
+            ('child_first_name',          lambda: gen_first_name(fn)),
+            ('child_middle_name',         lambda: gen_middle_name(ln)),
+            ('child_last_name',           lambda: gen_last_name(ln)),
+            ('sex',                       lambda: gen_sex()),
+            ('dob_day',                   lambda: gen_date_day()),
+            ('dob_month',                 lambda: gen_date_month()),
+            ('dob_year',                  lambda: gen_date_year()),
+            ('place_birth_hospital',      lambda: f"Ospital ng {gen_place_city()}"),
+            ('place_birth_city',          lambda: gen_place_city()),
+            ('place_birth_province',      lambda: gen_place_province()),
+            ('weight_at_birth',           lambda: gen_weight()),
+            ('type_of_birth',             lambda: random.choice(['Single', 'Twin', 'Triplet'])),
+            ('mother_first_name',         lambda: gen_first_name(fn)),
+            ('mother_middle_name',        lambda: gen_middle_name(ln)),
+            ('mother_last_name',          lambda: gen_last_name(ln)),
+            ('mother_citizenship',        lambda: gen_citizenship()),
+            ('mother_religion',           lambda: gen_religion()),
+            ('mother_occupation',         lambda: gen_occupation()),
+            ('mother_age_at_birth',       lambda: str(random.randint(16, 45))),
+            ('mother_residence_house',    lambda: gen_address()),
+            ('mother_residence_city',     lambda: gen_place_city()),
+            ('mother_residence_province', lambda: gen_place_province()),
+            ('father_first_name',         lambda: gen_first_name(fn)),
+            ('father_middle_name',        lambda: gen_middle_name(ln)),
+            ('father_last_name',          lambda: gen_last_name(ln)),
+            ('father_citizenship',        lambda: gen_citizenship()),
+            ('father_religion',           lambda: gen_religion()),
+            ('father_occupation',         lambda: gen_occupation()),
+            ('father_age_at_birth',       lambda: str(random.randint(18, 55))),
+            ('parents_marriage_month',    lambda: gen_date_month()),
+            ('parents_marriage_day',      lambda: gen_date_day()),
+            ('parents_marriage_year',     lambda: gen_date_year()),
+            ('parents_marriage_city',     lambda: gen_place_city()),
+            ('informant_name',            lambda: gen_full_name(fn, ln, False)),
+            ('informant_address',         lambda: gen_address()),
+            ('informant_date',            lambda: gen_date_slash()),
+        ]
+
+    elif form_type == 'form2a':   # Death Certificate
+        return [
+            ('province',               lambda: gen_place_province()),
+            ('registry_no',            lambda: gen_registry_no()),
+            ('city_municipality',      lambda: gen_place_city()),
+            ('deceased_first_name',    lambda: gen_first_name(fn)),
+            ('deceased_middle_name',   lambda: gen_middle_name(ln)),
+            ('deceased_last_name',     lambda: gen_last_name(ln)),
+            ('sex',                    lambda: gen_sex()),
+            ('religion',               lambda: gen_religion()),
+            ('age_years',              lambda: gen_age()),
+            ('place_death_full',       lambda: f"{gen_place_city()}, {gen_place_province()}"),
+            ('dod_day',                lambda: gen_date_day()),
+            ('dod_month',              lambda: gen_date_month()),
+            ('dod_year',               lambda: gen_date_year()),
+            ('citizenship',            lambda: gen_citizenship()),
+            ('residence_full',         lambda: gen_address()),
+            ('civil_status',           lambda: gen_civil_status()),
+            ('occupation',             lambda: gen_occupation()),
+            ('cause_immediate',        lambda: gen_death_cause()),
+            ('cause_antecedent',       lambda: gen_death_cause()),
+            ('cause_underlying',       lambda: gen_death_cause()),
+            ('cause_other',            lambda: gen_death_cause()),
+            ('informant_name',         lambda: gen_full_name(fn, ln, False)),
+            ('informant_address',      lambda: gen_address()),
+            ('informant_date',         lambda: gen_date_slash()),
+        ]
+
+    elif form_type == 'form3a':   # Marriage Certificate
+        return [
+            ('province',                      lambda: gen_place_province()),
+            ('city_municipality',             lambda: gen_place_city()),
+            ('registry_no',                   lambda: gen_registry_no()),
+            ('husband_first_name',            lambda: gen_first_name(fn)),
+            ('husband_middle_name',           lambda: gen_middle_name(ln)),
+            ('husband_last_name',             lambda: gen_last_name(ln)),
+            ('wife_first_name',               lambda: gen_first_name(fn)),
+            ('wife_middle_name',              lambda: gen_middle_name(ln)),
+            ('wife_last_name',                lambda: gen_last_name(ln)),
+            ('husband_dob_day',               lambda: gen_date_day()),
+            ('husband_dob_month',             lambda: gen_date_month()),
+            ('husband_dob_year',              lambda: gen_date_year()),
+            ('husband_age',                   lambda: gen_age()),
+            ('wife_dob_day',                  lambda: gen_date_day()),
+            ('wife_dob_month',                lambda: gen_date_month()),
+            ('wife_dob_year',                 lambda: gen_date_year()),
+            ('wife_age',                      lambda: gen_age()),
+            ('husband_place_birth_city',      lambda: gen_place_city()),
+            ('husband_place_birth_province',  lambda: gen_place_province()),
+            ('wife_place_birth_city',         lambda: gen_place_city()),
+            ('wife_place_birth_province',     lambda: gen_place_province()),
+            ('husband_citizenship',           lambda: gen_citizenship()),
+            ('wife_citizenship',              lambda: gen_citizenship()),
+            ('husband_religion',              lambda: gen_religion()),
+            ('wife_religion',                 lambda: gen_religion()),
+            ('husband_civil_status',          lambda: gen_civil_status()),
+            ('wife_civil_status',             lambda: gen_civil_status()),
+            ('husband_father_first',          lambda: gen_first_name(fn)),
+            ('husband_father_last',           lambda: gen_last_name(ln)),
+            ('wife_father_first',             lambda: gen_first_name(fn)),
+            ('wife_father_last',              lambda: gen_last_name(ln)),
+            ('husband_mother_first',          lambda: gen_first_name(fn)),
+            ('husband_mother_last',           lambda: gen_last_name(ln)),
+            ('wife_mother_first',             lambda: gen_first_name(fn)),
+            ('wife_mother_last',              lambda: gen_last_name(ln)),
+            ('place_marriage_city',           lambda: gen_place_city()),
+            ('place_marriage_province',       lambda: gen_place_province()),
+            ('date_marriage_day',             lambda: gen_date_day()),
+            ('date_marriage_month',           lambda: gen_date_month()),
+            ('date_marriage_year',            lambda: gen_date_year()),
+        ]
+
+    elif form_type == 'form90':   # Marriage License Application
+        return [
+            ('province',               lambda: gen_place_province()),
+            ('city_municipality',      lambda: gen_place_city()),
+            ('registry_no',            lambda: gen_registry_no()),
+            ('husband_first_name',     lambda: gen_first_name(fn)),
+            ('husband_middle_name',    lambda: gen_middle_name(ln)),
+            ('husband_last_name',      lambda: gen_last_name(ln)),
+            ('wife_first_name',        lambda: gen_first_name(fn)),
+            ('wife_middle_name',       lambda: gen_middle_name(ln)),
+            ('wife_last_name',         lambda: gen_last_name(ln)),
+            ('husband_age',            lambda: gen_age()),
+            ('wife_age',               lambda: gen_age()),
+            ('husband_citizenship',    lambda: gen_citizenship()),
+            ('wife_citizenship',       lambda: gen_citizenship()),
+            ('husband_residence',      lambda: gen_address()),
+            ('wife_residence',         lambda: gen_address()),
+            ('application_date',       lambda: gen_date_slash()),
+        ]
+
+    return []
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+#  MAIN GENERATOR
+# ─────────────────────────────────────────────────────────────────────────────
+
+def generate_dataset():
+    print("=" * 65)
+    print("  fix_data.py — Synthetic Training Data Generator")
+    print("=" * 65)
+
+    # Load Filipino names
+    print("\n[1/4] Loading Filipino names...")
+    first_names, last_names, middle_names = load_ph_names()
+
+    # Populate global MIDDLE_NAMES so all generators use the full 700+ pool
+    global MIDDLE_NAMES
+    MIDDLE_NAMES.clear()
+    MIDDLE_NAMES.extend(middle_names)
+    print(f"  Middle names pool active: {len(MIDDLE_NAMES)} entries")
+
+    # Create output directories
+    print("\n[2/4] Creating output directories...")
+    for split in ['train', 'val']:
+        for form in ['form1a', 'form2a', 'form3a', 'form90']:
+            Path(f'data/{split}/{form}').mkdir(parents=True, exist_ok=True)
+    print("  ✓ Directories ready")
+
+    # Load font pool — multiple typefaces so model generalises across fonts
+    print("\n[3/4] Loading fonts...")
+    font_pool = load_font_pool(FONT_SIZE)
+    print(f"  ✓ {len(font_pool)} font(s) loaded")
+
+    # Generate images
+    print("\n[4/4] Generating images...")
+    print(f"  {'Form':<10} {'Total':>7} {'Train':>7} {'Val':>7}")
+    print(f"  {'-'*35}")
+
+    train_annotations = []
+    val_annotations   = []
+    total_generated   = 0
+
+    for form_type, n_samples in SAMPLES_PER_FORM.items():
+        fields            = get_form_fields(form_type, first_names, last_names)
+        samples_per_field = max(1, n_samples // len(fields))
+        form_train        = 0
+        form_val          = 0
+
+        # Pre-build shuffled val assignment for unbiased 10% split
+        total_this_form = samples_per_field * len(fields)
+        val_flags       = [False] * total_this_form
+        val_indices     = random.sample(
+            range(total_this_form),
+            max(1, int(total_this_form * VAL_SPLIT))
+        )
+        for vi in val_indices:
+            val_flags[vi] = True
+
+        img_idx = 0
+        for field_name, generator in fields:
+            for _ in range(samples_per_field):
+                text = generator()
+                if not text or not text.strip():
+                    img_idx += 1
+                    continue
+
+                # 70% handwriting / 30% printed
+                use_handwriting = random.random() < 0.70
+                # Pick a random font from the pool each image — forces
+                # the model to generalise across typefaces, not memorise one font
+                font = random.choice(font_pool)
+                img   = render_text_image(text, font, handwriting=use_handwriting)
+                fname = f"{field_name}_{img_idx:06d}.jpg"
+
+                is_val = val_flags[img_idx] if img_idx < len(val_flags) else False
+
+                if is_val:
+                    out_path = f"data/val/{form_type}/{fname}"
+                    val_annotations.append({
+                        'image_path': f"{form_type}/{fname}",
+                        'text':       text,
+                    })
+                    form_val += 1
+                else:
+                    out_path = f"data/train/{form_type}/{fname}"
+                    train_annotations.append({
+                        'image_path': f"{form_type}/{fname}",
+                        'text':       text,
+                    })
+                    form_train += 1
+
+                img.save(out_path, quality=95)
+                img_idx += 1
+
+        total_generated += form_train + form_val
+        print(f"  {form_type:<10} {form_train + form_val:>7,} "
+              f"{form_train:>7,} {form_val:>7,}")
+
+    # Save annotation files
+    with open('data/train_annotations.json', 'w', encoding='utf-8') as f:
+        json.dump(train_annotations, f, indent=2, ensure_ascii=False)
+
+    with open('data/val_annotations.json', 'w', encoding='utf-8') as f:
+        json.dump(val_annotations, f, indent=2, ensure_ascii=False)
+
+    # Summary
+    print(f"\n{'=' * 65}")
+    print(f"  DONE!")
+    print(f"{'=' * 65}")
+    print(f"  Total images generated : {total_generated:,}")
+    print(f"  Train images           : {len(train_annotations):,}")
+    print(f"  Val images             : {len(val_annotations):,}")
+    print(f"\n  Saved:")
+    print(f"    data/train_annotations.json  ({len(train_annotations)} entries)")
+    print(f"    data/val_annotations.json    ({len(val_annotations)} entries)")
+    print(f"\n  Next step: python train.py")
+    print(f"{'=' * 65}")
+
+
+if __name__ == '__main__':
+    generate_dataset()

generate_dummy_forms.py

@@ -0,0 +1,375 @@
+"""
+generate_dummy_forms.py
+=======================
+Generates dummy-filled civil registry forms by overlaying
+handwritten-style text onto the ACTUAL blank PDF form templates.
+
+Coordinates measured directly from grid images (200 DPI render).
+  Form 102: 1700 x 2800 px
+  Form 103: 1700 x 2600 px
+  Form  97: 1700 x 2600 px
+  Form  90: 1700 x 2600 px
+
+Usage:
+    python generate_dummy_forms.py
+"""
+
+import os, random
+import fitz
+from PIL import Image, ImageDraw, ImageFont
+
+OUT_DIR   = "dummy_forms"
+FORMS_DIR = "CRNN+CTC"
+os.makedirs(OUT_DIR, exist_ok=True)
+
+PDF_102 = os.path.join(FORMS_DIR, "FORM 102 (BIRTH CERTIFICATE).pdf")
+PDF_103 = os.path.join(FORMS_DIR, "FORM 103 (DEATH CERTIFICATE).pdf")
+PDF_97  = os.path.join(FORMS_DIR, "FORM 97 (MARRIAGE CERTIFICATE).pdf")
+PDF_90  = os.path.join(FORMS_DIR, "FORM 90-MARRIAGE-LICENCE-FORM.pdf")
+
+HW_FONT = "C:/Windows/Fonts/Inkfree.ttf"
+
+def get_font(size):
+    try:    return ImageFont.truetype(HW_FONT, size)
+    except: return ImageFont.load_default()
+
+# ── Filipino dummy data ──────────────────────────────────────────────────────
+FM  = ["Juan","Pedro","Jose","Carlos","Roberto","Eduardo","Miguel","Antonio","Ramon","Fernando","Andres","Ricardo"]
+FF  = ["Maria","Rosa","Elena","Luisa","Carmen","Gloria","Lourdes","Felicitas","Conchita","Remedios","Natividad","Cristina"]
+MID = ["dela Cruz","Reyes","Santos","Garcia","Lopez","Mendoza","Torres","Aquino","Bautista","Villanueva","Castro","Ramos"]
+LN  = ["Santos","Reyes","Cruz","Garcia","Mendoza","Torres","Lopez","Ramos","Bautista","Aquino","Villanueva","Castro"]
+CTY = ["Tarlac City","Makati City","Quezon City","Manila","Caloocan","Pasig City","Marikina City","Malabon"]
+PRV = ["Tarlac","Metro Manila","Cavite","Laguna","Bulacan","Pampanga","Rizal","Batangas"]
+HSP = ["Tarlac Provincial Hospital","Ospital ng Maynila","Philippine General Hospital",
+       "Quezon City Medical Center","San Juan de Dios Hospital","Capitol Medical Center"]
+REL = ["Roman Catholic","Iglesia ni Cristo","Protestant","Born Again Christian"]
+OCC = ["Farmer","Teacher","Engineer","Driver","Housewife","Businessman","Carpenter","Nurse","Laborer"]
+CST = ["Single","Married","Widowed"]
+MON = ["January","February","March","April","May","June",
+       "July","August","September","October","November","December"]
+VEN = ["Saint Joseph Parish","City Hall","San Sebastian Cathedral","Sto. Nino Parish","Saint Peter Parish"]
+COD = ["Cardiopulmonary Arrest","Pneumonia","Myocardial Infarction","Renal Failure","Sepsis"]
+
+rm  = lambda: random.choice(FM)
+rf  = lambda: random.choice(FF)
+rmd = lambda: random.choice(MID)
+rln = lambda: random.choice(LN)
+rc  = lambda: random.choice(CTY)
+rp  = lambda: random.choice(PRV)
+rd  = lambda: str(random.randint(1, 28))
+rmo = lambda: random.choice(MON)
+ry  = lambda s=1960, e=2005: str(random.randint(s, e))
+rrn = lambda pfx: f"{random.randint(2020,2025)}-{pfx}-{random.randint(1000,9999):05d}"
+
+# ── Core helpers ─────────────────────────────────────────────────────────────
+def pdf_to_image(pdf_path, dpi=200):
+    doc = fitz.open(pdf_path)
+    mat = fitz.Matrix(dpi/72, dpi/72)
+    pix = doc[0].get_pixmap(matrix=mat)
+    img = Image.frombytes("RGB", [pix.width, pix.height], pix.samples)
+    doc.close()
+    return img
+
+X_OFFSET = 170  # shift all fields right — increase if still too far left
+
+
+def hw(draw, x, y, text, size=20, color="#1a1a6e"):
+    font = get_font(size)
+    ox = random.randint(-1, 1)
+    oy = random.randint(-1, 1)
+    draw.text((x + X_OFFSET + ox, y+oy), str(text), fill=color, font=font)
+
+def save(img, name):
+    path = os.path.join(OUT_DIR, name)
+    img.save(path, dpi=(200, 200))
+    print(f"  Saved: {path}")
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+#  FORM 102 — Certificate of Live Birth  (1700 x 2800)
+# ══════════════════════════════════════════════════════════════════════════════
+def generate_form_102(n):
+    img, draw = pdf_to_image(PDF_102), None
+    draw = ImageDraw.Draw(img)
+
+    def f(x, y, text, size=20): hw(draw, x, y, text, size)
+
+    # ── Header ──────────────────────────────────────────────────
+    f(178, 322, rp())                          # Province
+    f(1255, 322, rrn("BC"))                    # Registry No
+    f(178, 370, rc())                          # City/Municipality
+
+    # ── CHILD ─────────────��─────────────────────────────────────
+    f(205, 438, rm())                          # 1. NAME First
+    f(600, 438, rmd())                         #    Middle
+    f(1060, 438, rln())                        #    Last
+
+    f(205, 495, random.choice(["Male","Female"]))  # 2. SEX
+    f(585, 495, rd())                          # 3. DATE OF BIRTH Day
+    f(742, 495, rmo())                         #    Month
+    f(975, 495, ry(1970,2024))                 #    Year
+
+    f(295, 552, random.choice(HSP))            # 4. PLACE OF BIRTH Hospital
+    f(738, 552, rc())                          #    City
+    f(1120, 552, rp())                         #    Province
+
+    f(205, 603, "Single")                      # 5a. TYPE OF BIRTH
+    f(900, 603, "First")                       # 5c. BIRTH ORDER
+
+    # ── MOTHER ──────────────────────────────────────────────────
+    f(205, 695, rf())                          # 7. MAIDEN NAME First
+    f(600, 695, rmd())                         #    Middle
+    f(1060, 695, rln())                        #    Last
+
+    f(205, 752, "Filipino")                    # 8. CITIZENSHIP
+    f(685, 752, random.choice(REL))            # 9. RELIGION
+
+    f(645, 810, random.choice(OCC))            # 11. OCCUPATION
+    f(1415, 810, str(random.randint(20,50)))   # 12. AGE
+
+    f(295, 870, f"{random.randint(1,999)} Rizal St., Brgy. San Antonio")  # 13. RESIDENCE
+    f(738, 870, rc())
+    f(1120, 870, rp())
+
+    # ── FATHER ──────────────────────────────────────────────────
+    f(205, 985, rm())                          # 14. NAME First
+    f(600, 985, rmd())                         #     Middle
+    f(1060, 985, rln())                        #     Last
+
+    f(205, 1048, "Filipino")                   # 15. CITIZENSHIP
+    f(425, 1048, random.choice(REL))           # 16. RELIGION
+    f(785, 1048, random.choice(OCC))           # 17. OCCUPATION
+    f(1415, 1048, str(random.randint(22,55)))  # 18. AGE
+
+    f(295, 1105, f"{random.randint(1,999)} Mabini St., Brgy. Poblacion")  # 19. RESIDENCE
+    f(738, 1105, rc())
+    f(1120, 1105, rp())
+
+    # ── MARRIAGE OF PARENTS ──────────────────────────────────────
+    f(175, 1215, rmo())                        # 20a. DATE Month
+    f(385, 1215, rd())                         #       Day
+    f(510, 1215, ry(1960,2010))                #       Year
+    f(762, 1215, rc())                         # 20b. PLACE City
+    f(1062, 1215, rp())                        #       Province
+
+    save(img, f"form_102_{n:03d}.png")
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+#  FORM 103 — Certificate of Death  (1700 x 2600)
+# ══════════════════════════════════════════════════════════════════════════════
+def generate_form_103(n):
+    img  = pdf_to_image(PDF_103)
+    draw = ImageDraw.Draw(img)
+
+    def f(x, y, text, size=20): hw(draw, x, y, text, size)
+
+    # ── Header ──────────────────────────────────────────────────
+    f(178, 218, rp())                          # Province
+    f(1255, 218, rrn("DC"))                    # Registry No
+    f(178, 260, rc())                          # City/Municipality
+
+    # ── Row 1 NAME + 2 SEX ──────────────────────────────────────
+    f(178, 335, rm())                          # 1. NAME First
+    f(535, 335, rmd())                         #    Middle
+    f(878, 335, rln())                         #    Last
+    f(1325, 335, random.choice(["Male","Female"]))  # 2. SEX
+
+    # ── Row 3 DATE OF DEATH + 4 DATE OF BIRTH + 5 AGE ───────────
+    f(178, 418, f"{rd()} {rmo()} {ry(2010,2025)}")  # 3. DATE OF DEATH
+    f(618, 418, rd())                          # 4. DATE OF BIRTH Day
+    f(712, 418, rmo())                         #    Month
+    f(830, 418, ry(1930,1990))                 #    Year
+    f(1055, 418, str(random.randint(30,90)))   # 5. AGE
+
+    # ── Row 6 PLACE OF DEATH + 7 CIVIL STATUS ───────────────────
+    f(178, 480, f"{random.choice(HSP)}, {rc()}")    # 6. PLACE OF DEATH
+    f(1255, 480, random.choice(CST))           # 7. CIVIL STATUS
+
+    # ── Row 8 RELIGION + 9 CITIZENSHIP + 10 RESIDENCE ───────────
+    f(178, 548, random.choice(REL))            # 8. RELIGION
+    f(555, 548, "Filipino")                    # 9. CITIZENSHIP
+    f(818, 548, f"{random.randint(1,999)} Rizal St., {rc()}")  # 10. RESIDENCE
+
+    # ── Row 11 OCCUPATION + 12 FATHER + 13 MOTHER ───────────────
+    f(178, 628, random.choice(OCC))            # 11. OCCUPATION
+    f(428, 628, f"{rm()} {rmd()} {rln()}")     # 12. NAME OF FATHER
+    f(1028, 628, f"{rf()} {rmd()} {rln()}")    # 13. MAIDEN NAME OF MOTHER
+
+    # ── 19b CAUSES OF DEATH ──────────────────────────────────────
+    f(368, 905, random.choice(COD))            # Immediate cause
+    f(368, 953, random.choice(["Hypertensive CVD","COPD","Septicemia"]))   # Antecedent
+    f(368, 1003, random.choice(["Hypertension","Diabetes Mellitus","Old Age"]))  # Underlying
+
+    save(img, f"form_103_{n:03d}.png")
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+#  FORM 97 — Certificate of Marriage  (1700 x 2600)
+# ══════════════════════════════════════════════════════════════════════════════
+def generate_form_97(n):
+    img  = pdf_to_image(PDF_97)
+    draw = ImageDraw.Draw(img)
+
+    def f(x, y, text, size=20): hw(draw, x, y, text, size)
+
+    # ── Header ──────────────────────────────────────────────────
+    f(178, 242, rp())                          # Province
+    f(1385, 242, rrn("MC"))                    # Registry No
+    f(178, 282, rc())                          # City/Municipality
+
+    # ── Row 1 NAME ───────────────────────────────────────────────
+    # HUSBAND                                  # WIFE
+    f(178, 372, rm());     f(892, 372, rf())   # First
+    f(178, 410, rmd());    f(892, 410, rmd())  # Middle
+    f(178, 448, rln());    f(892, 448, rln())  # Last
+
+    # ── Row 2a DATE OF BIRTH / 2b AGE ───────────────────────────
+    h_dob_y = ry(1975, 2000)
+    w_dob_y = ry(1975, 2000)
+    h_age   = str(random.randint(18,45))
+    w_age   = str(random.randint(18,45))
+
+    f(178, 493, rd());     f(892, 493, rd())   # Day
+    f(308, 493, rmo());    f(1022, 493, rmo()) # Month
+    f(478, 493, h_dob_y);  f(1188, 493, w_dob_y)  # Year
+    f(635, 493, h_age);    f(1348, 493, w_age) # Age
+
+    # ── Row 3 PLACE OF BIRTH ─────────────────────────────────────
+    f(178, 548, rc());     f(892, 548, rc())   # City
+    f(418, 548, rp());     f(1122, 548, rp())  # Province
+
+    # ── Row 4a SEX / 4b CITIZENSHIP ─────────────────────────────
+    f(178, 618, "Male");   f(892, 618, "Female")
+    f(352, 618, "Filipino"); f(1062, 618, "Filipino")
+
+    # ── Row 5 RESIDENCE ──────────────────────────────────────────
+    f(178, 672, f"{random.randint(1,999)} Rizal St., {rc()}, {rp()}")
+    f(892, 672, f"{random.randint(1,999)} Mabini St., {rc()}, {rp()}")
+
+    # ── Row 6 RELIGION ───────────────────────────────────────────
+    f(178, 768, random.choice(REL));  f(892, 768, random.choice(REL))
+
+    # ── Row 7 CIVIL STATUS ───────────────────────────────────────
+    f(178, 835, "Single");  f(892, 835, "Single")
+
+    # ── Row 8 NAME OF FATHER ─────────────────────────────────────
+    f(178, 902, rm());     f(892, 902, rm())   # First
+    f(368, 902, rmd());    f(1088, 902, rmd()) # Middle
+    f(562, 902, rln());    f(1278, 902, rln()) # Last
+
+    # ── Row 9 CITIZENSHIP (Father) ───────────────────────────────
+    f(178, 985, "Filipino");  f(892, 985, "Filipino")
+
+    # ── Row 10 NAME OF MOTHER ────────────────────────────────────
+    f(178, 1050, rf());    f(892, 1050, rf())  # First
+    f(368, 1050, rmd());   f(1088, 1050, rmd())  # Middle
+    f(562, 1050, rln());   f(1278, 1050, rln())  # Last
+
+    # ── Row 11 CITIZENSHIP (Mother) ──────────────────────────────
+    f(178, 1138, "Filipino");  f(892, 1138, "Filipino")
+
+    # ── Row 15 PLACE OF MARRIAGE ─────────────────────────────────
+    f(222, 1578, random.choice(VEN))           # Office/Church
+    f(698, 1578, rc())                         # City
+    f(1102, 1578, rp())                        # Province
+
+    # ── Row 16 DATE OF MARRIAGE ──────────────────────────────────
+    f(178, 1638, rd())                         # Day
+    f(308, 1638, rmo())                        # Month
+    f(502, 1638, ry(2015,2025))                # Year
+    f(1282, 1638, f"{random.randint(8,11)}:00 {random.choice(['AM','PM'])}")  # 17. TIME
+
+    save(img, f"form_97_{n:03d}.png")
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+#  FORM 90 — Application for Marriage License  (1700 x 2600)
+# ══════════════════════════════════════════════════════════════════════════════
+def generate_form_90(n):
+    img  = pdf_to_image(PDF_90)
+    draw = ImageDraw.Draw(img)
+
+    def f(x, y, text, size=20): hw(draw, x, y, text, size)
+
+    # ── Header ──────────────────────────────────────────────────
+    f(178, 232, rp())                          # Province
+    f(1252, 232, rrn("ML"))                    # Registry No
+    f(178, 272, rc())                          # City/Municipality
+    f(1002, 308, rrn("LN"))                    # Marriage License No
+    f(1002, 348, f"{rmo()} {rd()}, {ry(2023,2025)}")  # Date of Issuance
+
+    # ── 1. NAME OF APPLICANT ─────────────────────────────────────
+    # GROOM (left)                             # BRIDE (right)
+    f(102, 708, rm());     f(872, 708, rf())   # First
+    f(102, 752, rmd());    f(872, 752, rmd())  # Middle
+    f(102, 795, rln());    f(872, 795, rln())  # Last
+
+    # ── 2. DATE OF BIRTH / AGE ───────────────────────────────────
+    f(102, 835, rd());     f(872, 835, rd())   # Day
+    f(228, 835, rmo());    f(998, 835, rmo())  # Month
+    f(388, 835, ry(1980,2005)); f(1158, 835, ry(1980,2005))  # Year
+    f(568, 835, str(random.randint(18,45))); f(1338, 835, str(random.randint(18,45)))  # Age
+
+    # ── 3. PLACE OF BIRTH ────────────────────────────────────────
+    f(102, 882, rc());     f(872, 882, rc())   # City
+    f(288, 882, rp());     f(1058, 882, rp())  # Province
+
+    # ── 4. SEX / CITIZENSHIP ─────────────────────────────────────
+    f(102, 952, "Male");   f(872, 952, "Female")
+    f(268, 952, "Filipino"); f(1038, 952, "Filipino")
+
+    # ── 5. RESIDENCE ─────────────────────────────────────────────
+    f(102, 1015, f"{random.randint(1,999)} Rizal St., {rc()}")
+    f(872, 1015, f"{random.randint(1,999)} Mabini St., {rc()}")
+
+    # ── 6. RELIGION ──────────────────────────────────────────────
+    f(102, 1100, random.choice(REL))
+    f(872, 1100, random.choice(REL))
+
+    # ── 7. CIVIL STATUS ──────────────────────────────────────────
+    f(102, 1175, "Single");  f(872, 1175, "Single")
+
+    # ── 12. NAME OF FATHER ───────────────────────────────────────
+    f(102, 1562, rm());    f(872, 1562, rm())  # First
+    f(272, 1562, rmd());   f(1042, 1562, rmd()) # Middle
+    f(462, 1562, rln());   f(1232, 1562, rln()) # Last
+
+    # ── 13. CITIZENSHIP (Father) ─────────────────────────────────
+    f(102, 1642, "Filipino");  f(872, 1642, "Filipino")
+
+    # ── 15. MAIDEN NAME OF MOTHER ────────────────────────────────
+    f(102, 1762, rf());    f(872, 1762, rf())  # First
+    f(272, 1762, rmd());   f(1042, 1762, rmd()) # Middle
+    f(462, 1762, rln());   f(1232, 1762, rln()) # Last
+
+    # ── 16. CITIZENSHIP (Mother) ─────────────────────────────────
+    f(102, 1842, "Filipino");  f(872, 1842, "Filipino")
+
+    save(img, f"form_90_{n:03d}.png")
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+#  MAIN
+# ══════════════════════════════════════════════════════════════════════════════
+if __name__ == "__main__":
+    COUNT = 1  # ← set to 30 when alignment is confirmed
+
+    missing = [p for p in [PDF_102, PDF_103, PDF_97, PDF_90] if not os.path.exists(p)]
+    if missing:
+        print("ERROR: Missing PDF form files:")
+        for m in missing: print(f"  {m}")
+        exit(1)
+
+    print(f"Generating {COUNT} dummy forms per type ({COUNT*4} total)...")
+    print(f"Output: {os.path.abspath(OUT_DIR)}/\n")
+
+    for i in range(1, COUNT + 1):
+        print(f"[{i}/{COUNT}]")
+        generate_form_102(i)
+        generate_form_103(i)
+        generate_form_97(i)
+        generate_form_90(i)
+
+    print(f"\nDone! {COUNT*4} forms saved to: {os.path.abspath(OUT_DIR)}/")
+    print("\nNext: upload dummy_forms/ to Roboflow and annotate field bounding boxes.")

generate_form_samples.py

@@ -0,0 +1,389 @@
+"""
+generate_form_samples.py
+========================
+Generates thousands of synthetic filled civil registry form images
+using the blank PDF forms + template_matcher.py coordinates.
+
+Each form is filled with random Filipino names/dates in handwriting fonts.
+Crops are saved with labels → ready for CRNN+CTC fine-tuning.
+
+Usage:
+    python generate_form_samples.py
+
+Output:
+    data/train/real_forms/  -- cropped field images
+    data/real_annotations.json  -- labels for fine-tuning
+"""
+
+import os
+import sys
+import json
+import random
+import datetime
+
+from PIL import Image, ImageDraw, ImageFont
+
+# ── Paths ─────────────────────────────────────────────────────
+BASE_DIR   = os.path.dirname(os.path.abspath(__file__))
+ROOT_DIR   = os.path.dirname(BASE_DIR)
+PYTHON_DIR = ROOT_DIR  # template_matcher.py is here
+
+NAMES_FILE  = os.path.join(BASE_DIR, 'data', 'ph_names.json')
+OUT_IMG_DIR = os.path.join(BASE_DIR, 'data', 'train', 'real_forms')
+OUT_ANN     = os.path.join(BASE_DIR, 'data', 'real_annotations.json')
+
+FONTS_DIR = os.path.join(ROOT_DIR, 'test_images', 'handwriting_fonts')
+
+# Only verified-working Google Fonts URLs
+GOOGLE_FONTS = {
+    'Kalam-Regular.ttf':          'https://github.com/google/fonts/raw/main/ofl/kalam/Kalam-Regular.ttf',
+    'Kalam-Bold.ttf':             'https://github.com/google/fonts/raw/main/ofl/kalam/Kalam-Bold.ttf',
+    'Kalam-Light.ttf':            'https://github.com/google/fonts/raw/main/ofl/kalam/Kalam-Light.ttf',
+    'PatrickHand-Regular.ttf':    'https://github.com/google/fonts/raw/main/ofl/patrickhand/PatrickHand-Regular.ttf',
+    'IndieFlower-Regular.ttf':    'https://github.com/google/fonts/raw/main/ofl/indieflower/IndieFlower-Regular.ttf',
+    'Handlee-Regular.ttf':        'https://github.com/google/fonts/raw/main/ofl/handlee/Handlee-Regular.ttf',
+    'GochiHand-Regular.ttf':      'https://github.com/google/fonts/raw/main/ofl/gochihand/GochiHand-Regular.ttf',
+    'ArchitectsDaughter.ttf':     'https://github.com/google/fonts/raw/main/ofl/architectsdaughter/ArchitectsDaughter-Regular.ttf',
+    'ShadowsIntoLight.ttf':       'https://github.com/google/fonts/raw/main/ofl/shadowsintolight/ShadowsIntoLight.ttf',
+    'ShadowsIntoLightTwo.ttf':    'https://github.com/google/fonts/raw/main/ofl/shadowsintolighttwo/ShadowsIntoLightTwo-Regular.ttf',
+    'Kristi-Regular.ttf':         'https://github.com/google/fonts/raw/main/ofl/kristi/Kristi-Regular.ttf',
+    'AmaticSC-Regular.ttf':       'https://github.com/google/fonts/raw/main/ofl/amaticsc/AmaticSC-Regular.ttf',
+    'AmaticSC-Bold.ttf':          'https://github.com/google/fonts/raw/main/ofl/amaticsc/AmaticSC-Bold.ttf',
+    'BadScript-Regular.ttf':      'https://github.com/google/fonts/raw/main/ofl/badscript/BadScript-Regular.ttf',
+    'Sacramento-Regular.ttf':     'https://github.com/google/fonts/raw/main/ofl/sacramento/Sacramento-Regular.ttf',
+    'GreatVibes-Regular.ttf':     'https://github.com/google/fonts/raw/main/ofl/greatvibes/GreatVibes-Regular.ttf',
+    'Allura-Regular.ttf':         'https://github.com/google/fonts/raw/main/ofl/allura/Allura-Regular.ttf',
+    'AlexBrush-Regular.ttf':      'https://github.com/google/fonts/raw/main/ofl/alexbrush/AlexBrush-Regular.ttf',
+    'Parisienne-Regular.ttf':     'https://github.com/google/fonts/raw/main/ofl/parisienne/Parisienne-Regular.ttf',
+    'Tangerine-Regular.ttf':      'https://github.com/google/fonts/raw/main/ofl/tangerine/Tangerine-Regular.ttf',
+    'Tangerine-Bold.ttf':         'https://github.com/google/fonts/raw/main/ofl/tangerine/Tangerine-Bold.ttf',
+    'Courgette-Regular.ttf':      'https://github.com/google/fonts/raw/main/ofl/courgette/Courgette-Regular.ttf',
+    'Niconne-Regular.ttf':        'https://github.com/google/fonts/raw/main/ofl/niconne/Niconne-Regular.ttf',
+    'MarckScript-Regular.ttf':    'https://github.com/google/fonts/raw/main/ofl/marckscript/MarckScript-Regular.ttf',
+    'Norican-Regular.ttf':        'https://github.com/google/fonts/raw/main/ofl/norican/Norican-Regular.ttf',
+    'Damion-Regular.ttf':         'https://github.com/google/fonts/raw/main/ofl/damion/Damion-Regular.ttf',
+    'Satisfy-Regular.ttf':        'https://github.com/google/fonts/raw/main/ofl/satisfy/Satisfy-Regular.ttf',
+    'Pacifico-Regular.ttf':       'https://github.com/google/fonts/raw/main/ofl/pacifico/Pacifico-Regular.ttf',
+    'Italianno-Regular.ttf':      'https://github.com/google/fonts/raw/main/ofl/italianno/Italianno-Regular.ttf',
+    'Pompiere-Regular.ttf':       'https://github.com/google/fonts/raw/main/ofl/pompiere/Pompiere-Regular.ttf',
+}
+
+FONT_PATHS = [
+    # Downloaded handwriting fonts
+    *[os.path.join(FONTS_DIR, name) for name in GOOGLE_FONTS],
+    # Already available
+    os.path.join(ROOT_DIR, 'test_images', 'Caveat-Regular.ttf'),
+    # Windows fallbacks
+    r'C:\Windows\Fonts\segoepr.ttf',
+    r'C:\Windows\Fonts\segoeprb.ttf',
+    r'C:\Windows\Fonts\comic.ttf',
+]
+
+def download_fonts():
+    """Download handwriting fonts from Google Fonts if not present."""
+    import urllib.request
+    os.makedirs(FONTS_DIR, exist_ok=True)
+    ok = 0
+    for fname, url in GOOGLE_FONTS.items():
+        dest = os.path.join(FONTS_DIR, fname)
+        if os.path.exists(dest) and os.path.getsize(dest) > 10000:
+            ok += 1
+            continue
+        try:
+            print(f"  Downloading {fname}...")
+            with urllib.request.urlopen(url, timeout=10) as r, open(dest, 'wb') as f:
+                f.write(r.read())
+            # Validate: real TTF files are > 10KB
+            if os.path.getsize(dest) < 10000:
+                os.remove(dest)
+                print(f"  Skipped {fname} (invalid file)")
+            else:
+                ok += 1
+        except Exception as e:
+            print(f"  Failed {fname}: {e}")
+            if os.path.exists(dest):
+                os.remove(dest)
+    print(f"  {ok} fonts ready")
+
+PDF_FORMS = {
+    '97':  os.path.join(ROOT_DIR, 'python', 'CRNN+CTC', 'FORM 97 (MARRIAGE CERTIFICATE).pdf'),
+    '102': os.path.join(ROOT_DIR, 'python', 'CRNN+CTC', 'FORM 102 (BIRTH CERTIFICATE).pdf'),
+    '103': os.path.join(ROOT_DIR, 'python', 'CRNN+CTC', 'FORM 103 (DEATH CERTIFICATE).pdf'),
+    '90':  os.path.join(ROOT_DIR, 'python', 'CRNN+CTC', 'FORM 90-MARRIAGE-LICENCE-FORM.pdf'),
+}
+
+SAMPLES_PER_FORM = 1000   # forms to generate per type
+IMG_W = 64
+IMG_H = 512
+
+# ── Load TEMPLATES from template_matcher ─────────────────────
+sys.path.insert(0, PYTHON_DIR)
+from template_matcher import TEMPLATES
+
+# ── Load Filipino names ───────────────────────────────────────
+def load_names():
+    if not os.path.exists(NAMES_FILE):
+        print(f"ERROR: {NAMES_FILE} not found. Run generate_ph_names.py first.")
+        sys.exit(1)
+    with open(NAMES_FILE) as f:
+        data = json.load(f)
+    return data
+
+# ── Random data generators ────────────────────────────────────
+MONTHS = ['January','February','March','April','May','June',
+          'July','August','September','October','November','December']
+RELIGIONS = ['Roman Catholic','Islam','Baptist','Iglesia ni Cristo',
+             'Seventh Day Adventist','Born Again Christian']
+CIVIL_STATUSES = ['Single','Married','Widowed','Legally Separated']
+CITIZENSHIPS = ['Filipino','American','Chinese','Japanese']
+PROVINCES = ['Cebu','Davao del Sur','Metro Manila','Iloilo','Pampanga',
+             'Batangas','Laguna','Cavite','Bulacan','Quezon City']
+CITIES = ['Cebu City','Davao City','Manila','Iloilo City','San Fernando',
+          'Batangas City','Santa Rosa','Bacoor','Malolos','Quezon City']
+
+def rand_name(names, key):
+    pool = names.get(key, ['Juan'])
+    return random.choice(pool).upper()
+
+def rand_date():
+    y = random.randint(1950, 2005)
+    m = random.randint(1, 12)
+    d = random.randint(1, 28)
+    return f"{d:02d}", MONTHS[m-1], str(y)
+
+def rand_age():
+    return str(random.randint(18, 80))
+
+def rand_province():
+    return random.choice(PROVINCES).upper()
+
+def rand_city():
+    return random.choice(CITIES).upper()
+
+def rand_religion():
+    return random.choice(RELIGIONS).upper()
+
+def rand_civil_status():
+    return random.choice(CIVIL_STATUSES).upper()
+
+def rand_citizenship():
+    return random.choice(CITIZENSHIPS).upper()
+
+def rand_registry_no():
+    return f"{random.randint(2000,2024)}-{random.randint(1000,9999)}"
+
+def rand_time():
+    h = random.randint(6, 18)
+    m = random.choice(['00','15','30','45'])
+    return f"{h:02d}:{m} {'AM' if h < 12 else 'PM'}"
+
+def generate_field_value(field_name, names):
+    """Generate a plausible random value for a given field name."""
+    f = field_name.lower()
+    if 'province' in f:          return rand_province()
+    if 'registry' in f:          return rand_registry_no()
+    if 'city' in f or 'municipality' in f: return rand_city()
+    if 'first' in f and ('name' in f or 'father' in f or 'mother' in f):
+        return rand_name(names, 'first')
+    if 'middle' in f:            return rand_name(names, 'middle')
+    if 'last' in f:              return rand_name(names, 'last')
+    if '_name' in f and 'father' not in f and 'mother' not in f:
+        return rand_name(names, 'first')
+    if 'father_name' in f or 'mother_name' in f:
+        return f"{rand_name(names,'first')} {rand_name(names,'middle')} {rand_name(names,'last')}"
+    if 'dob_day' in f or 'day' in f:    return rand_date()[0]
+    if 'dob_month' in f or 'month' in f: return rand_date()[1]
+    if 'dob_year' in f or 'year' in f:   return rand_date()[2]
+    if 'dob' in f and 'day' not in f and 'month' not in f and 'year' not in f:
+        d,m,y = rand_date(); return f"{d} {m} {y}"
+    if 'age' in f:               return rand_age()
+    if 'birth' in f and 'place' in f: return rand_city()
+    if 'place_of_birth' in f:    return rand_city()
+    if 'sex' in f:               return random.choice(['MALE','FEMALE'])
+    if 'citizenship' in f:       return rand_citizenship()
+    if 'residence' in f:         return f"{rand_city()}, {rand_province()}"
+    if 'religion' in f:          return rand_religion()
+    if 'civil_status' in f:      return rand_civil_status()
+    if 'place_of_marriage' in f: return rand_city()
+    if 'date_of_marriage' in f:
+        d,m,y = rand_date(); return f"{d} {m} {y}"
+    if 'time_of_marriage' in f:  return rand_time()
+    if 'marriage_date' in f:
+        d,m,y = rand_date(); return f"{d} {m} {y}"
+    if 'marriage_place' in f:    return rand_city()
+    if 'marriage_license' in f:  return rand_registry_no()
+    if 'date_issued' in f:
+        d,m,y = rand_date(); return f"{d} {m} {y}"
+    if 'occupation' in f:        return random.choice(['FARMER','TEACHER','NURSE','ENGINEER','DRIVER','HOUSEWIFE'])
+    if 'type_of_birth' in f:     return random.choice(['SINGLE','TWIN','TRIPLET'])
+    if 'birth_order' in f:       return random.choice(['1ST','2ND','3RD','4TH'])
+    if 'weight' in f:            return f"{random.randint(2,5)}.{random.randint(0,9)} KG"
+    if 'cause' in f:             return random.choice(['CARDIAC ARREST','PNEUMONIA','DIABETES','HYPERTENSION'])
+    if 'father_name' in f:       return f"{rand_name(names,'first')} {rand_name(names,'last')}"
+    if 'mother_name' in f:       return f"{rand_name(names,'first')} {rand_name(names,'last')}"
+    return rand_name(names, 'first')
+
+# ── Load fonts ────────────────────────────────────────────────
+def load_fonts():
+    fonts = []
+    for path in FONT_PATHS:
+        if os.path.exists(path):
+            for size in [14, 16, 18, 20]:
+                try:
+                    fonts.append(ImageFont.truetype(path, size))
+                except:
+                    pass
+    if not fonts:
+        fonts = [ImageFont.load_default()]
+    print(f"  Loaded {len(fonts)} font variants")
+    return fonts
+
+# ── Load blank form image ─────────────────────────────────────
+def load_blank_form(form_type):
+    """Convert PDF to image or use a reference scan as background."""
+    pdf_path = PDF_FORMS.get(form_type)
+
+    # Try pdf2image first
+    if pdf_path and os.path.exists(pdf_path):
+        try:
+            from pdf2image import convert_from_path
+            pages = convert_from_path(pdf_path, dpi=150)
+            if pages:
+                return pages[0].convert('RGB')
+        except Exception as e:
+            print(f"  pdf2image failed: {e}")
+
+    # Fallback: use reference image (try png, jpg, jpeg)
+    for ext in ['png', 'jpg', 'jpeg']:
+        ref_path = os.path.join(ROOT_DIR, 'references', f'reference_{form_type}.{ext}')
+        if os.path.exists(ref_path):
+            return Image.open(ref_path).convert('RGB')
+    # Also try hyphen variant (e.g. reference-90.jpg)
+    for ext in ['png', 'jpg', 'jpeg']:
+        ref_path = os.path.join(ROOT_DIR, 'references', f'reference-{form_type}.{ext}')
+        if os.path.exists(ref_path):
+            return Image.open(ref_path).convert('RGB')
+
+    print(f"  WARNING: No blank form found for {form_type} — skipping")
+    return None
+
+# ── Render text on form ───────────────────────────────────────
+def render_field(draw, x1r, y1r, x2r, y2r, text, img_w, img_h, fonts):
+    """Draw handwritten-style text in a field box."""
+    x1 = int(x1r * img_w)
+    y1 = int(y1r * img_h)
+    x2 = int(x2r * img_w)
+    y2 = int(y2r * img_h)
+
+    box_w = max(x2 - x1, 1)
+    box_h = max(y2 - y1, 1)
+
+    # Pick a font that fits
+    font = random.choice(fonts)
+    for f in fonts:
+        bbox = f.getbbox(text)
+        fw = bbox[2] - bbox[0]
+        fh = bbox[3] - bbox[1]
+        if fw <= box_w * 0.95 and fh <= box_h * 1.2:
+            font = f
+            break
+
+    # Random pen color (dark blue/black like ballpen)
+    r = random.randint(0, 40)
+    g = random.randint(0, 40)
+    b = random.randint(60, 120)
+    color = (r, g, b)
+
+    # Center text vertically in box
+    bbox = font.getbbox(text)
+    fh = bbox[3] - bbox[1]
+    ty = y1 + (box_h - fh) // 2
+
+    # Slight random x offset
+    tx = x1 + random.randint(2, max(3, box_w // 10))
+
+    draw.text((tx, ty), text, fill=color, font=font)
+
+# ── Crop a field ──────────────────────────────────────────────
+def crop_field(img, x1r, y1r, x2r, y2r):
+    w, h = img.size
+    x1 = max(0, int(x1r * w) - 4)
+    y1 = max(0, int(y1r * h) - 4)
+    x2 = min(w, int(x2r * w) + 4)
+    y2 = min(h, int(y2r * h) + 4)
+    return img.crop((x1, y1, x2, y2))
+
+# ── Main ────────────────────────────────────────────────────��─
+def main():
+    print("=" * 60)
+    print("  Form Sample Generator")
+    print("=" * 60)
+
+    os.makedirs(OUT_IMG_DIR, exist_ok=True)
+    print("\n  Downloading handwriting fonts...")
+    download_fonts()
+    names = load_names()
+    fonts = load_fonts()
+    annotations = []
+    total = 0
+
+    for form_type, template in TEMPLATES.items():
+        print(f"\n  Generating Form {form_type}...")
+
+        blank = load_blank_form(form_type)
+        if blank is None:
+            continue
+
+        for i in range(SAMPLES_PER_FORM):
+            # Fresh copy of blank form
+            form_img = blank.copy()
+            draw = ImageDraw.Draw(form_img)
+            img_w, img_h = form_img.size
+
+            field_values = {}
+            for field_name, coords in template.items():
+                x1r, y1r, x2r, y2r, _ = coords
+                text = generate_field_value(field_name, names)
+                field_values[field_name] = text
+                render_field(draw, x1r, y1r, x2r, y2r, text, img_w, img_h, fonts)
+
+            # Save full form preview (first sample only)
+            if i == 0:
+                preview_path = os.path.join(OUT_IMG_DIR, f'form{form_type}_preview.png')
+                form_img.save(preview_path)
+                print(f"    Preview saved: {preview_path}")
+
+            # Crop each field and save
+            for field_name, coords in template.items():
+                x1r, y1r, x2r, y2r, _ = coords
+                crop = crop_field(form_img, x1r, y1r, x2r, y2r)
+                crop = crop.convert('L')  # grayscale
+
+                fname = f"form{form_type}_{i:05d}_{field_name}.png"
+                fpath = os.path.join(OUT_IMG_DIR, fname)
+                crop.save(fpath)
+
+                annotations.append({
+                    "image_path": f"real_forms/{fname}",
+                    "text": field_values[field_name]
+                })
+                total += 1
+
+            if (i + 1) % 100 == 0:
+                print(f"    {i+1}/{SAMPLES_PER_FORM} forms done ({total} crops so far)")
+
+        print(f"  Form {form_type} done.")
+
+    # Save annotations
+    with open(OUT_ANN, 'w') as f:
+        json.dump(annotations, f, indent=2)
+
+    print(f"\n{'='*60}")
+    print(f"  DONE!")
+    print(f"  Total crops : {total}")
+    print(f"  Annotations : {OUT_ANN}")
+    print(f"  Next step   : upload to Kaggle and run fine-tune")
+    print(f"{'='*60}")
+
+if __name__ == '__main__':
+    main()

generate_ph_names.py

@@ -0,0 +1,350 @@
+"""
+generate_ph_names.py
+====================
+Run this file ONCE to extract Filipino names from the
+names-dataset library and save them to data/ph_names.json.
+
+Install first:
+    pip install names-dataset
+
+Usage:
+    python generate_ph_names.py
+
+Output:
+    data/ph_names.json  <-- used by fix_data.py every run
+"""
+
+import json
+import os
+
+print("=" * 60)
+print("  Filipino Name Extractor  |  names-dataset (PyPI)")
+print("=" * 60)
+
+# ── Step 1: Load NameDataset ──────────────────────────────────
+print("\n[1/5] Loading NameDataset...")
+print("      (This takes 30-60 seconds and needs ~3.2 GB RAM)")
+
+try:
+    from names_dataset import NameDataset
+    nd = NameDataset()
+    print("      OK - Dataset loaded!")
+except ImportError:
+    print("\n  ERROR: names-dataset is not installed.")
+    print("  Fix:   pip install names-dataset")
+    exit(1)
+except MemoryError:
+    print("\n  ERROR: Not enough RAM. Need ~3.2 GB free.")
+    exit(1)
+
+# ── Step 2: Extract Filipino FIRST names ─────────────────────
+print("\n[2/5] Extracting Filipino first names (Male + Female)...")
+
+ph_male   = nd.get_top_names(n=300, gender='Male',   country_alpha2='PH')
+ph_female = nd.get_top_names(n=300, gender='Female', country_alpha2='PH')
+
+# API returns: { 'PH': { 'M': [...] } }
+male_first   = ph_male.get('PH',   {}).get('M', [])
+female_first = ph_female.get('PH', {}).get('F', [])
+all_first    = male_first + female_first
+
+print(f"      Male   first names : {len(male_first)}")
+print(f"      Female first names : {len(female_first)}")
+print(f"      Total  first names : {len(all_first)}")
+print(f"      Sample (male)      : {male_first[:5]}")
+print(f"      Sample (female)    : {female_first[:5]}")
+
+# ── Step 3: Extract Filipino LAST names ──────────────────────
+print("\n[3/5] Extracting Filipino last names...")
+
+ph_last_raw = nd.get_top_names(n=300, country_alpha2='PH', use_first_names=False)
+print(f"      Raw last name API type : {type(ph_last_raw)}")
+
+ph_last_ph = ph_last_raw.get('PH', {})
+print(f"      PH entry type          : {type(ph_last_ph)}")
+
+raw_last = []
+
+if isinstance(ph_last_ph, list):
+    raw_last = ph_last_ph
+elif isinstance(ph_last_ph, dict):
+    first_val = next(iter(ph_last_ph.values()), None)
+    if isinstance(first_val, list):
+        for lst in ph_last_ph.values():
+            raw_last.extend(lst)
+    elif isinstance(first_val, dict):
+        raw_last = list(ph_last_ph.keys())
+    else:
+        raw_last = list(ph_last_ph.keys())
+
+# Deduplicate while preserving order
+seen     = set()
+all_last = []
+for name in raw_last:
+    if isinstance(name, str) and name not in seen:
+        seen.add(name)
+        all_last.append(name)
+
+print(f"      Total last names   : {len(all_last)}")
+print(f"      Sample             : {all_last[:5]}")
+
+if len(all_last) == 0:
+    print("\n  WARNING: Could not extract last names from API.")
+    print("  Using common Filipino last names as fallback...")
+    all_last = [
+        'Santos', 'Reyes', 'Cruz', 'Bautista', 'Ocampo',
+        'Garcia', 'Mendoza', 'Torres', 'Flores', 'Aquino',
+        'Dela Cruz', 'Del Rosario', 'San Jose', 'De Guzman',
+        'Villanueva', 'Gonzales', 'Ramos', 'Diaz', 'Castro',
+        'Morales', 'Ortega', 'Gutierrez', 'Lopez', 'Ramirez',
+        'Navarro', 'Aguilar', 'Espinosa', 'Mercado', 'Tolentino',
+        'Lim', 'Tan', 'Go', 'Chua', 'Sy', 'Ong', 'Co',
+        'Macaraeg', 'Macapagal', 'Magsaysay', 'Magno',
+        'Pascual', 'Buenaventura', 'Concepcion', 'Resurreccion',
+        'Ilagan', 'Manalo', 'Soriano', 'Evangelista', 'Salazar',
+    ]
+    print(f"      Fallback last names: {len(all_last)}")
+
+# ── Step 4: Build MIDDLE names pool ──────────────────────────
+# Middle names in Filipino naming convention are the mother's
+# maiden last name. We build a large pool by combining:
+#   A) The last names pool already extracted (primary source)
+#   B) A curated extended list of common Filipino surnames
+#      used specifically as middle names
+print("\n[4/5] Building middle names pool...")
+
+EXTENDED_MIDDLE_NAMES = [
+    # Common Filipino surnames used as middle names
+    'Abad', 'Abaya', 'Abella', 'Ablaza', 'Abrera',
+    'Acosta', 'Adriano', 'Afable', 'Africa', 'Agcaoili',
+    'Agno', 'Agpalo', 'Aguinaldo', 'Agustin', 'Ahorro',
+    'Alano', 'Alba', 'Albano', 'Alberto', 'Alcantara',
+    'Alcazar', 'Alcon', 'Aldana', 'Alegre', 'Alejandro',
+    'Aligaen', 'Alim', 'Alinea', 'Alipio', 'Almario',
+    'Almeda', 'Almendras', 'Alminiana', 'Almodiel', 'Alonto',
+    'Alvarado', 'Alvarez', 'Amante', 'Amaro', 'Ambrocio',
+    'Amor', 'Amores', 'Amparo', 'Anastacio', 'Andal',
+    'Andaya', 'Angeles', 'Angsioco', 'Antiporda', 'Antonio',
+    'Apalisok', 'Apolinario', 'Apostol', 'Aquino', 'Araneta',
+    'Aranas', 'Aranda', 'Arceo', 'Arenas', 'Arias',
+    'Ariate', 'Arillo', 'Arimado', 'Arjona', 'Arlante',
+    'Arnaldo', 'Arnaiz', 'Arnoco', 'Arocena', 'Arroyo',
+    'Asejo', 'Asuncion', 'Austria', 'Avecilla', 'Avena',
+    'Avila', 'Avinante', 'Ayala', 'Azucena', 'Azul',
+    'Bacani', 'Bacunawa', 'Baguio', 'Bagunu', 'Balagtas',
+    'Balangue', 'Balbin', 'Balde', 'Baldeo', 'Balgos',
+    'Balili', 'Balinas', 'Balitaan', 'Balladares', 'Ballesteros',
+    'Balmeo', 'Balmores', 'Banaag', 'Banaag', 'Bandola',
+    'Bangayan', 'Bansil', 'Bansode', 'Bantigue', 'Bantug',
+    'Barbin', 'Barcenas', 'Bareng', 'Barrion', 'Barroga',
+    'Bartolome', 'Bases', 'Batac', 'Bataller', 'Batanes',
+    'Batungbakal', 'Bautista', 'Bayani', 'Bayot', 'Baysic',
+    'Belarmino', 'Beldia', 'Belen', 'Belgica', 'Bello',
+    'Benavides', 'Bendaña', 'Benedicto', 'Benigno', 'Benitez',
+    'Bernardino', 'Bernardo', 'Bernarte', 'Besares', 'Billones',
+    'Binay', 'Binayas', 'Biscocho', 'Blanco', 'Bondoc',
+    'Borja', 'Borromeo', 'Bravo', 'Buenaobra', 'Buenaflor',
+    'Buenafe', 'Buenaseda', 'Buenconsejo', 'Buendia', 'Bugarin',
+    'Bulalacao', 'Bulalacao', 'Bulatao', 'Bumanlag', 'Bunag',
+    'Caballero', 'Cabigting', 'Cabral', 'Cabreros', 'Cacal',
+    'Cagampan', 'Cagas', 'Caguioa', 'Cahilig', 'Cajucom',
+    'Calagos', 'Calamba', 'Calasanz', 'Calatrava', 'Calderon',
+    'Calimag', 'Calimutan', 'Calinawan', 'Calleja', 'Callejo',
+    'Caluag', 'Calugay', 'Camacho', 'Camino', 'Campaner',
+    'Camposano', 'Candelario', 'Canete', 'Caning', 'Canlas',
+    'Caoile', 'Capili', 'Carandang', 'Carbonell', 'Cariaga',
+    'Carino', 'Carunungan', 'Casaje', 'Casas', 'Casidsid',
+    'Castañeda', 'Castillo', 'Castillo', 'Catalan', 'Catapang',
+    'Cayabyab', 'Cayco', 'Celdran', 'Cerillo', 'Cervantes',
+    'Chico', 'Chikiamco', 'Chiongbian', 'Cipriano', 'Clarin',
+    'Claudio', 'Clavecillas', 'Climaco', 'Cobankiat', 'Colambo',
+    'Collado', 'Comafay', 'Comia', 'Concepcion', 'Condino',
+    'Consing', 'Contraras', 'Coquia', 'Cordero', 'Corotan',
+    'Corpus', 'Cosico', 'Costales', 'Crisostomo', 'Cristobal',
+    'Cueto', 'Culala', 'Cunanan', 'Cunanon', 'Curato',
+    'Dadivas', 'Daep', 'Daez', 'Daguplo', 'Dalida',
+    'Dalisay', 'Dalmacion', 'Dalusong', 'Damasco', 'Damo',
+    'Danao', 'Dancel', 'Dandan', 'Danila', 'Daquigan',
+    'Dario', 'Datoc', 'Datumanong', 'David', 'Dayao',
+    'Dayrit', 'De Borja', 'De Castro', 'De Jesus', 'De Jose',
+    'De La Cruz', 'De La Pena', 'De La Rosa', 'De Leon', 'De Lima',
+    'De Los Angeles', 'De Los Reyes', 'De Los Santos', 'De Luna', 'De Mesa',
+    'De Ocampo', 'De Paz', 'De Vera', 'De Villa', 'Delos Reyes',
+    'Demaisip', 'Delos Santos', 'Demillo', 'Demonteverde', 'Denosta',
+    'Derequito', 'Deri', 'Detablan', 'Deveraturda', 'Diaz',
+    'Dichoso', 'Diego', 'Diesto', 'Dimaano', 'Dimabuyu',
+    'Dimagiba', 'Dimaguila', 'Dimaio', 'Dimanlig', 'Dimayuga',
+    'Dingal', 'Dinglasan', 'Dionisio', 'Dioquino', 'Ditan',
+    'Diwata', 'Domingo', 'Dominguez', 'Donato', 'Dorado',
+    'Doria', 'Duallo', 'Duenas', 'Duerme', 'Dulay',
+    'Dumalaog', 'Dumpit', 'Duque', 'Duran', 'Durante',
+    'Ebdane', 'Echavez', 'Echevarria', 'Edralin', 'Ejercito',
+    'Elago', 'Elazegui', 'Elises', 'Elumba', 'Enage',
+    'Encarnacion', 'Enriquez', 'Escobar', 'Escueta', 'Escutin',
+    'Esguerra', 'Eslit', 'Espejo', 'Espeleta', 'Espinas',
+    'Espino', 'Espiritu', 'Estepa', 'Esteves', 'Estrada',
+    'Estrellas', 'Evangelista', 'Evasco', 'Evidente', 'Eyas',
+    'Fabella', 'Fabros', 'Faelnar', 'Fajardo', 'Fajutag',
+    'Famadico', 'Famador', 'Faustino', 'Favila', 'Feliciano',
+    'Felipe', 'Fermin', 'Fernandez', 'Fernando', 'Ferrer',
+    'Figueras', 'Fider', 'Florendo', 'Florentino', 'Floreta',
+    'Flores', 'Florido', 'Floriza', 'Foja', 'Fonacier',
+    'Fontanilla', 'Formoso', 'Fornier', 'Fortich', 'Fortuna',
+    'Francisco', 'Frano', 'Frasco', 'Frias', 'Fuentes',
+    'Gaabucayan', 'Gabutero', 'Gaerlan', 'Gaffud', 'Galapon',
+    'Galera', 'Galicia', 'Galindez', 'Gallardo', 'Gallo',
+    'Galvez', 'Gamalinda', 'Gamboa', 'Gammad', 'Gandionco',
+    'Ganzon', 'Garado', 'Garayblas', 'Garcia', 'Garduce',
+    'Garrido', 'Gatdula', 'Gatmaitan', 'Gatus', 'Gawat',
+    'Gelera', 'Gelua', 'Gemora', 'Genato', 'Generoso',
+    'Gequillana', 'Gerona', 'Gerundio', 'Gianan', 'Gimenez',
+    'Gloria', 'Glorioso', 'Glova', 'Golez', 'Gomez',
+    'Gonzaga', 'Gonzales', 'Gordoncillo', 'Gorre', 'Grafilo',
+    'Gregorio', 'Griño', 'Guanzon', 'Guerrero', 'Guevara',
+    'Guiao', 'Guillen', 'Guinto', 'Guison', 'Gullas',
+    'Gutierrez', 'Guzman', 'Hernandez', 'Herrera', 'Hizon',
+    'Honasan', 'Hontiveros', 'Horca', 'Hufana', 'Humilde',
+    'Ibañez', 'Ignacio', 'Ilustre', 'Imbong', 'Imperial',
+    'Infante', 'Inion', 'Inocentes', 'Inso', 'Iringan',
+    'Jacinto', 'Javier', 'Jimenez', 'Jose', 'Joson',
+    'Juan', 'Juico', 'Jurado', 'Kabigting', 'Kalaw',
+    'Kho', 'Lacaba', 'Lacadin', 'Lacson', 'Ladesma',
+    'Laderas', 'Lagman', 'Lagua', 'Laguna', 'Lainez',
+    'Lajarca', 'Lamayo', 'Lambino', 'Lapid', 'Lapuz',
+    'Lara', 'Largo', 'Lariza', 'Larizal', 'Laserna',
+    'Latorre', 'Laurel', 'Laurente', 'Lazaro', 'Leano',
+    'Legarda', 'Leonor', 'Leynes', 'Libunao', 'Licup',
+    'Lim', 'Limkaichong', 'Limpag', 'Liwanag', 'Llanes',
+    'Llamado', 'Llaneta', 'Locsin', 'Logarta', 'Lopez',
+    'Lorenzo', 'Lorilla', 'Lozada', 'Lucero', 'Luistro',
+    'Luna', 'Luneta', 'Luzon', 'Macalintal', 'Macam',
+    'Maceda', 'Madera', 'Madrazo', 'Magtanggol', 'Malabanan',
+    'Malacaman', 'Malajacan', 'Malanyaon', 'Malaya', 'Malbas',
+    'Malcampo', 'Maldia', 'Maligalig', 'Malinao', 'Malonzo',
+    'Mangahas', 'Mangubat', 'Manigbas', 'Manila', 'Manlangit',
+    'Manlapaz', 'Manlongat', 'Manrique', 'Mansalay', 'Mante',
+    'Manuel', 'Manzano', 'Marcelo', 'Marcos', 'Mariano',
+    'Maristela', 'Marquez', 'Maravilla', 'Masangkay', 'Masapol',
+    'Mateo', 'Matienzo', 'Matining', 'Matugas', 'Maula',
+    'Maulion', 'Mayuga', 'Medina', 'Mejia', 'Melchor',
+    'Melo', 'Menor', 'Mercado', 'Mesina', 'Miguel',
+    'Miralles', 'Miranda', 'Molano', 'Molina', 'Mondejar',
+    'Monreal', 'Montano', 'Montenegro', 'Montero', 'Montes',
+    'Montesa', 'Montoya', 'Moraga', 'Moraleda', 'Moreno',
+    'Morial', 'Muncal', 'Muñoz', 'Murillo', 'Musni',
+    'Nacion', 'Nadal', 'Nagrampa', 'Nalzaro', 'Napeñas',
+    'Narciso', 'Natividad', 'Navales', 'Navarro', 'Neri',
+    'Nicolas', 'Nisperos', 'Nolasco', 'Noynay', 'Nuñez',
+    'Oaminal', 'Ocampo', 'Ocfemia', 'Ochoa', 'Olaguera',
+    'Olano', 'Oliva', 'Olivares', 'Oliveros', 'Olpindo',
+    'Omadto', 'Ombion', 'Onate', 'Ong', 'Orbeta',
+    'Orbita', 'Ordoño', 'Orendain', 'Orense', 'Orobia',
+    'Orozco', 'Ortega', 'Osmeña', 'Osorio', 'Ostrea',
+    'Ouano', 'Pabiton', 'Pableo', 'Pabriaga', 'Pacanan',
+    'Padayao', 'Padilla', 'Padua', 'Paguio', 'Pagulayan',
+    'Palad', 'Palacios', 'Palafox', 'Palaganas', 'Palattao',
+    'Palencia', 'Palma', 'Palo', 'Paloma', 'Palomares',
+    'Pamaran', 'Pamintuan', 'Panaligan', 'Panganiban', 'Pangilinan',
+    'Panopio', 'Papa', 'Paqueo', 'Paras', 'Paredes',
+    'Parreño', 'Pascua', 'Pascual', 'Pastor', 'Paterno',
+    'Patron', 'Pavia', 'Pecaña', 'Pecho', 'Pedrosa',
+    'Pelayo', 'Peña', 'Peñaflor', 'Peñaranda', 'Penarroyo',
+    'Peralta', 'Perez', 'Perlas', 'Pernia', 'Pesquera',
+    'Pestano', 'Piccio', 'Picardal', 'Pineda', 'Pimentel',
+    'Pilapil', 'Pili', 'Piliin', 'Pillar', 'Pilorin',
+    'Poblete', 'Poliquit', 'Ponce', 'Ponferrada', 'Porras',
+    'Prado', 'Prieto', 'Prodigalidad', 'Prudente', 'Punsalan',
+    'Quezon', 'Quiambao', 'Quiaoit', 'Quijano', 'Quimpo',
+    'Quinit', 'Quinones', 'Quiogue', 'Quirino', 'Quisao',
+    'Racelis', 'Rada', 'Ramirez', 'Ramon', 'Ramos',
+    'Ravalo', 'Rayala', 'Razon', 'Recinto', 'Recometa',
+    'Reforma', 'Regalado', 'Reganit', 'Regio', 'Regidor',
+    'Regis', 'Reodica', 'Respicio', 'Revilla', 'Reyes',
+    'Ricafort', 'Ricalde', 'Ridad', 'Rillo', 'Rivera',
+    'Rivero', 'Rizal', 'Robles', 'Roca', 'Rocamora',
+    'Rocero', 'Rodriguez', 'Rojas', 'Romero', 'Ronquillo',
+    'Rosales', 'Rosario', 'Rosete', 'Rotor', 'Roxas',
+    'Rubio', 'Rufino', 'Ruiz', 'Sabal', 'Sabando',
+    'Sabido', 'Sabijon', 'Sabio', 'Saceda', 'Saclolo',
+    'Sagum', 'Salceda', 'Salcedo', 'Salgado', 'Salinas',
+    'Saludar', 'Saluta', 'Salvador', 'Sambrano', 'Samson',
+    'Sanchez', 'Sandoval', 'Sangalang', 'Santiago', 'Santillan',
+    'Sanz', 'Sarino', 'Sarmiento', 'Sarona', 'Savellano',
+    'Sebastian', 'Segovia', 'Sendin', 'Seneres', 'Serafica',
+    'Sereno', 'Senga', 'Serrano', 'Sierra', 'Sigua',
+    'Silva', 'Silvestre', 'Simon', 'Sinco', 'Singson',
+    'Siy', 'Sobejana', 'Soberano', 'Socrates', 'Soliman',
+    'Solis', 'Soliven', 'Solomon', 'Sotto', 'Suansing',
+    'Suarez', 'Subido', 'Sulit', 'Sultan', 'Sumagaysay',
+    'Sunga', 'Tabamo', 'Tabinas', 'Tabuena', 'Tagle',
+    'Taguba', 'Tajonera', 'Talabong', 'Talavera', 'Talento',
+    'Taleon', 'Talosig', 'Tamano', 'Tambalo', 'Tanada',
+    'Tandoc', 'Tañada', 'Tarriela', 'Tating', 'Tautho',
+    'Tayag', 'Tayco', 'Tecson', 'Tejano', 'Tejero',
+    'Teodoro', 'Tibay', 'Tigas', 'Tiglao', 'Timbol',
+    'Tingzon', 'Tiongco', 'Tiongson', 'Tirol', 'Tobias',
+    'Toledo', 'Tolentino', 'Tomelden', 'Tomas', 'Tomaro',
+    'Tomaroy', 'Torino', 'Torralba', 'Torrente', 'Torno',
+    'Trea', 'Trinidad', 'Tuazon', 'Tubig', 'Tubigan',
+    'Tugade', 'Tumbocon', 'Tupas', 'Tuquero', 'Turla',
+    'Umagat', 'Umali', 'Usman', 'Uson', 'Uy',
+    'Valdez', 'Valencia', 'Valenciano', 'Valentin', 'Valera',
+    'Valiao', 'Varela', 'Vargas', 'Vasquez', 'Velarde',
+    'Velasco', 'Velasquez', 'Velez', 'Vera', 'Vergara',
+    'Vibandor', 'Vicente', 'Victorino', 'Vidal', 'Viernes',
+    'Villacorta', 'Villaflor', 'Villafranca', 'Villagomez', 'Villagonzalo',
+    'Villanueva', 'Villar', 'Villareal', 'Villaruel', 'Villaverde',
+    'Villena', 'Virata', 'Vista', 'Vivar', 'Vizconde',
+    'Yabes', 'Yap', 'Yasay', 'Yatco', 'Ylagan',
+    'Yñiguez', 'Yorac', 'Yulo', 'Zabala', 'Zaldivar',
+    'Zamora', 'Zapanta', 'Zaragoza', 'Zosa', 'Zulueta',
+]
+
+# Combine last names pool + extended middle names, deduplicated
+middle_seen = set()
+all_middle  = []
+for name in (all_last + EXTENDED_MIDDLE_NAMES):
+    if isinstance(name, str) and name not in middle_seen:
+        middle_seen.add(name)
+        all_middle.append(name)
+
+print(f"      Total middle names : {len(all_middle)}")
+print(f"      Sample             : {all_middle[:5]}")
+
+# ── Step 5: Save to JSON ──────────────────────────────────────
+print("\n[5/5] Saving to data/ph_names.json ...")
+
+os.makedirs('data', exist_ok=True)
+
+output = {
+    "first_names": {
+        "male":   male_first,
+        "female": female_first,
+        "all":    all_first
+    },
+    "last_names":   all_last,
+    "middle_names": all_middle,
+    "metadata": {
+        "source":              "names-dataset (PyPI) -- country_alpha2='PH'",
+        "total_first":         len(all_first),
+        "total_last":          len(all_last),
+        "total_middle":        len(all_middle),
+        "total_name_combos":   len(all_first) * len(all_middle) * len(all_last),
+    }
+}
+
+with open('data/ph_names.json', 'w', encoding='utf-8') as f:
+    json.dump(output, f, indent=2, ensure_ascii=False)
+
+# ── Summary ───────────────────────────────────────────────────
+print("\n" + "=" * 60)
+print("  DONE!")
+print("=" * 60)
+print(f"  Male first names   : {len(male_first)}")
+print(f"  Female first names : {len(female_first)}")
+print(f"  Last names         : {len(all_last)}")
+print(f"  Middle names       : {len(all_middle)}")
+print(f"  Possible 3-part name combos : {len(all_first) * len(all_middle) * len(all_last):,}")
+print(f"\n  Saved to: data/ph_names.json")
+print(f"\n  Next step: python fix_data.py")
+print("=" * 60)

inference.py

@@ -0,0 +1,395 @@
+"""
+Inference Script for CRNN+CTC Civil Registry OCR
+
+TWO NORMALIZERS:
+  SimpleNormalizer   — for PIL-rendered synthetic images (matches training exactly)
+  AdaptiveNormalizer — for physical/scanned images (any zoom, any size)
+
+AUTO-DETECT MODE: automatically decides which pipeline to use based on
+text density in the image — zoomed-in images get adaptive treatment,
+clean synthetic images get simple treatment.
+"""
+
+import torch
+import cv2
+import numpy as np
+from pathlib import Path
+from typing import Dict, List
+
+from crnn_model import get_crnn_model
+from utils import decode_ctc_predictions, extract_form_fields
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# HELPERS
+# ─────────────────────────────────────────────────────────────────────────────
+
+def _to_gray(img: np.ndarray) -> np.ndarray:
+    if len(img.shape) == 3:
+        return cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+    return img.copy()
+
+
+def _binarize(gray: np.ndarray) -> np.ndarray:
+    """Otsu, falls back to adaptive for uneven backgrounds."""
+    _, otsu = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
+    white_ratio = np.mean(otsu == 255)
+    if white_ratio < 0.30 or white_ratio > 0.97:
+        return cv2.adaptiveThreshold(
+            gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
+            cv2.THRESH_BINARY, 11, 2)
+    return otsu
+
+
+def _crop_to_text(gray: np.ndarray, pad_ratio=0.15) -> np.ndarray:
+    """Crop tightly around dark pixels (the text)."""
+    inv = cv2.bitwise_not(gray)
+    _, thresh = cv2.threshold(inv, 20, 255, cv2.THRESH_BINARY)
+    coords = np.column_stack(np.where(thresh > 0))
+    if len(coords) == 0:
+        return gray
+    y_min, x_min = coords.min(axis=0)
+    y_max, x_max = coords.max(axis=0)
+    pad   = max(4, int((y_max - y_min) * pad_ratio))
+    y_min = max(0, y_min - pad)
+    x_min = max(0, x_min - pad)
+    y_max = min(gray.shape[0] - 1, y_max + pad)
+    x_max = min(gray.shape[1] - 1, x_max + pad)
+    return gray[y_min:y_max+1, x_min:x_max+1]
+
+
+def _aspect_resize(gray: np.ndarray, H: int, W: int) -> np.ndarray:
+    """Resize preserving aspect ratio, pad with white to fill canvas."""
+    h, w = gray.shape
+    if h == 0 or w == 0:
+        return np.ones((H, W), dtype=np.uint8) * 255
+    scale = H / h
+    new_w = int(w * scale)
+    new_h = H
+    if new_w > W:
+        scale = W / w
+        new_h = int(h * scale)
+        new_w = W
+    resized = cv2.resize(gray, (new_w, new_h), interpolation=cv2.INTER_LANCZOS4)
+    canvas  = np.ones((H, W), dtype=np.uint8) * 255
+    y_off   = (H - new_h) // 2
+    x_off   = (W - new_w) // 2
+    canvas[y_off:y_off+new_h, x_off:x_off+new_w] = resized
+    return canvas
+
+
+def _detect_mode(gray: np.ndarray) -> str:
+    """
+    Auto-detect whether image needs adaptive or simple normalization.
+
+    Logic:
+      - If >25% of pixels are dark, text is very large/zoomed → adaptive.
+      - If image size is far from training size (512x64) → adaptive.
+      - Otherwise → simple (matches training pipeline).
+    """
+    h, w  = gray.shape
+    _, bw = cv2.threshold(gray, 127, 255, cv2.THRESH_BINARY)
+    dark_px = np.mean(bw == 0)
+
+    # Text fills too much of the image → zoomed in (like shane.jpg)
+    if dark_px > 0.25:
+        return 'adaptive'
+
+    # Image is far from expected training size (allow 50% tolerance)
+    if not (256 <= w <= 1024 and 32 <= h <= 128):
+        return 'adaptive'
+
+    return 'simple'
+
+
+def _to_tensor(img: np.ndarray) -> torch.Tensor:
+    return torch.FloatTensor(
+        img.astype(np.float32) / 255.0
+    ).unsqueeze(0).unsqueeze(0)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# SIMPLE NORMALIZER  ← for PIL-rendered / training-matched images
+# ─────────────────────────────────────────────────────────────────────────────
+
+class SimpleNormalizer:
+    """
+    Matches fix_data.py training pipeline exactly:
+      grayscale → resize → binarize
+    Best for test images created by create_test_images.py.
+    """
+    def __init__(self, H=64, W=512):
+        self.H, self.W = H, W
+
+    def normalize(self, img: np.ndarray) -> np.ndarray:
+        gray    = _to_gray(img)
+        resized = cv2.resize(gray, (self.W, self.H), interpolation=cv2.INTER_LANCZOS4)
+        return _binarize(resized)
+
+    def normalize_from_path(self, path: str) -> np.ndarray:
+        img = cv2.imread(str(path))
+        if img is None:
+            raise ValueError(f"Cannot load: {path}")
+        return self.normalize(img)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# ADAPTIVE NORMALIZER  ← for real / physical / scanned images
+# ─────────────────────────────────────────────────────────────────────────────
+
+class AdaptiveNormalizer:
+    """
+    For physical documents or images with non-standard zoom/size:
+      grayscale → denoise → crop text → aspect-ratio resize → binarize
+
+    Crops to actual text first, so a zoomed-in image like shane.jpg
+    gets scaled down to training size instead of being squeezed/stretched.
+    """
+    def __init__(self, H=64, W=512):
+        self.H, self.W = H, W
+
+    def normalize(self, img: np.ndarray) -> np.ndarray:
+        gray   = _to_gray(img)
+        gray   = cv2.fastNlMeansDenoising(gray, None, 10, 7, 21)
+        gray   = _crop_to_text(gray)
+        canvas = _aspect_resize(gray, self.H, self.W)
+        return _binarize(canvas)
+
+    def normalize_from_path(self, path: str) -> np.ndarray:
+        img = cv2.imread(str(path))
+        if img is None:
+            raise ValueError(f"Cannot load: {path}")
+        return self.normalize(img)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# AUTO NORMALIZER  ← detects which pipeline to use per image automatically
+# ─────────────────────────────────────────────────────────────────────────────
+
+class AutoNormalizer:
+    """
+    Automatically picks Simple or Adaptive based on image characteristics.
+
+    Examples:
+      demo.jpg  (clean 512x64 PIL)   → Simple   (matches training)
+      name1.jpg (clean 512x64 PIL)   → Simple
+      shane.jpg (huge zoomed text)   → Adaptive (crop then resize)
+      real scan (any size/zoom)      → Adaptive
+    """
+    def __init__(self, H=64, W=512, verbose=False):
+        self.H, self.W = H, W
+        self.verbose   = verbose
+        self._simple   = SimpleNormalizer(H, W)
+        self._adaptive = AdaptiveNormalizer(H, W)
+
+    def normalize(self, img: np.ndarray) -> np.ndarray:
+        gray = _to_gray(img)
+        mode = _detect_mode(gray)
+        if self.verbose:
+            print(f"      auto → {mode}")
+        return self._simple.normalize(img) if mode == 'simple' \
+               else self._adaptive.normalize(img)
+
+    def normalize_from_path(self, path: str) -> np.ndarray:
+        img = cv2.imread(str(path))
+        if img is None:
+            raise ValueError(f"Cannot load: {path}")
+        gray = _to_gray(img)
+        mode = _detect_mode(gray)
+        if self.verbose:
+            print(f"      [{Path(path).name}] → {mode}")
+        return self._simple.normalize(img) if mode == 'simple' \
+               else self._adaptive.normalize(img)
+
+    def to_tensor(self, img: np.ndarray) -> torch.Tensor:
+        return _to_tensor(img)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# MAIN OCR CLASS
+# ─────────────────────────────────────────────────────────────────────────────
+
+class CivilRegistryOCR:
+
+    def __init__(self, checkpoint_path, device='cuda', mode='auto', verbose=False):
+        """
+        Args:
+            checkpoint_path : path to best_model_v4.pth
+            device          : 'cuda' or 'cpu'
+            mode            : 'auto'     → auto-detect per image  (recommended)
+                              'simple'   → always use simple pipeline
+                              'adaptive' → always use adaptive pipeline
+            verbose         : print which mode was chosen per image
+        """
+        if device == 'cuda' and not torch.cuda.is_available():
+            device = 'cpu'
+
+        self.device  = torch.device(device)
+        self.verbose = verbose
+        print(f"Loading model from {checkpoint_path}...")
+
+        checkpoint = torch.load(checkpoint_path, map_location=self.device,
+                                weights_only=False)
+
+        self.char_to_idx = checkpoint['char_to_idx']
+        self.idx_to_char = checkpoint['idx_to_char']
+        self.config      = checkpoint.get('config', {})
+
+        img_height = self.config.get('img_height', 64)
+        img_width  = self.config.get('img_width',  512)
+
+        if mode == 'simple':
+            self.normalizer = SimpleNormalizer(img_height, img_width)
+        elif mode == 'adaptive':
+            self.normalizer = AdaptiveNormalizer(img_height, img_width)
+        else:
+            self.normalizer = AutoNormalizer(img_height, img_width, verbose=verbose)
+
+        self.model = get_crnn_model(
+            model_type=self.config.get('model_type', 'standard'),
+            img_height=img_height,
+            num_chars=checkpoint['model_state_dict']['fc.weight'].shape[0],
+            hidden_size=self.config.get('hidden_size', 128),
+            num_lstm_layers=self.config.get('num_lstm_layers', 1)
+        )
+        self.model.load_state_dict(checkpoint['model_state_dict'])
+        self.model = self.model.to(self.device)
+        self.model.eval()
+
+        print(f"Model loaded successfully")
+        # Support both key names: val_loss (fine-tuned) and val_cer (synthetic baseline)
+        # FIXED Bug 5: removed incorrect `val_cer < 10` heuristic that mislabelled
+        # the metric. The key name alone is the reliable indicator.
+        val_loss = checkpoint.get('val_loss', None)
+        val_cer  = checkpoint.get('val_cer',  None)
+        if val_loss is not None and val_cer is not None:
+            print(f"  Val Loss : {val_loss:.4f} | Val CER: {val_cer:.2f}%")
+        elif val_loss is not None:
+            print(f"  Val Loss : {val_loss:.4f}  (fine-tuned checkpoint — run compare_live_cer.py for true CER)")
+        elif val_cer is not None:
+            print(f"  Val CER  : {val_cer:.2f}%")
+        else:
+            print(f"  Val CER  : N/A (run check_cer.py for true CER)")
+        print(f"  Device   : {self.device}")
+        print(f"  Mode     : {mode}  ({img_height}x{img_width})")
+
+    def _preprocess(self, image_path) -> torch.Tensor:
+        normalized = self.normalizer.normalize_from_path(str(image_path))
+        return _to_tensor(normalized)
+
+    def predict(self, image_path, decode_method='greedy') -> str:
+        img = self._preprocess(image_path).to(self.device)
+        with torch.no_grad():
+            outputs = self.model(img)
+            decoded = decode_ctc_predictions(
+                outputs.cpu(), self.idx_to_char, method=decode_method)
+        return decoded[0]
+
+    def predict_batch(self, image_paths, decode_method='greedy') -> List[Dict]:
+        results = []
+        for image_path in image_paths:
+            try:
+                text = self.predict(image_path, decode_method)
+                results.append({'image_path': str(image_path),
+                                'text': text, 'success': True})
+            except Exception as e:
+                results.append({'image_path': str(image_path),
+                                'error': str(e), 'success': False})
+        return results
+
+    def process_form(self, form_image_path, form_type) -> Dict:
+        text   = self.predict(form_image_path)
+        fields = extract_form_fields(text, form_type)
+        fields['raw_text'] = text
+        return fields
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# FORM FIELD EXTRACTOR
+# ─────────────────────────────────────────────────────────────────────────────
+
+class FormFieldExtractor:
+    def __init__(self, ocr_model: CivilRegistryOCR):
+        self.ocr = ocr_model
+
+    def extract_form1a_fields(self, path):
+        text = self.ocr.predict(path)
+        return {'form_type': 'Form 1A - Birth Certificate', 'raw_text': text}
+
+    def extract_form2a_fields(self, path):
+        text = self.ocr.predict(path)
+        return {'form_type': 'Form 2A - Death Certificate', 'raw_text': text}
+
+    def extract_form3a_fields(self, path):
+        text = self.ocr.predict(path)
+        return {'form_type': 'Form 3A - Marriage Certificate', 'raw_text': text}
+
+    def extract_form90_fields(self, path):
+        text = self.ocr.predict(path)
+        return {'form_type': 'Form 90 - Marriage License Application',
+                'raw_text': text}
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# DEMO
+# ─────────────────────────────────────────────────────────────────────────────
+
+def demo_inference():
+    print("=" * 70)
+    print("Civil Registry OCR  (auto-adaptive normalizer)")
+    print("=" * 70)
+
+    ocr = CivilRegistryOCR(
+        checkpoint_path='checkpoints/best_model_v4.pth',
+        device='cuda',
+        mode='adaptive',  # force adaptive for demo images (many are zoomed/physical)
+        verbose=True   # shows which mode each image triggers
+    )
+
+    print("\n1. Single Prediction:")
+    try:
+        result = ocr.predict('test_images/date1.jpg')
+        print(f"   Recognized text: {result}")
+    except Exception as e:
+        print(f"   Error: {e}")
+
+    print("\n2. Batch Prediction:")
+    '''batch_results = ocr.predict_batch([
+        'test_images/name1.jpg',
+        'test_images/shane.jpg',
+        'test_images/date1.jpg',
+        'test_images/place1.jpg',
+    ])
+    for r in batch_results:
+        status = r['text'] if r['success'] else f"ERROR - {r['error']}"
+        print(f"   {r['image_path']}: {status}")'''
+
+    print("\n3. Form Processing:")
+    try:
+        form_data = ocr.process_form('test_images/form1a_sample.jpg', 'form1a')
+        print(f"   Form Type: Form 1A - Birth Certificate")
+        print(f"   Raw Text: {form_data['raw_text']}")
+    except Exception as e:
+        print(f"   Error: {e}")
+
+
+def create_inference_api():
+    class OCR_API:
+        def __init__(self, checkpoint_path, mode='auto'):
+            self.ocr       = CivilRegistryOCR(checkpoint_path, mode=mode)
+            self.extractor = FormFieldExtractor(self.ocr)
+        def recognize_text(self, p):
+            return {'text': self.ocr.predict(p), 'success': True}
+        def process_birth_certificate(self, p):
+            return self.extractor.extract_form1a_fields(p)
+        def process_death_certificate(self, p):
+            return self.extractor.extract_form2a_fields(p)
+        def process_marriage_certificate(self, p):
+            return self.extractor.extract_form3a_fields(p)
+        def process_marriage_license(self, p):
+            return self.extractor.extract_form90_fields(p)
+    return OCR_API
+
+
+if __name__ == "__main__":
+    demo_inference()

pipeline.py

@@ -0,0 +1,234 @@
+# pipeline.py
+# ============================================================
+# FULL PIPELINE — connects all three algorithms
+#
+#   CRNN+CTC (field_extractor.py)
+#       ↓  field dict
+#   bridge.py  →  MNB (form_classifier.py)
+#       ↓  form type
+#   spaCy NER (extractor.py)
+#       ↓
+#   Populated Form object (Form1A / Form2A / Form3A / Form90)
+#
+# USAGE:
+#   from pipeline import CivilRegistryPipeline
+#
+#   pipeline = CivilRegistryPipeline()
+#
+#   # Path A — single cert PDF (birth / death / marriage)
+#   result = pipeline.process_pdf("form_102.pdf", form_type="birth")
+#   print(result["name_of_child"])
+#
+#   # Path B — Form 90 (needs two birth certs)
+#   result = pipeline.process_form90(
+#       groom_pdf="groom_birth_cert.pdf",
+#       bride_pdf="bride_birth_cert.pdf"
+#   )
+# ============================================================
+
+import sys
+import os
+import json
+from pathlib import Path
+
+# ── Make sure all three algorithm folders are importable ─────
+_ROOT = Path(__file__).parent
+for folder in ["CRNN+CTC", "MNB", "spacyNER"]:
+    p = str(_ROOT / folder)
+    if p not in sys.path:
+        sys.path.insert(0, p)
+sys.path.insert(0, str(_ROOT))
+
+# ── Import CRNN+CTC (Irish's module) ─────────────────────────
+# CRNN+CTC folder is already on sys.path, import directly
+from field_extractor import (
+    pdf_to_image,
+    extract_field_images,
+    run_crnn_ocr,
+    load_crnn_model,
+    BIRTH_FIELDS,
+    DEATH_FIELDS,
+    MARRIAGE_FIELDS,
+)
+
+# ── Import bridge (MNB + NER connector) ──────────────────────
+from bridge import CivilRegistryBridge, crnn_birth_to_text
+
+import torch
+
+# ── Config ────────────────────────────────────────────────────
+CRNN_CHECKPOINT = str(_ROOT / "CRNN+CTC" / "checkpoints" / "best_model.pth")
+NER_MODEL_PATH  = str(_ROOT / "spacyNER" / "models" / "civil_registry_model" / "model-best")
+MNB_MODEL_DIR   = str(_ROOT / "MNB" / "models")
+
+FORM_FIELDS_MAP = {
+    "birth":    BIRTH_FIELDS,
+    "death":    DEATH_FIELDS,
+    "marriage": MARRIAGE_FIELDS,
+}
+
+
+class CivilRegistryPipeline:
+    """
+    Connects CRNN+CTC → MNB → spaCy NER in one call.
+
+    Each member's code is untouched:
+      - Irish   : crnn_ctc/field_extractor.py  (CRNN+CTC)
+      - Princess: mnb/form_classifier.py        (MNB)
+      - Shane   : spacyNER/extractor.py         (NER)
+    """
+
+    def __init__(self,
+                 crnn_checkpoint: str = CRNN_CHECKPOINT,
+                 ner_model_path:  str = NER_MODEL_PATH,
+                 mnb_model_dir:   str = MNB_MODEL_DIR):
+
+        print("=" * 55)
+        print("  Initializing Civil Registry Pipeline")
+        print("=" * 55)
+
+        # ── 1. Load CRNN model (Irish) ────────────────────────
+        self.device = torch.device(
+            "cuda" if torch.cuda.is_available() else "cpu"
+        )
+        print(f"\n  [CRNN] Loading model...")
+        self.crnn_model, self.idx_to_char, self.img_h, self.img_w = (
+            load_crnn_model(crnn_checkpoint, self.device)
+        )
+
+        # ── 2. Load MNB + NER via bridge ─────────────────────
+        print(f"\n  [MNB + NER] Loading models...")
+        self.bridge = CivilRegistryBridge(ner_model_path=ner_model_path, mnb_model_dir=mnb_model_dir)
+
+        print("\n  ✅ Pipeline ready\n")
+
+    # ─────────────────────────────────────────────────────────
+    # PATH A — Single certification form (102 / 103 / 97)
+    # ─────────────────────────────────────────────────────────
+    def process_pdf(self,
+                    pdf_path:  str,
+                    form_type: str = None,
+                    dpi:       int = 200) -> dict:
+        """
+        Process one PDF through the full pipeline.
+
+        Parameters
+        ----------
+        pdf_path  : str   Path to scanned PDF
+        form_type : str   'birth' | 'death' | 'marriage'
+                          If None, MNB auto-detects.
+        dpi       : int   Render DPI (default 200)
+
+        Returns
+        -------
+        dict with all extracted form fields
+        """
+        print(f"\n  Processing: {pdf_path}")
+
+        # Step 1 — CRNN+CTC: PDF → field dict
+        crnn_fields = self._run_crnn(pdf_path, form_type, dpi)
+
+        # Step 2+3 — MNB → NER via bridge
+        form_obj = self.bridge.process(crnn_fields, form_hint=form_type)
+
+        result = form_obj.to_dict()
+        print(f"  ✅ Done — {len([v for v in result.values() if v])} fields extracted")
+        return result
+
+    # ─────────────────────────────────────────────────────────
+    # PATH B — Form 90 (two birth certs → marriage license)
+    # ─────────────────────────────────────────────────────────
+    def process_form90(self,
+                       groom_pdf: str,
+                       bride_pdf: str,
+                       dpi: int = 200) -> dict:
+        """
+        Process two birth cert PDFs into a Form 90.
+
+        Parameters
+        ----------
+        groom_pdf : str   Path to groom's birth certificate PDF
+        bride_pdf : str   Path to bride's birth certificate PDF
+        dpi       : int   Render DPI (default 200)
+
+        Returns
+        -------
+        dict with all Form 90 fields (groom_* and bride_*)
+        """
+        print(f"\n  Processing Form 90")
+        print(f"  Groom: {groom_pdf}")
+        print(f"  Bride: {bride_pdf}")
+
+        # Step 1 — CRNN+CTC both birth certs
+        groom_fields = self._run_crnn(groom_pdf, "birth", dpi)
+        bride_fields = self._run_crnn(bride_pdf,  "birth", dpi)
+
+        # Step 2+3 — bridge fills Form 90
+        form90 = self.bridge.process_marriage_license(groom_fields, bride_fields)
+
+        result = form90.to_dict()
+        print(f"  ✅ Done — {len([v for v in result.values() if v])} fields extracted")
+        return result
+
+    # ─────────────────────────────────────────────────────────
+    # Internal — run CRNN on one PDF
+    # ─────────────────────────────────────────────────────────
+    def _run_crnn(self, pdf_path: str, form_type: str, dpi: int) -> dict:
+        """Convert PDF → run CRNN → return field dict."""
+        # 1. PDF → image
+        page_image = pdf_to_image(pdf_path, dpi=dpi)
+
+        # 2. Resolve form_type — default to 'birth' if unknown/None
+        # extract_field_images uses the form_type string to pick the
+        # correct field map (BIRTH_FIELDS / DEATH_FIELDS / MARRIAGE_FIELDS)
+        resolved_type = form_type if form_type in FORM_FIELDS_MAP else "birth"
+
+        # 3. Crop fields from image using form_type string
+        crops = extract_field_images(page_image, form_type=resolved_type)
+
+        # 4. CRNN OCR each crop
+        crnn_output = run_crnn_ocr(
+            crops, self.crnn_model, self.idx_to_char,
+            self.img_h, self.img_w, self.device
+        )
+        return crnn_output
+# ── Standalone demo ───────────────────────────────────────────
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Civil Registry Pipeline — CRNN + MNB + NER"
+    )
+    parser.add_argument("--pdf",      required=True, help="Path to PDF")
+    parser.add_argument("--form",     default=None,
+                        choices=["birth", "death", "marriage", "form90"],
+                        help="Form type (optional, MNB auto-detects if not given)")
+    parser.add_argument("--groom",    default=None, help="Groom birth cert PDF (Form 90 only)")
+    parser.add_argument("--bride",    default=None, help="Bride birth cert PDF (Form 90 only)")
+    parser.add_argument("--output",   default=None, help="Save result to JSON file")
+    parser.add_argument("--dpi",      type=int, default=200)
+    args = parser.parse_args()
+
+    pipeline = CivilRegistryPipeline()
+
+    if args.form == "form90":
+        if not args.groom or not args.bride:
+            print("ERROR: --groom and --bride required for form90")
+        else:
+            result = pipeline.process_form90(args.groom, args.bride, dpi=args.dpi)
+    else:
+        result = pipeline.process_pdf(args.pdf, form_type=args.form, dpi=args.dpi)
+
+    # Print result
+    print("\n" + "=" * 55)
+    print("  EXTRACTED FIELDS")
+    print("=" * 55)
+    for field, value in result.items():
+        if value:
+            print(f"  {field:<35} {value}")
+
+    # Save to JSON
+    if args.output:
+        with open(args.output, "w", encoding="utf-8") as f:
+            json.dump(result, f, ensure_ascii=False, indent=2)
+        print(f"\n  Saved → {args.output}")

prepare_emnist.py

@@ -0,0 +1,97 @@
+import torchvision
+import torchvision.transforms as transforms
+from PIL import Image
+import numpy as np
+import os
+import json
+
+print("Preparing EMNIST data for CRNN training...")
+print("Using 'balanced' split (47 classes — digits, uppercase, selected lowercase)")
+
+# MAX_SAMPLES: how many EMNIST images to use out of 112,800 available.
+# 50,000 chosen deliberately:
+#   - ~1,064 images per class (47 classes) — enough for solid character recognition
+#   - Keeps a healthy ~3:1 ratio vs synthetic data (16,000) in mixed training
+#   - Going higher (e.g. full 112,800) would drown out synthetic Filipino-specific
+#     patterns since EMNIST would be 88% of the mixed dataset
+#   - IAM fine-tuning and physical scans handle remaining handwriting gaps
+MAX_SAMPLES = 50000
+VAL_RATIO   = 0.10   # 90% train, 10% val — proper percentage split
+
+train_data = torchvision.datasets.EMNIST(
+    root='datasets/emnist',
+    split='balanced',       # balanced split — already downloaded
+    train=True,
+    download=False,         # files already exist, skip download
+    transform=transforms.ToTensor()
+)
+
+# balanced split has 47 classes:
+# 0-9 digits, A-Z uppercase, and selected lowercase
+# mapping follows EMNIST balanced label order
+LABELS = [
+    '0','1','2','3','4','5','6','7','8','9',
+    'A','B','C','D','E','F','G','H','I','J','K','L','M',
+    'N','O','P','Q','R','S','T','U','V','W','X','Y','Z',
+    'a','b','d','e','f','g','h','n','q','r','t',
+]  # 47 classes exactly matching balanced split label indices
+
+os.makedirs('data/train/emnist', exist_ok=True)
+os.makedirs('data/val/emnist', exist_ok=True)
+
+annotations_train = []
+annotations_val   = []
+
+val_cutoff = int(MAX_SAMPLES * (1 - VAL_RATIO))  # 45,000 train / 5,000 val
+
+print(f"Dataset size : {len(train_data)} images available")
+print(f"Using        : {MAX_SAMPLES} ({MAX_SAMPLES/len(train_data)*100:.1f}% of full dataset)")
+print(f"Train / Val  : {val_cutoff} / {MAX_SAMPLES - val_cutoff} (90/10 split)")
+print("Saving images...")
+
+saved = 0   # count of successfully saved images (skips bad label indices)
+for i, (img_tensor, label_idx) in enumerate(train_data):
+    if saved >= MAX_SAMPLES:
+        break
+
+    # Safety check — skip if label index is out of range for our LABELS list
+    if label_idx >= len(LABELS):
+        continue
+
+    char = LABELS[label_idx]
+    img  = img_tensor.squeeze().numpy()
+    img  = (img * 255).astype(np.uint8)
+
+    # EMNIST images are transposed — rotate and flip to correct orientation
+    img  = np.rot90(img, k=3)
+    img  = np.fliplr(img)
+
+    pil_img = Image.fromarray(img).convert('RGB')
+    pil_img = pil_img.resize((512, 64))   # must match IMG_WIDTH=512
+
+    fname = f'emnist_{saved:05d}.jpg'   # sequential filenames based on saved count
+
+    # FIXED: proper percentage-based split (was hardcoded `if i < 5000`)
+    if saved < val_cutoff:
+        pil_img.save(f'data/train/emnist/{fname}')
+        annotations_train.append({'image_path': f'emnist/{fname}', 'text': char})
+    else:
+        pil_img.save(f'data/val/emnist/{fname}')
+        annotations_val.append({'image_path': f'emnist/{fname}', 'text': char})
+
+    saved += 1
+    if saved % 5000 == 0:
+        print(f"  Processed {saved}/{MAX_SAMPLES} images...")
+
+with open('data/emnist_train_annotations.json', 'w') as f:
+    json.dump(annotations_train, f, indent=2)
+with open('data/emnist_val_annotations.json', 'w') as f:
+    json.dump(annotations_val, f, indent=2)
+
+print(f"\nDone!")
+print(f"  Train : {len(annotations_train)} images  (~{len(annotations_train)//47} per class)")
+print(f"  Val   : {len(annotations_val)} images")
+print(f"  Total : {len(annotations_train) + len(annotations_val)} / {len(train_data)} used")
+print(f"  Labels: {sorted(set(a['text'] for a in annotations_train))}")
+print(f"\nClass coverage: {len(set(a['text'] for a in annotations_train))}/47 classes in train")
+print("\nNext step: python train_with_emnist.py")

requirements.txt

@@ -0,0 +1,26 @@
+paddleocr==3.4.0
+scikit-learn==1.7.2
+opencv-python-headless>=4.8.0
+Pillow>=10.0.0
+pdf2image>=1.17.0
+pytesseract>=0.3.13
+
+numpy>=1.24.0
+pandas>=2.0.0
+editdistance>=0.6.2
+tqdm>=4.65.0
+
+flask>=3.0.0
+flask-cors>=4.0.0
+
+pymysql>=1.1.0
+
+spacy>=3.7.0
+transformers>=4.35.0
+sentencepiece>=0.1.99
+
+python-dotenv>=1.0.0
+requests>=2.31.0
+pyyaml>=6.0
+albumentations>=1.3.0
+python-docx>=1.1.0

template_matcher.py

@@ -0,0 +1,1569 @@
+"""
+template_matcher.py
+================================================
+Extracts field values from Philippine civil registry scanned forms.
+
+PIPELINE
+--------
+1. Pre-flight image quality check (upside-down, skew, blur, aspect, ORB fit)
+2. Auto-correct image (rotate 180° if upside-down, de-skew if tilted)
+3. Detect form type
+4. Align image to reference (perspective + ECC + ORB)
+5. Preprocess aligned image
+6. Use PaddleOCR ONLY for text-box detection / field localization
+7. Batch all field crops → single CRNN+CTC forward pass
+8. Smart-merge CRNN and PaddleOCR text using _text_quality_score
+
+NOTES
+-----
+- PaddleOCR is not the final OCR engine for all fields; CRNN+CTC remains the
+  primary text reader.
+- PaddleOCR is used for detection/localization and as selective assist text
+  for certain fields such as province, registry number, municipality, etc.
+- This file is written to be a drop-in replacement for the EasyOCR-based version.
+"""
+
+import sys as _sys
+import os
+import sys
+import re as _re
+
+import numpy as np
+from PIL import Image
+
+try:
+    import cv2 as _cv2
+    _CV2_OK = True
+except ImportError:
+    _CV2_OK = False
+
+# ── Reference images ─────────────────────────────────────────────
+_REF_DIR = os.path.join(os.path.dirname(__file__), 'references')
+REFERENCE_IMAGES = {
+    '102': os.path.join(_REF_DIR, 'reference-102.png'),
+    '103': os.path.join(_REF_DIR, 'reference-103.png'),
+    '90':  os.path.join(_REF_DIR, 'reference-90.png'),
+    '97':  os.path.join(_REF_DIR, 'reference-97.png'),
+}
+
+# ── Reference image cache (avoid repeated disk reads) ────────────
+_REF_CACHE: dict = {}
+
+
+def _get_ref_gray(form_type: str):
+    """Return cached grayscale reference image for form_type, or None."""
+    if form_type not in _REF_CACHE:
+        path = REFERENCE_IMAGES.get(form_type)
+        if path and os.path.exists(path) and _CV2_OK:
+            _REF_CACHE[form_type] = _cv2.imread(path, _cv2.IMREAD_GRAYSCALE)
+        else:
+            _REF_CACHE[form_type] = None
+    return _REF_CACHE[form_type]
+
+
+# ── CRNN+CTC engine ──────────────────────────────────────────────
+_CRNN_DIR = os.path.join(os.path.dirname(__file__), 'CRNN+CTC')
+if _CRNN_DIR not in _sys.path:
+    _sys.path.insert(0, _CRNN_DIR)
+
+_CRNN_CHECKPOINT = os.path.join(_CRNN_DIR, 'checkpoints', 'best_model_v6.pth')
+_crnn_ocr = None
+_crnn_decode = None
+
+
+def _get_crnn():
+    global _crnn_ocr, _crnn_decode
+    if _crnn_ocr is None:
+        try:
+            import torch
+            from inference import CivilRegistryOCR
+            from utils import decode_ctc_predictions as _dcp
+
+            print('[template_matcher] Loading CRNN+CTC model...')
+            device = 'cuda' if torch.cuda.is_available() else 'cpu'
+            _crnn_ocr = CivilRegistryOCR(
+                checkpoint_path=_CRNN_CHECKPOINT,
+                device=device,
+                mode='adaptive',
+            )
+            _crnn_decode = _dcp
+            print('[template_matcher] CRNN+CTC ready.')
+        except Exception as e:
+            print(f'[template_matcher] CRNN+CTC load error: {e}')
+    return _crnn_ocr
+
+
+def _crnn_read(crop_img: Image.Image) -> str:
+    """Run CRNN+CTC on a single PIL Image crop and return decoded text."""
+    ocr = _get_crnn()
+    if ocr is None or _crnn_decode is None:
+        return ''
+    try:
+        import torch
+
+        rgb = np.array(crop_img.convert('RGB'))
+        bgr = rgb[:, :, ::-1].copy()
+        normalized = ocr.normalizer.normalize(bgr)
+        tensor = torch.FloatTensor(
+            normalized.astype(np.float32) / 255.0
+        ).unsqueeze(0).unsqueeze(0).to(ocr.device)
+
+        with torch.no_grad():
+            outputs = ocr.model(tensor)
+
+        decoded = _crnn_decode(outputs.cpu(), ocr.idx_to_char, method='greedy')
+        return decoded[0].strip()
+    except Exception as e:
+        print(f'[template_matcher] CRNN+CTC read error: {e}')
+        return ''
+
+
+def _crnn_read_batch(crops: list) -> list:
+    """
+    Run CRNN+CTC on a list of PIL Image crops in one forward pass.
+    """
+    if not crops:
+        return []
+
+    ocr = _get_crnn()
+    if ocr is None or _crnn_decode is None:
+        return [''] * len(crops)
+
+    try:
+        import torch
+
+        tensors = []
+        for crop in crops:
+            rgb = np.array(crop.convert('RGB'))
+            bgr = rgb[:, :, ::-1].copy()
+            normalized = ocr.normalizer.normalize(bgr)
+            t = torch.FloatTensor(
+                normalized.astype(np.float32) / 255.0
+            ).unsqueeze(0).unsqueeze(0)
+            tensors.append(t)
+
+        batch = torch.cat(tensors, dim=0).to(ocr.device)
+
+        with torch.no_grad():
+            outputs = ocr.model(batch)
+
+        decoded = _crnn_decode(outputs.cpu(), ocr.idx_to_char, method='greedy')
+        return [d.strip() for d in decoded]
+
+    except Exception as e:
+        print(f'[template_matcher] CRNN batch error: {e}; falling back to serial')
+        return [_crnn_read(c) for c in crops]
+
+
+# ── PaddleOCR engine (DETECTION + OPTIONAL ASSIST TEXT) ──────────
+_paddle_reader = None
+_PADDLE_DETECT_SCALE = 0.75
+
+
+def _get_paddleocr():
+    global _paddle_reader
+    if _paddle_reader is None:
+        try:
+            from paddleocr import PaddleOCR
+            print('[template_matcher] Loading PaddleOCR...')
+            _paddle_reader = PaddleOCR(
+                use_angle_cls=True,
+                lang='en',
+            )
+            print('[template_matcher] PaddleOCR ready.')
+        except Exception as e:
+            print(f'[template_matcher] PaddleOCR unavailable: {e}')
+    return _paddle_reader
+
+
+def _paddle_detect(img: Image.Image, scale: float = _PADDLE_DETECT_SCALE):
+    """
+    Return PaddleOCR detections from a downscaled image and scale boxes back
+    to the original image coordinates.
+
+    Output:
+      [
+        {
+          'box': (x1, y1, x2, y2),
+          'text': 'detected text',
+          'conf': 0.95,
+          'cx': center_x,
+          'cy': center_y,
+          'poly': [[x, y], ...]
+        },
+        ...
+      ]
+    """
+    ocr = _get_paddleocr()
+    if ocr is None:
+        return []
+
+    try:
+        orig_w, orig_h = img.size
+        small_w = max(1, int(orig_w * scale))
+        small_h = max(1, int(orig_h * scale))
+        small = img.resize((small_w, small_h), Image.BILINEAR)
+        arr = np.array(small.convert('RGB'))
+
+        raw = ocr.ocr(arr, cls=True)
+        if not raw:
+            return []
+
+        detections = []
+        pages = raw if isinstance(raw, list) else [raw]
+        for page in pages:
+            if not page:
+                continue
+            for item in page:
+                if not item or len(item) < 2:
+                    continue
+                box, rec = item
+                text, conf = rec if isinstance(rec, (list, tuple)) and len(rec) >= 2 else ('', 0.0)
+                xs = [p[0] / scale for p in box]
+                ys = [p[1] / scale for p in box]
+                x1, y1 = int(min(xs)), int(min(ys))
+                x2, y2 = int(max(xs)), int(max(ys))
+                detections.append({
+                    'box': (x1, y1, x2, y2),
+                    'text': (text or '').strip(),
+                    'conf': float(conf),
+                    'cx': (x1 + x2) // 2,
+                    'cy': (y1 + y2) // 2,
+                    'poly': [[float(px) / scale, float(py) / scale] for px, py in box],
+                })
+
+        return detections
+    except Exception as e:
+        print(f'[template_matcher] PaddleOCR detect error: {e}')
+        return []
+
+
+def _paddle_read(crop_img: Image.Image) -> str:
+    """
+    Optional helper for debugging only.
+    Not used as final OCR in extraction unless selected by smart merge.
+    """
+    ocr = _get_paddleocr()
+    if ocr is None:
+        return ''
+
+    try:
+        arr = np.array(crop_img.convert('RGB'))
+        raw = ocr.ocr(arr, cls=True)
+        if not raw:
+            return ''
+
+        pieces = []
+        pages = raw if isinstance(raw, list) else [raw]
+        for page in pages:
+            if not page:
+                continue
+            page_sorted = sorted(
+                page,
+                key=lambda item: min(pt[0] for pt in item[0]) if item and item[0] else 0
+            )
+            for item in page_sorted:
+                if item and len(item) >= 2 and item[1]:
+                    pieces.append((item[1][0] or '').strip())
+
+        return ' '.join([p for p in pieces if p]).strip()
+    except Exception as e:
+        print(f'[template_matcher] PaddleOCR read error: {e}')
+        return ''
+
+
+# Backward-compatible aliases so old code paths still work.
+def _easyocr_detect(img: Image.Image, scale: float = _PADDLE_DETECT_SCALE):
+    return _paddle_detect(img, scale=scale)
+
+
+def _easyocr_read(crop_img: Image.Image) -> str:
+    return _paddle_read(crop_img)
+
+
+# Hint constants
+_LINE = 'line'
+_BLOCK = 'block'
+_WORD = 'word'
+
+# ── Post-processing ───────────────────────────────────────────────
+_SEX_KEYWORDS = {
+    'female': 'FEMALE', 'fem': 'FEMALE', 'f': 'FEMALE',
+    'male':   'MALE',   'm':  'MALE',
+}
+
+_NATIONALITY_CANONICAL = {
+    'filipino': 'Filipino', 'filipine': 'Filipino', 'filipioo': 'Filipino',
+    'filipiao': 'Filipino', 'filipinc': 'Filipino', 'filipin': 'Filipino',
+    'filipina': 'Filipino', 'fillipino': 'Filipino', 'fillipine': 'Filipino',
+    'philipino': 'Filipino', 'philippino': 'Filipino', 'pilipino': 'Filipino',
+    'pilipina': 'Filipino', 'pilipiino': 'Filipino', 'fiipino': 'Filipino',
+    'fllipino': 'Filipino', 'fiiipino': 'Filipino', 'filipno': 'Filipino',
+    'filipimo': 'Filipino', 'fihpino': 'Filipino',
+    'american': 'American', 'americian': 'American', 'amercan': 'American', 'amrican': 'American',
+    'chinese': 'Chinese', 'chineze': 'Chinese', 'chines': 'Chinese',
+    'japanese': 'Japanese', 'japanase': 'Japanese', 'japanes': 'Japanese',
+    'korean': 'Korean', 'koreon': 'Korean',
+    'british': 'British', 'britsh': 'British',
+    'australian': 'Australian', 'australan': 'Australian',
+    'indian': 'Indian', 'indin': 'Indian',
+    'spanish': 'Spanish', 'spansh': 'Spanish',
+    'indonesian': 'Indonesian', 'malaysian': 'Malaysian', 'thai': 'Thai',
+    'vietnamese': 'Vietnamese', 'singaporean': 'Singaporean', 'canadian': 'Canadian',
+    'german': 'German', 'french': 'French', 'italian': 'Italian', 'dutch': 'Dutch',
+}
+
+
+def _fix_nationality(text: str) -> str:
+    key = _re.sub(r'[^a-z]', '', text.lower())
+    if not key:
+        return text
+
+    if key in _NATIONALITY_CANONICAL:
+        return _NATIONALITY_CANONICAL[key]
+
+    if len(key) >= 5:
+        for canon_key, canon_val in _NATIONALITY_CANONICAL.items():
+            if canon_key.startswith(key) or key.startswith(canon_key[:max(5, len(key) - 1)]):
+                return canon_val
+
+    best_val = None
+    best_ratio = 0.0
+    for canon_key, canon_val in _NATIONALITY_CANONICAL.items():
+        longer = max(len(key), len(canon_key))
+        if longer == 0:
+            continue
+        matches = sum(a == b for a, b in zip(key, canon_key))
+        ratio = matches / longer
+        if ratio > best_ratio:
+            best_ratio = ratio
+            best_val = canon_val
+
+    if best_ratio >= 0.78 and best_val is not None:
+        return best_val
+
+    return text
+
+
+_MONTH_CANONICAL = {
+    'january': 'January', 'januray': 'January', 'janury': 'January',
+    'janaury': 'January', 'janary': 'January', 'januarry': 'January', 'jan': 'January',
+    'february': 'February', 'feburary': 'February', 'febuary': 'February',
+    'febraury': 'February', 'februray': 'February', 'februay': 'February', 'feb': 'February',
+    'march': 'March', 'marct': 'March', 'mauct': 'March', 'mauch': 'March',
+    'marh': 'March', 'marc': 'March', 'mach': 'March', 'mrach': 'March', 'mar': 'March',
+    'april': 'April', 'apirl': 'April', 'apil': 'April', 'aprl': 'April', 'apri': 'April', 'apr': 'April',
+    'may': 'May',
+    'june': 'June', 'jun': 'June', 'juen': 'June',
+    'july': 'July', 'jully': 'July', 'jul': 'July', 'juy': 'July', 'jly': 'July',
+    'august': 'August', 'augst': 'August', 'auguts': 'August', 'agust': 'August', 'aug': 'August',
+    'september': 'September', 'septmber': 'September', 'septembar': 'September',
+    'sepember': 'September', 'sepetmber': 'September', 'sep': 'September', 'sept': 'September',
+    'october': 'October', 'ocober': 'October', 'octber': 'October', 'octobr': 'October', 'oct': 'October',
+    'november': 'November', 'novmber': 'November', 'noveber': 'November', 'novembr': 'November', 'nov': 'November',
+    'december': 'December', 'decmber': 'December', 'deceber': 'December', 'decembr': 'December', 'dec': 'December',
+}
+
+_MONTH_ORDER = {
+    'January': 1, 'February': 2, 'March': 3, 'April': 4,
+    'May': 5, 'June': 6, 'July': 7, 'August': 8,
+    'September': 9, 'October': 10, 'November': 11, 'December': 12,
+}
+
+
+def _fix_month_word(word: str) -> str:
+    key = _re.sub(r'[^a-z]', '', word.lower())
+    if not key:
+        return word
+    if key in _MONTH_CANONICAL:
+        return _MONTH_CANONICAL[key]
+    if len(key) >= 3:
+        for mkey, mval in _MONTH_CANONICAL.items():
+            if mkey.startswith(key) or key.startswith(mkey):
+                return mval
+    return word
+
+
+def _fix_year(year_str: str, context_text: str = '') -> str:
+    y = _re.sub(r'[^0-9]', '', year_str)
+    if not y:
+        return year_str
+
+    if len(y) == 4:
+        yr = int(y)
+        if 1900 <= yr <= 2030:
+            return y
+        if y.startswith('0'):
+            candidate = '2' + y[1:]
+            if 1900 <= int(candidate) <= 2030:
+                return candidate
+        return y
+
+    if len(y) == 3:
+        specific = {
+            '202': '2022', '201': '2015', '200': '2000',
+            '199': '1999', '198': '1985', '197': '1975',
+            '196': '1965', '195': '1955',
+        }
+        if y in specific:
+            return specific[y]
+        return y + '0'
+
+    if len(y) == 2:
+        yr = int(y)
+        return str(1900 + yr) if yr >= 40 else str(2000 + yr)
+
+    return y
+
+
+def _fix_date_string(text: str) -> str:
+    text = _re.sub(r'[^\w\s\-/,.]', '', text).strip()
+    if not text:
+        return text
+
+    if _re.fullmatch(r'\d{4}[-/]\d{1,2}[-/]\d{1,2}', text):
+        return text
+    if _re.fullmatch(r'\d{1,2}[-/]\d{1,2}[-/]\d{2,4}', text):
+        parts = _re.split(r'[-/]', text)
+        sep = '-' if '-' in text else '/'
+        parts[-1] = _fix_year(parts[-1], text)
+        return sep.join(parts)
+
+    tokens = _re.split(r'([\s,\-/.]+)', text)
+    result = []
+
+    for tok in tokens:
+        stripped = tok.strip(' ,.-/')
+        if not stripped:
+            result.append(tok)
+            continue
+
+        if _re.fullmatch(r'\d+', stripped):
+            num = int(stripped)
+            if 1 <= num <= 31 and len(stripped) <= 2:
+                result.append(tok)
+            elif len(stripped) in (2, 3, 4):
+                fixed = _fix_year(stripped, text)
+                result.append(tok.replace(stripped, fixed))
+            else:
+                result.append(tok)
+            continue
+
+        corrected_month = _fix_month_word(stripped)
+        if corrected_month != stripped:
+            result.append(tok.replace(stripped, corrected_month))
+            continue
+
+        result.append(tok)
+
+    return ''.join(result).strip()
+
+
+_FIELD_TYPE = {
+    'sex': 'sex', 'groom_sex': 'sex', 'bride_sex': 'sex',
+    'husband_sex': 'sex', 'wife_sex': 'sex',
+    'dob_year': 'year',
+    'age': 'digits', 'groom_age': 'digits', 'bride_age': 'digits',
+    'husband_age': 'digits', 'wife_age': 'digits', 'dob_day': 'digits',
+    'registration_date': 'date', 'marriage_date': 'date',
+    'date_of_marriage': 'date', 'date_of_death': 'date',
+    'date_of_birth': 'date', 'date_issued': 'date',
+    'groom_dob': 'date', 'bride_dob': 'date',
+    'husband_dob': 'date', 'wife_dob': 'date',
+    'registry_no': 'registry', 'marriage_license_no': 'registry',
+    'mother_citizenship': 'nationality', 'father_citizenship': 'nationality',
+    'citizenship': 'nationality',
+    'groom_citizenship': 'nationality', 'bride_citizenship': 'nationality',
+    'husband_citizenship': 'nationality', 'wife_citizenship': 'nationality',
+    'groom_father_citizenship': 'nationality', 'groom_mother_citizenship': 'nationality',
+    'bride_father_citizenship': 'nationality', 'bride_mother_citizenship': 'nationality',
+    'husband_father_citizenship': 'nationality', 'husband_mother_citizenship': 'nationality',
+    'wife_father_citizenship': 'nationality', 'wife_mother_citizenship': 'nationality',
+}
+
+
+def _postprocess(text: str, field_name: str) -> str:
+    text = text.strip()
+    if not text:
+        return ''
+
+    rule = _FIELD_TYPE.get(field_name)
+
+    if rule == 'sex':
+        tl = text.lower()
+        for kw in sorted(_SEX_KEYWORDS, key=len, reverse=True):
+            if kw in tl:
+                return _SEX_KEYWORDS[kw]
+        return ''
+
+    if rule == 'nationality':
+        parts = text.split()
+        whole = _fix_nationality(text)
+        if whole.lower() != text.lower():
+            return whole
+        fixed = [_fix_nationality(p) for p in parts]
+        return ' '.join(fixed)
+
+    if rule == 'year':
+        m = _re.search(r'(19|20)\d{2}', text)
+        if m:
+            return m.group(0)
+        m3 = _re.search(r'\b(19\d|20\d)\b', text)
+        if m3:
+            return _fix_year(m3.group(0))
+        digits = _re.sub(r'\D', '', text)
+        if len(digits) >= 4:
+            return digits[:4]
+        if len(digits) == 3:
+            return _fix_year(digits)
+        return ''
+
+    if rule == 'digits':
+        d = _re.sub(r'\D', '', text)
+        return d if d else ''
+
+    if rule == 'date':
+        cleaned = _re.sub(r'[^\w\s\-/,.]', '', text).strip()
+        if len(cleaned) < 3:
+            return ''
+        return _fix_date_string(cleaned)
+
+    if rule == 'registry':
+        cleaned = _re.sub(r'[^\w\s\-/]', '', text).strip()
+        return cleaned if len(cleaned) >= 2 else ''
+
+    cleaned = _re.sub(r'\s+', ' ', text).strip()
+
+    if len(cleaned) == 1:
+        return ''
+
+    if len(cleaned) <= 2 and not _re.search(r'[aeiou0-9]', cleaned.lower()):
+        return ''
+
+    return cleaned
+
+
+def _is_valid_field_value(field_name: str, text: str) -> bool:
+    if not text:
+        return False
+
+    rule = _FIELD_TYPE.get(field_name)
+    if rule in ('digits', 'year', 'date', 'registry', 'sex', 'nationality'):
+        return True
+
+    cleaned = text.strip()
+    if not _re.search(r'[A-Za-z0-9]', cleaned):
+        return False
+    if len(cleaned) <= 1:
+        return False
+    return True
+
+
+def _text_quality_score(field_name: str, text: str) -> float:
+    if not text:
+        return -999.0
+
+    score = 0.0
+    t = text.strip()
+
+    score += len(t)
+    score -= len(_re.findall(r'[^A-Za-z0-9\s\-/,.]', t)) * 2.0
+    score += len(_re.findall(r'[A-Za-z0-9]', t)) * 0.5
+
+    rule = _FIELD_TYPE.get(field_name)
+
+    if rule == 'digits':
+        if _re.fullmatch(r'\d+', _re.sub(r'\D', '', t)):
+            score += 8.0
+    elif rule == 'year':
+        if _re.search(r'(19|20)\d{2}', t):
+            score += 10.0
+    elif rule == 'date':
+        if _re.search(r'\b\d{1,2}\b', t) or _re.search(
+            r'(JAN|FEB|MAR|APR|MAY|JUN|JUL|AUG|SEP|OCT|NOV|DEC)', t.upper()
+        ):
+            score += 8.0
+        for month in _MONTH_ORDER:
+            if month in t:
+                score += 5.0
+                break
+        if _re.search(r'(19|20)\d{2}', t):
+            score += 5.0
+    elif rule == 'sex':
+        tl = t.lower()
+        if 'male' in tl or 'female' in tl or tl in ('m', 'f'):
+            score += 10.0
+    elif rule == 'registry':
+        if _re.search(r'[A-Za-z0-9]', t):
+            score += 8.0
+    elif rule == 'nationality':
+        key = _re.sub(r'[^a-z]', '', t.lower())
+        if key in _NATIONALITY_CANONICAL:
+            score += 12.0
+        elif len(key) >= 5 and any(k.startswith(key[:5]) for k in _NATIONALITY_CANONICAL):
+            score += 6.0
+
+    return score
+
+
+def _smart_merge(field_name: str, crnn_text: str, assist_text: str) -> str:
+    crnn_post = _postprocess(crnn_text, field_name)
+    assist_post = _postprocess(assist_text, field_name)
+
+    crnn_ok = _is_valid_field_value(field_name, crnn_post)
+    assist_ok = _is_valid_field_value(field_name, assist_post)
+
+    if crnn_ok and not assist_ok:
+        return crnn_post
+    if assist_ok and not crnn_ok:
+        return assist_post
+    if not crnn_ok and not assist_ok:
+        return crnn_post or assist_post or ''
+
+    crnn_score = _text_quality_score(field_name, crnn_post)
+    assist_score = _text_quality_score(field_name, assist_post)
+    return crnn_post if crnn_score >= assist_score else assist_post
+
+
+TEMPLATES = {
+    '102': {
+        'province':             (0.169, 0.109, 0.608, 0.134, _LINE),
+        'registry_no':          (0.613, 0.119, 0.884, 0.152, _LINE),
+        'city_municipality':    (0.220, 0.132, 0.608, 0.153, _LINE),
+        'name_first':           (0.132, 0.165, 0.398, 0.185, _LINE),
+        'name_middle':          (0.397, 0.165, 0.646, 0.186, _LINE),
+        'name_last':            (0.646, 0.165, 0.882, 0.185, _LINE),
+        'sex':                  (0.122, 0.195, 0.325, 0.215, _WORD),
+        'dob_day':              (0.458, 0.197, 0.565, 0.216, _WORD),
+        'dob_month':            (0.564, 0.195, 0.750, 0.216, _LINE),
+        'dob_year':             (0.748, 0.196, 0.883, 0.216, _WORD),
+        'place_of_birth':       (0.380, 0.225, 0.886, 0.244, _LINE),
+        'type_of_birth':        (0.124, 0.268, 0.329, 0.290, _WORD),
+        'birth_order':          (0.543, 0.275, 0.746, 0.290, _WORD),
+        'weight_at_birth':      (0.752, 0.257, 0.838, 0.289, _WORD),
+        'mother_name':          (0.184, 0.302, 0.885, 0.322, _LINE),
+        'mother_citizenship':   (0.126, 0.332, 0.503, 0.354, _LINE),
+        'mother_religion':      (0.508, 0.335, 0.882, 0.354, _LINE),
+        'mother_occupation':    (0.512, 0.364, 0.759, 0.392, _LINE),
+        'mother_age_at_birth':  (0.758, 0.373, 0.888, 0.392, _WORD),
+        'mother_residence':     (0.139, 0.402, 0.888, 0.426, _LINE),
+        'father_name':          (0.129, 0.437, 0.885, 0.458, _LINE),
+        'father_citizenship':   (0.124, 0.470, 0.314, 0.497, _LINE),
+        'father_religion':      (0.316, 0.470, 0.546, 0.498, _LINE),
+        'father_occupation':    (0.546, 0.470, 0.750, 0.496, _LINE),
+        'father_age_at_birth':  (0.750, 0.478, 0.887, 0.498, _WORD),
+        'father_residence':     (0.139, 0.508, 0.889, 0.531, _LINE),
+        'marriage_date':        (0.105, 0.556, 0.397, 0.581, _LINE),
+        'marriage_place':       (0.399, 0.557, 0.887, 0.582, _LINE),
+        'registration_date':    (0.540, 0.898, 0.880, 0.917, _LINE),
+    },
+    '103': {
+        'province':           (0.164, 0.082, 0.628, 0.102, _LINE),
+        'registry_no':        (0.636, 0.093, 0.925, 0.123, _LINE),
+        'city_municipality':  (0.219, 0.099, 0.629, 0.122, _LINE),
+        'deceased_name':      (0.106, 0.144, 0.721, 0.174, _LINE),
+        'sex':                (0.723, 0.140, 0.925, 0.174, _WORD),
+        'date_of_death':      (0.094, 0.192, 0.311, 0.220, _LINE),
+        'date_of_birth':      (0.315, 0.192, 0.560, 0.218, _LINE),
+        'age':                (0.562, 0.199, 0.703, 0.218, _WORD),
+        'place_of_death':     (0.092, 0.233, 0.703, 0.258, _LINE),
+        'civil_status':       (0.701, 0.236, 0.930, 0.258, _WORD),
+        'religion':           (0.092, 0.273, 0.312, 0.298, _LINE),
+        'citizenship':        (0.311, 0.272, 0.507, 0.298, _LINE),
+        'residence':          (0.507, 0.269, 0.929, 0.297, _LINE),
+        'occupation':         (0.090, 0.309, 0.285, 0.336, _LINE),
+        'father_name':        (0.284, 0.311, 0.603, 0.334, _LINE),
+        'mother_name':        (0.601, 0.309, 0.932, 0.333, _LINE),
+        'cause_immediate':    (0.295, 0.373, 0.690, 0.389, _LINE),
+        'cause_antecedent':   (0.301, 0.388, 0.697, 0.407, _LINE),
+        'cause_underlying':   (0.301, 0.406, 0.685, 0.425, _LINE),
+        'registration_date':  (0.559, 0.955, 0.922, 0.974, _LINE),
+    },
+    '90': {
+        'province':                  (0.199, 0.094, 0.637, 0.116, _LINE),
+        'registry_no':               (0.645, 0.108, 0.909, 0.133, _LINE),
+        'city_municipality':         (0.248, 0.114, 0.634, 0.133, _LINE),
+        'marriage_license_no':       (0.666, 0.133, 0.916, 0.151, _LINE),
+        'date_issued':               (0.766, 0.148, 0.916, 0.166, _LINE),
+        'groom_name_first':          (0.170, 0.292, 0.467, 0.311, _LINE),
+        'groom_name_middle':         (0.172, 0.307, 0.471, 0.323, _LINE),
+        'groom_name_last':           (0.172, 0.323, 0.471, 0.338, _LINE),
+        'bride_name_first':          (0.617, 0.292, 0.918, 0.307, _LINE),
+        'bride_name_middle':         (0.621, 0.308, 0.917, 0.324, _LINE),
+        'bride_name_last':           (0.615, 0.323, 0.915, 0.338, _LINE),
+        'groom_dob':                 (0.133, 0.348, 0.396, 0.370, _LINE),
+        'groom_age':                 (0.396, 0.347, 0.473, 0.368, _WORD),
+        'bride_dob':                 (0.574, 0.349, 0.840, 0.369, _LINE),
+        'bride_age':                 (0.842, 0.348, 0.921, 0.370, _WORD),
+        'groom_place_of_birth':      (0.136, 0.380, 0.480, 0.402, _LINE),
+        'bride_place_of_birth':      (0.577, 0.379, 0.923, 0.402, _LINE),
+        'groom_sex':                 (0.133, 0.408, 0.267, 0.426, _WORD),
+        'groom_citizenship':         (0.265, 0.409, 0.476, 0.428, _LINE),
+        'bride_sex':                 (0.581, 0.408, 0.711, 0.429, _WORD),
+        'bride_citizenship':         (0.708, 0.410, 0.921, 0.430, _LINE),
+        'groom_residence':           (0.133, 0.437, 0.479, 0.463, _LINE),
+        'bride_residence':           (0.579, 0.439, 0.932, 0.466, _LINE),
+        'groom_religion':            (0.129, 0.465, 0.480, 0.494, _LINE),
+        'bride_religion':            (0.580, 0.464, 0.927, 0.490, _LINE),
+        'groom_civil_status':        (0.128, 0.493, 0.480, 0.518, _WORD),
+        'bride_civil_status':        (0.580, 0.493, 0.925, 0.517, _WORD),
+        'groom_father_name':         (0.132, 0.648, 0.477, 0.670, _LINE),
+        'groom_father_citizenship':  (0.128, 0.668, 0.475, 0.691, _LINE),
+        'bride_father_name':         (0.575, 0.649, 0.925, 0.670, _LINE),
+        'bride_father_citizenship':  (0.575, 0.671, 0.925, 0.693, _LINE),
+        'groom_mother_name':         (0.125, 0.740, 0.476, 0.762, _LINE),
+        'groom_mother_citizenship':  (0.122, 0.762, 0.477, 0.780, _LINE),
+        'bride_mother_name':         (0.575, 0.739, 0.923, 0.762, _LINE),
+        'bride_mother_citizenship':  (0.572, 0.760, 0.922, 0.780, _LINE),
+    },
+    '97': {
+        'province':                    (0.186, 0.092, 0.603, 0.113, _LINE),
+        'registry_no':                 (0.743, 0.094, 0.941, 0.129, _LINE),
+        'city_municipality':           (0.184, 0.112, 0.603, 0.132, _LINE),
+        'husband_name_first':          (0.244, 0.154, 0.553, 0.175, _LINE),
+        'husband_name_middle':         (0.245, 0.175, 0.549, 0.196, _LINE),
+        'husband_name_last':           (0.244, 0.198, 0.553, 0.215, _LINE),
+        'wife_name_first':             (0.631, 0.154, 0.940, 0.176, _LINE),
+        'wife_name_middle':            (0.630, 0.174, 0.941, 0.195, _LINE),
+        'wife_name_last':              (0.633, 0.197, 0.942, 0.216, _LINE),
+        'husband_dob':                 (0.191, 0.228, 0.475, 0.249, _LINE),
+        'husband_age':                 (0.480, 0.230, 0.543, 0.248, _WORD),
+        'wife_dob':                    (0.579, 0.226, 0.862, 0.248, _LINE),
+        'wife_age':                    (0.863, 0.228, 0.937, 0.248, _WORD),
+        'husband_place_of_birth':      (0.169, 0.259, 0.554, 0.279, _LINE),
+        'wife_place_of_birth':         (0.557, 0.258, 0.953, 0.280, _LINE),
+        'husband_sex':                 (0.211, 0.282, 0.309, 0.309, _WORD),
+        'wife_sex':                    (0.597, 0.281, 0.701, 0.310, _WORD),
+        'husband_citizenship':         (0.309, 0.290, 0.553, 0.310, _LINE),
+        'wife_citizenship':            (0.698, 0.289, 0.939, 0.310, _LINE),
+        'husband_residence':           (0.177, 0.324, 0.550, 0.361, _LINE),
+        'wife_residence':              (0.566, 0.323, 0.942, 0.362, _LINE),
+        'husband_religion':            (0.177, 0.363, 0.550, 0.391, _LINE),
+        'wife_religion':               (0.563, 0.363, 0.943, 0.387, _LINE),
+        'husband_civil_status':        (0.171, 0.392, 0.554, 0.416, _WORD),
+        'wife_civil_status':           (0.570, 0.395, 0.955, 0.415, _WORD),
+        'husband_father_name':         (0.181, 0.427, 0.551, 0.448, _LINE),
+        'wife_father_name':            (0.561, 0.425, 0.955, 0.446, _LINE),
+        'husband_father_citizenship':  (0.175, 0.449, 0.551, 0.466, _LINE),
+        'wife_father_citizenship':     (0.561, 0.447, 0.943, 0.467, _LINE),
+        'husband_mother_name':         (0.181, 0.476, 0.557, 0.496, _LINE),
+        'wife_mother_name':            (0.564, 0.477, 0.955, 0.499, _LINE),
+        'husband_mother_citizenship':  (0.184, 0.500, 0.550, 0.518, _LINE),
+        'wife_mother_citizenship':     (0.561, 0.499, 0.939, 0.518, _LINE),
+        'place_of_marriage':           (0.179, 0.640, 0.941, 0.665, _LINE),
+        'date_of_marriage':            (0.182, 0.674, 0.556, 0.696, _LINE),
+        'time_of_marriage':            (0.734, 0.674, 0.889, 0.696, _LINE),
+        'registration_date':           (0.655, 0.749, 0.935, 0.769, _LINE),
+    },
+}
+
+USE_SELECTIVE_PADDLE_ASSIST = True
+PADDLE_ASSIST_FIELDS = {
+    'province',
+    'registry_no',
+    'city_municipality',
+    'date_issued',
+    'registration_date',
+    'marriage_license_no',
+}
+
+
+def warmup():
+    print('[template_matcher] Warming up models and caches...')
+    _get_crnn()
+    _get_paddleocr()
+    for ft in REFERENCE_IMAGES:
+        img = _get_ref_gray(ft)
+        status = 'OK' if img is not None else 'NOT FOUND'
+        print(f'[template_matcher] Reference {ft}: {status}')
+    print('[template_matcher] Warmup complete.')
+
+
+def _order_corners(pts: np.ndarray) -> np.ndarray:
+    s = pts.sum(axis=1)
+    d = np.diff(pts, axis=1).flatten()
+    return np.array([
+        pts[np.argmin(s)],
+        pts[np.argmin(d)],
+        pts[np.argmax(s)],
+        pts[np.argmax(d)],
+    ], dtype=np.float32)
+
+
+def _correct_perspective(scan_rgb: np.ndarray, ref_w: int, ref_h: int) -> np.ndarray:
+    if not _CV2_OK:
+        return scan_rgb
+
+    gray = _cv2.cvtColor(scan_rgb, _cv2.COLOR_RGB2GRAY)
+    kernel = _cv2.getStructuringElement(_cv2.MORPH_RECT, (5, 5))
+    blur = _cv2.GaussianBlur(gray, (7, 7), 0)
+    _, thresh = _cv2.threshold(blur, 0, 255, _cv2.THRESH_BINARY + _cv2.THRESH_OTSU)
+    dilated = _cv2.dilate(thresh, kernel, iterations=2)
+    contours, _ = _cv2.findContours(dilated, _cv2.RETR_EXTERNAL, _cv2.CHAIN_APPROX_SIMPLE)
+
+    if not contours:
+        return scan_rgb
+
+    c = max(contours, key=_cv2.contourArea)
+    area = _cv2.contourArea(c)
+    if area < 0.30 * gray.shape[0] * gray.shape[1]:
+        print('[align] perspective: contour too small, skipping')
+        return scan_rgb
+
+    peri = _cv2.arcLength(c, True)
+    approx = _cv2.approxPolyDP(c, 0.02 * peri, True)
+    if len(approx) != 4:
+        print(f'[align] perspective: {len(approx)} corners (need 4), skipping')
+        return scan_rgb
+
+    src = _order_corners(approx.reshape(4, 2).astype(np.float32))
+    dst = np.array([
+        [0, 0], [ref_w - 1, 0],
+        [ref_w - 1, ref_h - 1], [0, ref_h - 1],
+    ], dtype=np.float32)
+
+    M = _cv2.getPerspectiveTransform(src, dst)
+    warped = _cv2.warpPerspective(
+        scan_rgb, M, (ref_w, ref_h),
+        flags=_cv2.INTER_LINEAR, borderMode=_cv2.BORDER_REPLICATE,
+    )
+    print('[align] perspective correction applied')
+    return warped
+
+
+def _ecc_align(scan_gray: np.ndarray, ref_gray: np.ndarray, scan_rgb: np.ndarray):
+    try:
+        h, w = ref_gray.shape
+        scale = min(1.0, 500.0 / max(h, w))
+        sh, sw = max(1, int(h * scale)), max(1, int(w * scale))
+
+        ref_s = _cv2.resize(ref_gray, (sw, sh))
+        scn_s = _cv2.resize(_cv2.resize(scan_gray, (w, h)), (sw, sh))
+
+        warp = np.eye(2, 3, dtype=np.float32)
+        criteria = (_cv2.TERM_CRITERIA_EPS | _cv2.TERM_CRITERIA_COUNT, 50, 1e-3)
+        cc, warp = _cv2.findTransformECC(ref_s, scn_s, warp, _cv2.MOTION_AFFINE, criteria)
+
+        if cc < 0.3:
+            print(f'[align] ECC low confidence (cc={cc:.4f}), skipping')
+            return None
+
+        angle = np.degrees(np.arctan2(warp[1, 0], warp[0, 0]))
+        if abs(angle) > 1.0:
+            clamped = np.radians(np.clip(angle, -1.0, 1.0))
+            warp[0, 0] = np.cos(clamped)
+            warp[0, 1] = -np.sin(clamped)
+            warp[1, 0] = np.sin(clamped)
+            warp[1, 1] = np.cos(clamped)
+
+        warp[0, 2] /= scale
+        warp[1, 2] /= scale
+
+        scan_full = _cv2.resize(scan_rgb, (w, h))
+        aligned = _cv2.warpAffine(
+            scan_full, warp, (w, h),
+            flags=_cv2.INTER_LINEAR, borderMode=_cv2.BORDER_REPLICATE,
+        )
+        print(f'[align] ECC applied (cc={cc:.4f} angle={angle:.2f}°)')
+        return aligned
+    except Exception as e:
+        print(f'[align] ECC failed: {e}')
+        return None
+
+
+def _orb_align(scan_gray: np.ndarray, ref_gray: np.ndarray, scan_rgb: np.ndarray):
+    h, w = scan_gray.shape
+    ref_resized = _cv2.resize(ref_gray, (w, h))
+
+    orb = _cv2.ORB_create(nfeatures=5000)
+    kp1, des1 = orb.detectAndCompute(scan_gray, None)
+    kp2, des2 = orb.detectAndCompute(ref_resized, None)
+
+    if des1 is None or des2 is None or len(kp1) < 10 or len(kp2) < 10:
+        return None, 0
+
+    matcher = _cv2.BFMatcher(_cv2.NORM_HAMMING, crossCheck=True)
+    matches = sorted(matcher.match(des1, des2), key=lambda m: m.distance)
+    good = matches[:max(10, len(matches) // 3)]
+
+    if len(good) < 6:
+        return None, 0
+
+    src_pts = np.float32([kp1[m.queryIdx].pt for m in good]).reshape(-1, 1, 2)
+    dst_pts = np.float32([kp2[m.trainIdx].pt for m in good]).reshape(-1, 1, 2)
+
+    M, mask = _cv2.estimateAffinePartial2D(
+        src_pts, dst_pts, method=_cv2.RANSAC, ransacReprojThreshold=5.0,
+    )
+    if M is None:
+        return None, 0
+
+    inliers = int(mask.sum()) if mask is not None else 0
+    aligned = _cv2.warpAffine(
+        scan_rgb, M, (w, h),
+        flags=_cv2.INTER_LINEAR, borderMode=_cv2.BORDER_REPLICATE,
+    )
+    print(f'[align] ORB applied ({inliers} inliers)')
+    return aligned, inliers
+
+
+def _orb_inliers(scan_gray: np.ndarray, ref_gray: np.ndarray) -> int:
+    orb = _cv2.ORB_create(nfeatures=3000)
+    kp1, des1 = orb.detectAndCompute(scan_gray, None)
+    kp2, des2 = orb.detectAndCompute(ref_gray, None)
+
+    if des1 is None or des2 is None or len(kp1) < 10 or len(kp2) < 10:
+        return 0
+
+    matcher = _cv2.BFMatcher(_cv2.NORM_HAMMING, crossCheck=True)
+    matches = sorted(matcher.match(des1, des2), key=lambda m: m.distance)
+    good = matches[:max(10, len(matches) // 3)]
+
+    if len(good) < 6:
+        return 0
+
+    src_pts = np.float32([kp1[m.queryIdx].pt for m in good]).reshape(-1, 1, 2)
+    dst_pts = np.float32([kp2[m.trainIdx].pt for m in good]).reshape(-1, 1, 2)
+
+    _, mask = _cv2.findHomography(src_pts, dst_pts, _cv2.RANSAC, 5.0)
+    return int(mask.sum()) if mask is not None else 0
+
+
+def check_image_quality(image_path: str, form_type: str) -> dict:
+    if not _CV2_OK:
+        return {
+            'ok': True,
+            'upside_down': False,
+            'skew_angle': 0.0,
+            'aspect_mismatch': 1.0,
+            'orb_fit': 0,
+            'orb_fit_normal': 0,
+            'orb_fit_180': 0,
+            'blur_score': 9999.0,
+            'warnings': ['OpenCV not available; skipping quality check'],
+        }
+
+    result = {}
+    warnings = []
+
+    try:
+        img = Image.open(image_path).convert('RGB')
+    except Exception as e:
+        return {
+            'ok': False,
+            'upside_down': False,
+            'skew_angle': 0.0,
+            'aspect_mismatch': 0.0,
+            'orb_fit': 0,
+            'orb_fit_normal': 0,
+            'orb_fit_180': 0,
+            'blur_score': 0.0,
+            'warnings': [f'Cannot open image: {e}'],
+        }
+
+    scan_rgb = np.array(img)
+    scan_gray = _cv2.cvtColor(scan_rgb, _cv2.COLOR_RGB2GRAY)
+    h, w = scan_gray.shape
+
+    blur_score = float(_cv2.Laplacian(scan_gray, _cv2.CV_64F).var())
+    result['blur_score'] = round(blur_score, 1)
+    if blur_score < 80:
+        warnings.append(
+            f'Image appears blurry (Laplacian variance={blur_score:.1f}; threshold 80).'
+        )
+
+    edges = _cv2.Canny(scan_gray, 50, 150, apertureSize=3)
+    lines = _cv2.HoughLinesP(
+        edges, 1, np.pi / 180, threshold=80,
+        minLineLength=60, maxLineGap=15,
+    )
+    skew_angle = 0.0
+    if lines is not None:
+        angles = [
+            np.degrees(np.arctan2(y2 - y1, x2 - x1))
+            for x1, y1, x2, y2 in lines[:, 0]
+            if abs(np.degrees(np.arctan2(y2 - y1, x2 - x1))) < 45
+        ]
+        if angles:
+            skew_angle = float(np.median(angles))
+
+    result['skew_angle'] = round(skew_angle, 2)
+    if abs(skew_angle) > 3.0:
+        warnings.append(f'Page is significantly skewed ({skew_angle:.1f}°).')
+
+    upside_down = False
+    orb_fit = 0
+    inliers_normal = 0
+    inliers_180 = 0
+
+    ref_gray = _get_ref_gray(form_type)
+    if ref_gray is not None:
+        ref_h, ref_w = ref_gray.shape
+        scan_rs = _cv2.resize(scan_gray, (ref_w, ref_h))
+        scan_180 = _cv2.rotate(scan_rs, _cv2.ROTATE_180)
+
+        inliers_normal = _orb_inliers(scan_rs, ref_gray)
+        inliers_180 = _orb_inliers(scan_180, ref_gray)
+        orb_fit = inliers_normal
+
+        if inliers_180 > inliers_normal * 1.5 and inliers_180 > 10:
+            upside_down = True
+            orb_fit = inliers_180
+            warnings.append(
+                f'Image appears upside down (ORB normal={inliers_normal}, rotated_180={inliers_180}).'
+            )
+
+        if orb_fit < 10:
+            warnings.append(f'Poor alignment fit for form {form_type} (ORB inliers={orb_fit}).')
+        elif orb_fit < 25:
+            warnings.append(f'Weak alignment fit for form {form_type} (ORB inliers={orb_fit}).')
+
+        scan_aspect = w / max(h, 1)
+        ref_aspect = ref_w / max(ref_h, 1)
+        aspect_ratio = scan_aspect / max(ref_aspect, 1e-6)
+        result['aspect_mismatch'] = round(aspect_ratio, 3)
+    else:
+        result['aspect_mismatch'] = 1.0
+
+    result['upside_down'] = upside_down
+    result['orb_fit'] = orb_fit
+    result['orb_fit_normal'] = inliers_normal
+    result['orb_fit_180'] = inliers_180
+    result['warnings'] = warnings
+    result['ok'] = len(warnings) == 0
+    return result
+
+
+def correct_image(img: Image.Image, quality: dict):
+    applied = []
+
+    if not _CV2_OK:
+        print('[correct_image] OpenCV not available; skipping corrections.')
+        return img, applied
+
+    rgb = np.array(img.convert('RGB'))
+
+    if quality.get('upside_down'):
+        rgb = _cv2.rotate(rgb, _cv2.ROTATE_180)
+        applied.append('rotated 180° (upside-down correction)')
+        print('[correct_image] Applied: 180° rotation')
+
+    skew_angle = quality.get('skew_angle', 0.0)
+    if 1.0 < abs(skew_angle) < 15.0:
+        correction_angle = -skew_angle
+        h, w = rgb.shape[:2]
+        center = (w / 2.0, h / 2.0)
+        M = _cv2.getRotationMatrix2D(center, correction_angle, 1.0)
+
+        cos_a = abs(M[0, 0])
+        sin_a = abs(M[0, 1])
+        new_w = int(h * sin_a + w * cos_a)
+        new_h = int(h * cos_a + w * sin_a)
+        M[0, 2] += (new_w - w) / 2.0
+        M[1, 2] += (new_h - h) / 2.0
+
+        rgb = _cv2.warpAffine(
+            rgb, M, (new_w, new_h),
+            flags=_cv2.INTER_CUBIC,
+            borderMode=_cv2.BORDER_REPLICATE,
+        )
+        applied.append(f'de-skewed {correction_angle:+.2f}°')
+        print(f'[correct_image] Applied: de-skew {correction_angle:+.2f}°')
+
+    result_img = Image.fromarray(rgb)
+    if img.mode != 'RGB':
+        result_img = result_img.convert(img.mode)
+    return result_img, applied
+
+
+def align_to_reference(img: Image.Image, form_type: str):
+    if not _CV2_OK:
+        return img, 0
+
+    ref_gray = _get_ref_gray(form_type)
+    if ref_gray is None:
+        return img, 0
+
+    ref_h, ref_w = ref_gray.shape
+    scan_rgb = np.array(img.convert('RGB'))
+
+    stage0 = _correct_perspective(scan_rgb, ref_w, ref_h)
+    stage0_gray = _cv2.cvtColor(stage0, _cv2.COLOR_RGB2GRAY)
+
+    precheck = _orb_inliers(stage0_gray, ref_gray)
+    print(f'[align] ORB pre-check: {precheck} inliers')
+
+    if precheck >= 40:
+        orb_aligned, orb_inliers = _orb_align(stage0_gray, ref_gray, stage0)
+        if orb_aligned is not None:
+            return Image.fromarray(orb_aligned), orb_inliers
+
+    ecc_aligned = _ecc_align(stage0_gray, ref_gray, stage0)
+    if ecc_aligned is not None:
+        ecc_gray = _cv2.cvtColor(ecc_aligned, _cv2.COLOR_RGB2GRAY)
+        orb_aligned, orb_inliers = _orb_align(ecc_gray, ref_gray, ecc_aligned)
+        if orb_aligned is not None:
+            return Image.fromarray(orb_aligned), orb_inliers
+        return Image.fromarray(ecc_aligned), _orb_inliers(ecc_gray, ref_gray)
+
+    orb_aligned, orb_inliers = _orb_align(stage0_gray, ref_gray, stage0)
+    if orb_aligned is not None:
+        return Image.fromarray(orb_aligned), orb_inliers
+
+    resized = _cv2.resize(stage0, (ref_w, ref_h))
+    return Image.fromarray(resized), precheck
+
+
+def _deskew(gray: np.ndarray) -> np.ndarray:
+    if not _CV2_OK:
+        return gray
+
+    edges = _cv2.Canny(gray, 50, 150, apertureSize=3)
+    lines = _cv2.HoughLinesP(
+        edges, 1, np.pi / 180, threshold=100,
+        minLineLength=100, maxLineGap=10,
+    )
+    if lines is None:
+        return gray
+
+    angles = [
+        np.degrees(np.arctan2(y2 - y1, x2 - x1))
+        for x1, y1, x2, y2 in lines[:, 0]
+        if -3 < np.degrees(np.arctan2(y2 - y1, x2 - x1)) < 3
+    ]
+
+    if not angles:
+        return gray
+
+    angle = float(np.median(angles))
+    if abs(angle) < 0.5:
+        return gray
+
+    h, w = gray.shape
+    M = _cv2.getRotationMatrix2D((w / 2, h / 2), angle, 1.0)
+    return _cv2.warpAffine(
+        gray, M, (w, h),
+        flags=_cv2.INTER_CUBIC, borderMode=_cv2.BORDER_REPLICATE,
+    )
+
+
+def _preprocess(img: Image.Image) -> Image.Image:
+    if not _CV2_OK:
+        return img.convert('L')
+    gray = np.array(img.convert('L'))
+    gray = _deskew(gray)
+    return Image.fromarray(gray)
+
+
+def _crop_field(img: Image.Image, x1r, y1r, x2r, y2r) -> Image.Image:
+    w, h = img.size
+    pad = 4
+    x1 = max(0, int(x1r * w) - pad)
+    y1 = max(0, int(y1r * h) - pad)
+    x2 = min(w, int(x2r * w) + pad)
+    y2 = min(h, int(y2r * h) + pad)
+    return img.crop((x1, y1, x2, y2))
+
+
+def _expand_box(box, img_w, img_h, pad_x=10, pad_y=8):
+    x1, y1, x2, y2 = box
+    return (
+        max(0, x1 - pad_x),
+        max(0, y1 - pad_y),
+        min(img_w, x2 + pad_x),
+        min(img_h, y2 + pad_y),
+    )
+
+
+def _crop_from_box(img: Image.Image, box):
+    return img.crop(box)
+
+
+def _norm_text(s: str) -> str:
+    return _re.sub(r'[^a-z0-9]+', '', (s or '').lower())
+
+
+def _find_nearby_detection(field_rect, detections, expected_hint=None):
+    fx1, fy1, fx2, fy2 = field_rect
+    fcx = (fx1 + fx2) / 2
+    fcy = (fy1 + fy2) / 2
+    fw = max(1, fx2 - fx1)
+    fh = max(1, fy2 - fy1)
+
+    best = None
+    best_score = -1e9
+
+    for det in detections:
+        x1, y1, x2, y2 = det['box']
+        dcx = det['cx']
+        dcy = det['cy']
+        dw = max(1, x2 - x1)
+        dh = max(1, y2 - y1)
+
+        dist = ((dcx - fcx) ** 2 + (dcy - fcy) ** 2) ** 0.5
+        overlap_x = max(0, min(fx2, x2) - max(fx1, x1))
+        overlap_y = max(0, min(fy2, y2) - max(fy1, y1))
+        overlap = overlap_x * overlap_y
+
+        size_penalty = abs(dw - fw) * 0.2 + abs(dh - fh) * 0.2
+        score = overlap * 0.02 - dist - size_penalty + det.get('conf', 0.0) * 40.0
+
+        text = (det.get('text') or '').strip()
+        if expected_hint == _WORD and len(text.split()) <= 3:
+            score += 10
+        elif expected_hint == _LINE and 1 <= len(text.split()) <= 12:
+            score += 8
+        elif expected_hint == _BLOCK and len(text.split()) >= 2:
+            score += 6
+
+        if score > best_score:
+            best_score = score
+            best = det
+
+    return best if best_score > -150 else None
+
+
+def _get_field_crop_with_paddle(processed_img: Image.Image, field_coords, detections):
+    w, h = processed_img.size
+    x1r, y1r, x2r, y2r, hint = field_coords
+
+    fx1 = int(x1r * w)
+    fy1 = int(y1r * h)
+    fx2 = int(x2r * w)
+    fy2 = int(y2r * h)
+    field_rect = (fx1, fy1, fx2, fy2)
+
+    det = _find_nearby_detection(field_rect, detections, expected_hint=hint)
+    if det is not None:
+        box = _expand_box(det['box'], w, h, pad_x=10, pad_y=8)
+        return _crop_from_box(processed_img, box), 'paddle-detect', det
+
+    return _crop_field(processed_img, x1r, y1r, x2r, y2r), 'absolute', None
+
+
+def _get_field_crop_with_easyocr(processed_img: Image.Image, field_coords, detections):
+    return _get_field_crop_with_paddle(processed_img, field_coords, detections)
+
+
+def detect_form_type(image_path: str) -> str:
+    if _CV2_OK:
+        try:
+            img = Image.open(image_path).convert('RGB')
+            scan_rgb = np.array(img)
+            scan_gray = _cv2.cvtColor(scan_rgb, _cv2.COLOR_RGB2GRAY)
+
+            best_type, best_inliers = None, 0
+            det_w = 800
+
+            for ft in REFERENCE_IMAGES:
+                ref_gray = _get_ref_gray(ft)
+                if ref_gray is None:
+                    continue
+
+                ref_h, ref_w = ref_gray.shape
+                sc = min(1.0, det_w / ref_w)
+                dw = max(1, int(ref_w * sc))
+                dh = max(1, int(ref_h * sc))
+                ref_ds = _cv2.resize(ref_gray, (dw, dh))
+                scan_ds = _cv2.resize(_cv2.resize(scan_gray, (ref_w, ref_h)), (dw, dh))
+
+                count = _orb_inliers(scan_ds, ref_ds)
+                print(f'[detect] Form {ft}: {count} ORB inliers')
+
+                if count > best_inliers:
+                    best_inliers, best_type = count, ft
+
+            if best_type and best_inliers >= 15:
+                print(f'[detect] Best: Form {best_type} ({best_inliers} inliers)')
+                return best_type
+
+            print(f'[detect] ORB inconclusive ({best_inliers}), trying OCR title')
+        except Exception as e:
+            print(f'[template_matcher] detect_form_type ORB error: {e}')
+
+    try:
+        img_l = Image.open(image_path).convert('L')
+        w, h = img_l.size
+        title_crop = img_l.crop((0, int(h * 0.04), w, int(h * 0.15)))
+        title = _crnn_read(title_crop).upper()
+
+        if title:
+            if 'LIVE BIRTH' in title or ('BIRTH' in title and 'DEATH' not in title and 'MARRIAGE' not in title):
+                return '102'
+            if 'DEATH' in title:
+                return '103'
+            if 'MARRIAGE' in title and 'LICENSE' in title:
+                return '90'
+            if 'MARRIAGE' in title:
+                return '97'
+    except Exception as e:
+        print(f'[template_matcher] detect_form_type OCR error: {e}')
+
+    print('[detect] Could not detect form type; defaulting to 102.')
+    return '102'
+
+
+def is_blank_image(img: Image.Image, threshold: float = 0.995) -> bool:
+    if not _CV2_OK:
+        return False
+
+    gray = np.array(img.convert('L'))
+    h, w = gray.shape
+
+    y1 = int(h * 0.20)
+    y2 = int(h * 0.80)
+    x1 = int(w * 0.20)
+    x2 = int(w * 0.80)
+    center = gray[y1:y2, x1:x2]
+
+    light_pixels = np.sum(center > 240)
+    total_pixels = center.size
+    ratio = light_pixels / max(total_pixels, 1)
+    variance = float(np.var(center))
+
+    print(f'[template_matcher] Blank check: {ratio:.2%} light pixels, variance={variance:.1f}')
+    return ratio >= threshold and variance < 50.0
+
+
+def extract_fields(image_path: str, form_type: str = None):
+    try:
+        if not form_type:
+            form_type = detect_form_type(image_path)
+
+        template = TEMPLATES.get(form_type)
+        if not template:
+            return {'status': 'error', 'message': f'No template for form {form_type}.'}
+
+        quality = check_image_quality(image_path, form_type)
+        img = Image.open(image_path).convert('RGB')
+
+        if is_blank_image(img):
+            return {'status': 'error', 'message': 'Blank or near-blank image detected.'}
+
+        img, corrections = correct_image(img, quality)
+        img, orb_fit = align_to_reference(img, form_type)
+        processed = _preprocess(img)
+        detections = _paddle_detect(processed)
+
+        fields = {}
+        debug_methods = {}
+        field_names = []
+        crops = []
+        assist_texts = []
+
+        for field_name, coords in template.items():
+            crop, method, det = _get_field_crop_with_paddle(processed, coords, detections)
+            field_names.append(field_name)
+            crops.append(crop)
+            debug_methods[field_name] = method
+
+            assist_text = ''
+            if USE_SELECTIVE_PADDLE_ASSIST and field_name in PADDLE_ASSIST_FIELDS:
+                if det is not None:
+                    assist_text = (det.get('text') or '').strip()
+                if not assist_text:
+                    assist_text = _paddle_read(crop)
+            assist_texts.append(assist_text)
+
+        crnn_texts = _crnn_read_batch(crops)
+
+        for field_name, crnn_text, assist_text in zip(field_names, crnn_texts, assist_texts):
+            final_text = _smart_merge(field_name, crnn_text, assist_text)
+            if final_text:
+                fields[field_name] = final_text
+
+        print(f'[template_matcher] Extracted: {len(fields)}/{len(template)} fields')
+        paddle_count = sum(1 for m in debug_methods.values() if m == 'paddle-detect')
+        abs_count = sum(1 for m in debug_methods.values() if m == 'absolute')
+        print(f'[template_matcher] Crop source: paddle={paddle_count}, absolute={abs_count}, orb_fit={orb_fit}')
+
+        if len(fields) == 0:
+            return {'status': 'error', 'message': 'No readable text found.'}
+
+        fields['_quality'] = quality
+        fields['_corrections'] = corrections
+        return fields
+    except Exception as e:
+        print(f'[template_matcher] extract_fields error: {e}')
+        return {'status': 'error', 'message': str(e)}
+
+
+def debug_draw_boxes(image_path: str, form_type: str, out_path: str = None) -> str:
+    from PIL import ImageDraw, ImageFont
+
+    template = TEMPLATES.get(form_type)
+    if not template:
+        print(f'No template for {form_type}')
+        return None
+
+    quality = check_image_quality(image_path, form_type)
+    img = Image.open(image_path).convert('RGB')
+    img, _ = correct_image(img, quality)
+    img, _ = align_to_reference(img, form_type)
+
+    draw = ImageDraw.Draw(img)
+    w, h = img.size
+
+    try:
+        font = ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf', 11)
+    except Exception:
+        try:
+            font = ImageFont.truetype('C:/Windows/Fonts/arial.ttf', 11)
+        except Exception:
+            font = ImageFont.load_default()
+
+    for field_name, coords in template.items():
+        x1r, y1r, x2r, y2r, _ = coords
+        bx1, by1 = int(x1r * w), int(y1r * h)
+        bx2, by2 = int(x2r * w), int(y2r * h)
+        draw.rectangle([bx1, by1, bx2, by2], outline='#1a6fd4', width=1)
+        draw.text((bx1 + 2, by1 + 2), field_name, fill='#1a6fd4', font=font)
+
+    base, ext = os.path.splitext(image_path)
+    out = out_path or f'{base}_debug_{form_type}{ext}'
+    img.save(out)
+    print(f'[template_matcher] Debug image saved: {out}')
+    return out
+
+
+def debug_draw_paddle_matches(image_path: str, form_type: str, out_path: str = None) -> str:
+    from PIL import ImageDraw, ImageFont
+
+    template = TEMPLATES.get(form_type)
+    if not template:
+        print(f'No template for {form_type}')
+        return None
+
+    quality = check_image_quality(image_path, form_type)
+    img = Image.open(image_path).convert('RGB')
+    img, _ = correct_image(img, quality)
+    img, _ = align_to_reference(img, form_type)
+    processed = _preprocess(img)
+    detections = _paddle_detect(processed)
+
+    canvas = img.copy()
+    draw = ImageDraw.Draw(canvas)
+    w, h = canvas.size
+
+    try:
+        font = ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf', 11)
+    except Exception:
+        try:
+            font = ImageFont.truetype('C:/Windows/Fonts/arial.ttf', 11)
+        except Exception:
+            font = ImageFont.load_default()
+
+    for det in detections:
+        x1, y1, x2, y2 = det['box']
+        draw.rectangle([x1, y1, x2, y2], outline='red', width=1)
+
+    for field_name, coords in template.items():
+        x1r, y1r, x2r, y2r, hint = coords
+        fx1 = int(x1r * w)
+        fy1 = int(y1r * h)
+        fx2 = int(x2r * w)
+        fy2 = int(y2r * h)
+        draw.rectangle([fx1, fy1, fx2, fy2], outline='blue', width=2)
+        draw.text((fx1 + 2, fy1 + 2), field_name, fill='blue', font=font)
+
+        det = _find_nearby_detection((fx1, fy1, fx2, fy2), detections, expected_hint=hint)
+        if det is not None:
+            dx1, dy1, dx2, dy2 = det['box']
+            draw.rectangle([dx1, dy1, dx2, dy2], outline='green', width=2)
+
+    base, ext = os.path.splitext(image_path)
+    out = out_path or f'{base}_paddle_debug_{form_type}{ext}'
+    canvas.save(out)
+    print(f'[template_matcher] Paddle debug image saved: {out}')
+    return out
+
+
+def debug_draw_easyocr_matches(image_path: str, form_type: str, out_path: str = None) -> str:
+    # Backward-compatible function name.
+    return debug_draw_paddle_matches(image_path, form_type, out_path)
+
+
+def pdf_to_image(pdf_path: str, page: int = 0) -> str:
+    try:
+        from pdf2image import convert_from_path
+        pages = convert_from_path(pdf_path, dpi=150)
+        out_path = pdf_path.replace('.pdf', f'_page{page}.png')
+        pages[page].save(out_path, 'PNG')
+        return out_path
+    except ImportError:
+        print('[template_matcher] pdf2image not installed.')
+        return None
+    except Exception as e:
+        print(f'[template_matcher] PDF conversion failed: {e}')
+        return None
+
+
+if __name__ == '__main__':
+    warmup()
+
+    if len(sys.argv) < 2:
+        print('Usage:')
+        print('  python template_matcher.py <image_path> <form_type> [out_path]')
+        print('  python template_matcher.py <image_path> check [form_type]')
+        print('  form_type: 102 | 103 | 90 | 97')
+        sys.exit(1)
+
+    img_path = sys.argv[1]
+
+    if len(sys.argv) >= 3 and sys.argv[2] == 'check':
+        ft = sys.argv[3] if len(sys.argv) > 3 else detect_form_type(img_path)
+        q = check_image_quality(img_path, ft)
+
+        print(f'\nQuality report for form {ft}:')
+        for k, v in q.items():
+            if k != 'warnings':
+                print(f'  {k:<22} = {v}')
+
+        if q['warnings']:
+            print('\nWarnings:')
+            for msg in q['warnings']:
+                print(f'  • {msg}')
+
+        img_pil = Image.open(img_path).convert('RGB')
+        _, corrections = correct_image(img_pil, q)
+        print('\nCorrections that would be applied:')
+        if corrections:
+            for c in corrections:
+                print(f'  ✓ {c}')
+        else:
+            print('  (none needed)')
+
+        sys.exit(0 if q['ok'] else 1)
+
+    form_type = sys.argv[2]
+    out_path = sys.argv[3] if len(sys.argv) > 3 else None
+
+    debug_draw_boxes(img_path, form_type, out_path)
+    debug_draw_paddle_matches(img_path, form_type)
+
+    result = extract_fields(img_path, form_type)
+    meta_keys = {'_quality', '_corrections'}
+
+    data_fields = {k: v for k, v in result.items() if k not in meta_keys}
+    print(f'\nExtracted fields ({len(data_fields)}):')
+    for k, v in data_fields.items():
+        print(f'  {k:<40} = {v}')
+
+    template = TEMPLATES.get(form_type, {})
+    missing = [k for k in template if k not in data_fields]
+    if missing:
+        print(f'\nEmpty fields ({len(missing)}):')
+        for k in missing:
+            print(f'  {k}')
+
+    corrections = result.get('_corrections', [])
+    if corrections:
+        print('\nAuto-corrections applied:')
+        for c in corrections:
+            print(f'  ✓ {c}')
+
+    quality = result.get('_quality', {})
+    if quality.get('warnings'):
+        print('\nQuality warnings:')
+        for w_msg in quality['warnings']:
+            print(f'  • {w_msg}')

test_flask.html

@@ -0,0 +1,299 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Flask Pipeline Test</title>
+    <link href="https://fonts.googleapis.com/css2?family=Poppins:wght@400;600&display=swap" rel="stylesheet">
+    <style>
+        * { box-sizing: border-box; margin: 0; padding: 0; }
+        body { font-family: 'Poppins', sans-serif; background: #f0fdf7; padding: 32px; color: #1a1a1a; }
+        h2  { color: #1a7a4a; margin-bottom: 6px; }
+        p   { color: #555; font-size: 14px; margin-bottom: 20px; }
+
+        .controls { display: flex; gap: 12px; align-items: center; flex-wrap: wrap; margin-bottom: 20px; }
+        button {
+            padding: 10px 22px; background: #1ec77c; color: white;
+            border: none; border-radius: 8px; font-size: 14px;
+            font-family: 'Poppins', sans-serif; font-weight: 600; cursor: pointer;
+        }
+        button:hover { background: #18a868; }
+        .btn-open { background: #3498db; }
+        .btn-open:hover { background: #2176ae; }
+
+        .status { font-weight: 600; font-size: 14px; }
+        .success { color: #1a7a4a; }
+        .error   { color: #e74c3c; }
+
+        /* ── Pipeline diagram ── */
+        .pipeline {
+            display: flex; align-items: center; gap: 0;
+            background: white; border-radius: 12px; padding: 20px 24px;
+            box-shadow: 0 2px 12px rgba(0,0,0,0.07); margin-bottom: 24px;
+            flex-wrap: wrap; gap: 8px;
+        }
+        .pipe-step {
+            background: #f0fdf7; border: 2px solid #1ec77c; border-radius: 8px;
+            padding: 10px 16px; text-align: center; font-size: 12px; min-width: 110px;
+        }
+        .pipe-step .title { font-weight: 700; color: #1a7a4a; font-size: 13px; }
+        .pipe-step .sub   { color: #888; font-size: 11px; margin-top: 2px; }
+        .pipe-step.active { background: #1ec77c; border-color: #1a7a4a; }
+        .pipe-step.active .title { color: white; }
+        .pipe-step.active .sub   { color: #d0f5e8; }
+        .arrow { font-size: 20px; color: #1ec77c; font-weight: bold; padding: 0 4px; }
+
+        /* ── Results panel ── */
+        .results { display: none; }
+        .card {
+            background: white; border-radius: 12px; padding: 24px;
+            box-shadow: 0 2px 12px rgba(0,0,0,0.07); margin-bottom: 20px;
+        }
+        .card h3 { color: #1a7a4a; margin-bottom: 14px; font-size: 15px; display: flex; align-items: center; gap: 8px; }
+
+        /* ── Form 102 raw fields ── */
+        .raw-table { width: 100%; border-collapse: collapse; font-size: 13px; }
+        .raw-table th { background: #f5f5f5; padding: 7px 12px; text-align: left; color: #555; font-weight: 600; border-bottom: 2px solid #eee; }
+        .raw-table td { padding: 6px 12px; border-bottom: 1px solid #f0f0f0; }
+        .raw-table td:first-child { color: #888; font-size: 12px; }
+        .raw-table td:last-child  { font-weight: 600; }
+
+        /* ── Mapping arrows ── */
+        .mapping { display: flex; flex-direction: column; gap: 6px; font-size: 13px; }
+        .map-row  { display: flex; align-items: center; gap: 10px; padding: 5px 8px; border-radius: 6px; background: #f9f9f9; }
+        .map-from { color: #888; min-width: 200px; font-size: 12px; }
+        .map-arr  { color: #1ec77c; font-weight: bold; font-size: 16px; }
+        .map-to   { color: #1a7a4a; font-weight: 600; min-width: 200px; }
+        .map-val  { color: #111; font-size: 12px; background: #fffde7; padding: 2px 8px; border-radius: 4px; border: 1px solid #f0d000; }
+
+        /* ── Confidence bars ── */
+        .conf-row  { display: flex; align-items: center; gap: 10px; margin-bottom: 7px; font-size: 12px; }
+        .conf-lbl  { min-width: 210px; color: #555; }
+        .conf-wrap { flex: 1; background: #eee; border-radius: 99px; height: 10px; overflow: hidden; }
+        .conf-bar  { height: 100%; border-radius: 99px; transition: width 0.5s; }
+        .conf-pct  { min-width: 40px; text-align: right; font-weight: 700; }
+
+        /* ── Saved file ── */
+        .file-box { display: flex; align-items: center; gap: 14px; padding: 14px 18px; background: #f0fdf7; border: 2px solid #1ec77c; border-radius: 10px; }
+        .file-box .icon { font-size: 28px; }
+        .file-box .name { font-weight: 700; color: #1a7a4a; font-size: 14px; }
+        .file-box .path { font-size: 12px; color: #888; margin-top: 2px; }
+    </style>
+</head>
+<body>
+
+<h2>🧪 Flask Pipeline Test</h2>
+<p>Simulates scanning <strong>Form 102</strong> → CRNN+CTC extracts fields → MNB classifies → NER maps to <strong>Form 1A</strong> → saved to <code>uploads/temp/</code></p>
+
+<!-- Pipeline diagram -->
+<div class="pipeline">
+    <div class="pipe-step" id="step1"><div class="title">📄 Form 102</div><div class="sub">Scanned input</div></div>
+    <div class="arrow">→</div>
+    <div class="pipe-step" id="step2"><div class="title">🔍 CRNN+CTC</div><div class="sub">OCR extraction</div></div>
+    <div class="arrow">→</div>
+    <div class="pipe-step" id="step3"><div class="title">📊 MNB</div><div class="sub">Classification</div></div>
+    <div class="arrow">→</div>
+    <div class="pipe-step" id="step4"><div class="title">🏷️ NER</div><div class="sub">Field mapping</div></div>
+    <div class="arrow">→</div>
+    <div class="pipe-step" id="step5"><div class="title">📋 Form 1A</div><div class="sub">Auto-filled</div></div>
+    <div class="arrow">→</div>
+    <div class="pipe-step" id="step6"><div class="title">💾 Saved</div><div class="sub">uploads/temp/</div></div>
+</div>
+
+<div class="controls">
+    <button onclick="testFlask()">▶ Run Pipeline Test</button>
+    <span id="status" class="status"></span>
+</div>
+
+<div class="results" id="results">
+
+    <!-- Step 1+2: Form 102 raw fields -->
+    <div class="card">
+        <h3>📄 Step 1+2 — Form 102 raw fields extracted by CRNN+CTC</h3>
+        <table class="raw-table">
+            <thead><tr><th>Form 102 Field Name</th><th>Extracted Value</th></tr></thead>
+            <tbody id="rawFieldsTable"></tbody>
+        </table>
+    </div>
+
+    <!-- Step 3+4: NER mapping -->
+    <div class="card">
+        <h3>🏷️ Step 3+4 — MNB classified as <span id="formClassBadge" style="background:#1ec77c;color:white;padding:2px 10px;border-radius:99px;font-size:13px"></span> &nbsp;→ NER mapped fields</h3>
+        <div class="mapping" id="mappingRows"></div>
+    </div>
+
+    <!-- Confidence scores -->
+    <div class="card">
+        <h3>📊 NER Confidence Scores</h3>
+        <div id="confidenceBars"></div>
+    </div>
+
+    <!-- Saved file -->
+    <div class="card">
+        <h3>💾 Saved to uploads/temp/</h3>
+        <div class="file-box">
+            <div class="icon">📄</div>
+            <div>
+                <div class="name" id="savedFileName"></div>
+                <div class="path" id="savedFilePath"></div>
+            </div>
+            <button class="btn-open" onclick="openPreview()" style="margin-left:auto">👁 Open Form 1A Preview</button>
+        </div>
+    </div>
+
+</div>
+
+<script>
+// Form 102 field → Form 1A field name mapping (for display)
+const mappingDef = {
+    'child_first_name + child_middle_name + child_last_name': 'child_name',
+    'sex':               'sex',
+    'date_of_birth':     'date_of_birth',
+    'place_of_birth':    'place_of_birth',
+    'mother_first_name + mother_middle_name + mother_last_name': 'mother_name',
+    'mother_citizenship':'mother_nationality',
+    'father_first_name + father_middle_name + father_last_name': 'father_name',
+    'father_citizenship':'father_nationality',
+    'date_of_marriage':  'parents_marriage_date',
+    'place_of_marriage': 'parents_marriage_place',
+    'registry_number':   'registry_number',
+    'date_of_registration': 'date_of_registration',
+    'civil_registrar':   'verified_by',
+    'civil_registrar_position': 'verified_position',
+};
+
+const fieldLabels = {
+    child_name:'child_name → Name of Child',
+    sex:'sex → Sex',
+    date_of_birth:'date_of_birth → Date of Birth',
+    place_of_birth:'place_of_birth → Place of Birth',
+    mother_name:'mother_name → Name of Mother',
+    mother_nationality:'mother_nationality → Nationality of Mother',
+    father_name:'father_name → Name of Father',
+    father_nationality:'father_nationality → Nationality of Father',
+    parents_marriage_date:'parents_marriage_date → Date of Marriage of Parents',
+    parents_marriage_place:'parents_marriage_place → Place of Marriage of Parents',
+    registry_number:'registry_number → Registry Number',
+    date_of_registration:'date_of_registration → Date of Registration',
+    verified_by:'verified_by → Verified By',
+    verified_position:'verified_position → Position',
+};
+
+let _previewUrl = '';
+
+async function testFlask() {
+    const status  = document.getElementById('status');
+    const results = document.getElementById('results');
+    status.textContent = '⏳ Processing...';
+    status.className   = 'status';
+    results.style.display = 'none';
+
+    // Animate pipeline steps
+    const steps = ['step1','step2','step3','step4','step5','step6'];
+    steps.forEach(s => document.getElementById(s).classList.remove('active'));
+    let i = 0;
+    const anim = setInterval(() => {
+        if (i < steps.length) document.getElementById(steps[i++]).classList.add('active');
+        else clearInterval(anim);
+    }, 300);
+
+    const fakeFile = new File(['fake'], 'form102_scan.jpg', { type: 'image/jpeg' });
+    const formData = new FormData();
+    formData.append('file', fakeFile);
+    formData.append('type', 'cert');
+    formData.append('form_hint', '1A');
+
+    try {
+        const res  = await fetch('http://127.0.0.1:5000/process', { method: 'POST', body: formData });
+        const data = await res.json();
+
+        if (data.status !== 'success') {
+            status.innerHTML = '⚠️ ' + data.message;
+            status.className = 'status error';
+            return;
+        }
+
+        // ── Populate raw fields table ─────────────────────────
+        const rawBody = document.getElementById('rawFieldsTable');
+        rawBody.innerHTML = '';
+        // Parse raw_text back into key-value (it's a stringified dict from Python)
+        // We'll show hardcoded Form 102 fields since app.py defines them
+        const form102Fields = {
+            "child_first_name":  "Maria Luisa",
+            "child_middle_name": "Dela Cruz",
+            "child_last_name":   "Santos",
+            "sex":               "Female",
+            "date_of_birth":     "January 10, 2026",
+            "place_of_birth":    "Tarlac City, Tarlac",
+            "mother_first_name": "Rosa",
+            "mother_middle_name":"Reyes",
+            "mother_last_name":  "Dela Cruz",
+            "mother_citizenship":"Filipino",
+            "father_first_name": "Juan",
+            "father_middle_name":"Pedro",
+            "father_last_name":  "Santos",
+            "father_citizenship":"Filipino",
+            "date_of_marriage":  "June 12, 2020",
+            "place_of_marriage": "Tarlac City, Tarlac",
+            "registry_number":   "2026-BC-00123",
+            "date_of_registration": "January 15, 2026",
+            "civil_registrar":   "John Doe",
+            "civil_registrar_position": "City Civil Registrar",
+        };
+        Object.entries(form102Fields).forEach(([k, v]) => {
+            rawBody.innerHTML += `<tr><td>${k}</td><td>${v}</td></tr>`;
+        });
+
+        // ── Populate mapping rows ─────────────────────────────
+        document.getElementById('formClassBadge').textContent = 'Form ' + data.form_class;
+        const mappingDiv = document.getElementById('mappingRows');
+        mappingDiv.innerHTML = '';
+        Object.entries(mappingDef).forEach(([from, to]) => {
+            const val = data.fields[to] || '';
+            if (!val) return;
+            mappingDiv.innerHTML += `
+                <div class="map-row">
+                    <span class="map-from">📄 ${from}</span>
+                    <span class="map-arr">→</span>
+                    <span class="map-to">🏷️ ${to}</span>
+                    <span class="map-arr">→</span>
+                    <span class="map-val">${val}</span>
+                </div>`;
+        });
+
+        // ── Confidence bars ───────────────────────────────────
+        const confDiv = document.getElementById('confidenceBars');
+        confDiv.innerHTML = '';
+        Object.entries(data.confidence).forEach(([key, val]) => {
+            const pct   = Math.round(val * 100);
+            const color = pct >= 90 ? '#1ec77c' : pct >= 75 ? '#f39c12' : '#e74c3c';
+            confDiv.innerHTML += `
+                <div class="conf-row">
+                    <span class="conf-lbl">${fieldLabels[key] || key}</span>
+                    <div class="conf-wrap">
+                        <div class="conf-bar" style="width:${pct}%;background:${color}"></div>
+                    </div>
+                    <span class="conf-pct" style="color:${color}">${pct}%</span>
+                </div>`;
+        });
+
+        // ── Saved file ────────────────────────────────────────
+        _previewUrl = data.preview_url;
+        document.getElementById('savedFileName').textContent = data.saved_file;
+        document.getElementById('savedFilePath').textContent = `C:\\xampp\\htdocs\\uploads\\temp\\${data.saved_file}`;
+
+        results.style.display = 'block';
+        status.innerHTML  = `✅ Done — Form ${data.form_class} filled and saved to uploads/temp/`;
+        status.className  = 'status success';
+
+    } catch (err) {
+        status.innerHTML = '❌ Could not reach Flask. Is python app.py running?';
+        status.className = 'status error';
+        console.error(err);
+    }
+}
+
+function openPreview() {
+    if (_previewUrl) window.open('http://localhost' + _previewUrl, '_blank');
+}
+</script>
+</body>
+</html>

train.py

@@ -0,0 +1,438 @@
+# Training Script for CRNN+CTC Civil Registry OCR Includes CTC loss, learning rate scheduling, and model checkpointing
+
+import torch
+import torch.nn as nn
+import torch.optim as optim
+from torch.utils.data import DataLoader
+import os
+from tqdm import tqdm
+import numpy as np
+from pathlib import Path
+import json
+
+from crnn_model import get_crnn_model, initialize_weights
+from dataset import CivilRegistryDataset, collate_fn
+from utils import (
+    decode_ctc_predictions,
+    calculate_cer,
+    calculate_wer,
+    EarlyStopping
+)
+
+
+class CRNNTrainer:
+    """
+    Trainer class for CRNN+CTC model
+    """
+    
+    def __init__(self, config):
+        self.config = config
+        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+        
+        # Create directories
+        self.checkpoint_dir = Path(config['checkpoint_dir'])
+        self.log_dir = Path(config['log_dir'])
+        self.checkpoint_dir.mkdir(parents=True, exist_ok=True)
+        self.log_dir.mkdir(parents=True, exist_ok=True)
+        
+        # Initialize datasets
+        print("Loading datasets...")
+        self.train_dataset = CivilRegistryDataset(
+            data_dir=config['train_data_dir'],
+            annotations_file=config['train_annotations'],
+            img_height=config['img_height'],
+            img_width=config['img_width'],
+            augment=True,
+            form_type=config.get('form_type', 'all')
+        )
+        
+        self.val_dataset = CivilRegistryDataset(
+            data_dir=config['val_data_dir'],
+            annotations_file=config['val_annotations'],
+            img_height=config['img_height'],
+            img_width=config['img_width'],
+            augment=False,
+            form_type=config.get('form_type', 'all')
+        )
+        
+        # Create data loaders
+        self.train_loader = DataLoader(
+            self.train_dataset,
+            batch_size=config['batch_size'],
+            shuffle=True,
+            num_workers=config['num_workers'],
+            collate_fn=collate_fn,
+            pin_memory=False
+        )
+        
+        self.val_loader = DataLoader(
+            self.val_dataset,
+            batch_size=config['batch_size'],
+            shuffle=False,
+            num_workers=config['num_workers'],
+            collate_fn=collate_fn,
+            pin_memory=False
+        )
+        
+        # Initialize model
+        print(f"Initializing model on {self.device}...")
+        self.model = get_crnn_model(
+            model_type=config.get('model_type', 'standard'),
+            img_height=config['img_height'],
+            num_chars=self.train_dataset.num_chars,
+            hidden_size=config['hidden_size'],
+            num_lstm_layers=config['num_lstm_layers']
+        )
+        
+        self.model = self.model.to(self.device)
+
+        # Loss function - CTC Loss
+        self.criterion = nn.CTCLoss(blank=0, zero_infinity=True)
+
+        # Optimizer — lower LR prevents CTC collapse on epoch 1
+        self.optimizer = optim.Adam(
+            self.model.parameters(),
+            lr=config['learning_rate'],
+            weight_decay=config.get('weight_decay', 1e-4)   # FIXED: fallback was 1e-5
+        )
+
+        # Warmup scheduler: ramp LR from near-zero to target over first N epochs,
+        # then hand off to ReduceLROnPlateau.
+        # This is the single most effective fix for CTC blank collapse.
+        warmup_epochs = config.get('warmup_epochs', 5)
+
+        def warmup_lambda(epoch):
+            if epoch < warmup_epochs:
+                return (epoch + 1) / warmup_epochs   # gradual: 0.2→0.4→0.6→0.8→1.0
+            return 1.0
+
+        self.warmup_scheduler = optim.lr_scheduler.LambdaLR(
+            self.optimizer, lr_lambda=warmup_lambda)
+
+        # ReduceLROnPlateau kicks in after warmup
+        self.scheduler = optim.lr_scheduler.ReduceLROnPlateau(
+            self.optimizer,
+            mode='min',
+            factor=0.5,
+            patience=config.get('lr_patience', 5),
+            min_lr=1e-6
+        )
+        self._warmup_epochs = warmup_epochs
+
+        # Early stopping
+        self.early_stopping = EarlyStopping(
+            patience=config.get('early_stopping_patience', 10),
+            min_delta=config.get('min_delta', 0.001)
+        )
+
+        # Training history
+        self.history = {
+            'train_loss': [],
+            'val_loss': [],
+            'val_cer': [],
+            'val_wer': [],
+            'learning_rates': []
+        }
+
+        # ── Resume from checkpoint if available ──────────────
+        self.start_epoch = 1
+        self.best_val_loss = float('inf')
+        resume_path = self.checkpoint_dir / 'latest_checkpoint.pth'
+
+        if resume_path.exists():
+            print(f"\n  Found checkpoint: {resume_path}")
+            print(f"  Resuming training from last saved epoch...")
+            ckpt = torch.load(resume_path, map_location=self.device, weights_only=False)
+            self.model.load_state_dict(ckpt['model_state_dict'])
+            self.optimizer.load_state_dict(ckpt['optimizer_state_dict'])
+            self.scheduler.load_state_dict(ckpt['scheduler_state_dict'])
+            if 'warmup_scheduler_state_dict' in ckpt:
+                self.warmup_scheduler.load_state_dict(ckpt['warmup_scheduler_state_dict'])
+            self.start_epoch = ckpt['epoch'] + 1
+            self.best_val_loss = ckpt.get('val_loss', float('inf'))
+            self.history = ckpt.get('history', self.history)
+            print(f"  ✓ Resumed from Epoch {ckpt['epoch']}  "
+                  f"(Val Loss: {ckpt['val_loss']:.4f}, CER: {ckpt['val_cer']:.2f}%)")
+        else:
+            print(f"  No checkpoint found — starting fresh.")
+            initialize_weights(self.model)
+
+        print(f"✓ Model ready with {sum(p.numel() for p in self.model.parameters()):,} parameters")
+    
+    def train_epoch(self, epoch):
+        """Train for one epoch"""
+        self.model.train()
+        total_loss = 0
+        
+        pbar = tqdm(self.train_loader, desc=f"Epoch {epoch}/{self.config['epochs']}")
+        
+        nan_count = 0
+        for batch_idx, (images, targets, target_lengths, _) in enumerate(pbar):
+            images = images.to(self.device)
+            targets = targets.to(self.device)
+
+            # FIXED: zero_grad before forward pass (was incorrectly placed after loss)
+            self.optimizer.zero_grad()
+
+            # Forward pass
+            outputs = self.model(images)  # [seq_len, batch, num_chars]
+            
+            # Apply log_softmax for CTC
+            log_probs = nn.functional.log_softmax(outputs, dim=2)
+            
+            # Calculate sequence lengths
+            batch_size = images.size(0)
+            input_lengths = torch.full(
+                size=(batch_size,),
+                fill_value=outputs.size(0),
+                dtype=torch.long
+            ).to(self.device)
+            
+            # CTC loss
+            loss = self.criterion(
+                log_probs,
+                targets,
+                input_lengths,
+                target_lengths
+            )
+
+            # FIXED: skip NaN/Inf batches — accumulating them corrupts gradients
+            if torch.isnan(loss) or torch.isinf(loss):
+                nan_count += 1
+                continue
+
+            # Backward pass
+            loss.backward()
+            
+            # Gradient clipping to prevent exploding gradients
+            torch.nn.utils.clip_grad_norm_(self.model.parameters(), 5.0)
+            
+            self.optimizer.step()
+            
+            total_loss += loss.item()
+            
+            # Update progress bar
+            pbar.set_postfix({
+                'loss': f'{loss.item():.4f}',
+                'avg_loss': f'{total_loss / (batch_idx + 1):.4f}'
+            })
+        if nan_count > 0:
+            print(f"  [WARNING] {nan_count} NaN/Inf batches skipped this epoch.")
+        
+        avg_loss = total_loss / len(self.train_loader)
+        return avg_loss
+    
+    def validate(self):
+        """Validate the model"""
+        self.model.eval()
+        total_loss = 0
+        all_predictions = []
+        all_ground_truths = []
+        
+        with torch.no_grad():
+            for images, targets, target_lengths, texts in tqdm(self.val_loader, desc="Validating"):
+                images = images.to(self.device)
+                targets_gpu = targets.to(self.device)
+                
+                # Forward pass
+                outputs = self.model(images)
+                log_probs = nn.functional.log_softmax(outputs, dim=2)
+                
+                # CTC loss
+                batch_size = images.size(0)
+                input_lengths = torch.full(
+                    size=(batch_size,),
+                    fill_value=outputs.size(0),
+                    dtype=torch.long
+                ).to(self.device)
+                
+                loss = self.criterion(log_probs, targets_gpu, input_lengths, target_lengths)
+                total_loss += loss.item()
+                
+                # Decode predictions
+                predictions = decode_ctc_predictions(
+                    outputs.cpu(),
+                    self.train_dataset.idx_to_char
+                )
+                
+                all_predictions.extend(predictions)
+                all_ground_truths.extend(texts)
+        
+        avg_loss = total_loss / len(self.val_loader)
+        
+        # Calculate metrics
+        cer = calculate_cer(all_predictions, all_ground_truths)
+        wer = calculate_wer(all_predictions, all_ground_truths)
+        
+        return avg_loss, cer, wer, all_predictions, all_ground_truths
+    
+    def train(self):
+        """Main training loop"""
+        print("\n" + "=" * 70)
+        print("Starting Training")
+        print("=" * 70)
+        
+        best_val_loss = self.best_val_loss
+
+        for epoch in range(self.start_epoch, self.config['epochs'] + 1):
+            print(f"\nEpoch {epoch}/{self.config['epochs']}")
+            print("-" * 70)
+            
+            # Train
+            train_loss = self.train_epoch(epoch)
+            
+            # Validate
+            val_loss, val_cer, val_wer, predictions, ground_truths = self.validate()
+            
+            # Learning rate scheduling
+            # Use warmup for first N epochs, then ReduceLROnPlateau
+            if epoch <= self._warmup_epochs:
+                self.warmup_scheduler.step()
+            else:
+                self.scheduler.step(val_loss)
+            current_lr = self.optimizer.param_groups[0]['lr']
+            
+            # Update history
+            self.history['train_loss'].append(train_loss)
+            self.history['val_loss'].append(val_loss)
+            self.history['val_cer'].append(val_cer)
+            self.history['val_wer'].append(val_wer)
+            self.history['learning_rates'].append(current_lr)
+            
+            # Print metrics
+            print(f"\nMetrics:")
+            print(f"  Train Loss: {train_loss:.4f}")
+            print(f"  Val Loss:   {val_loss:.4f}")
+            print(f"  Val CER:    {val_cer:.2f}%")
+            print(f"  Val WER:    {val_wer:.2f}%")
+            print(f"  LR:         {current_lr:.6f}")
+            
+            # Print sample predictions
+            print(f"\nSample Predictions:")
+            for i in range(min(3, len(predictions))):
+                print(f"  GT:   {ground_truths[i]}")
+                print(f"  Pred: {predictions[i]}")
+                print()
+
+            # show raw model output
+            with torch.no_grad():
+                sample_img = self.val_dataset[0][0].unsqueeze(0).to(self.device)
+                raw_out    = self.model(sample_img)
+                probs      = torch.softmax(raw_out, dim=2)
+                best_idx   = probs[:, 0, :].argmax(dim=1)
+                best_prob  = probs[:, 0, :].max(dim=1).values
+                blank_pct  = (best_idx == 0).float().mean().item() * 100
+                avg_conf   = best_prob.mean().item()
+                non_blank  = [self.train_dataset.idx_to_char.get(i.item(), '?')
+                              for i in best_idx if i.item() != 0]
+                print(f"  blank={blank_pct:.0f}%  conf={avg_conf:.3f}  "
+                      f"chars={''.join(non_blank[:20])!r}")
+
+            
+            # Save checkpoint
+            is_best = val_loss < best_val_loss
+            if is_best:
+                best_val_loss = val_loss
+            
+            self.save_checkpoint(epoch, val_loss, val_cer, is_best)
+            
+            # Early stopping
+            if self.early_stopping(val_loss):
+                print(f"\nEarly stopping triggered at epoch {epoch}")
+                break
+        
+        print("\n" + "=" * 70)
+        print("Training Complete!")
+        print(f"Best validation loss: {best_val_loss:.4f}")
+        print("=" * 70)
+        
+        # Save final training history
+        self.save_history()
+    
+    def save_checkpoint(self, epoch, val_loss, val_cer, is_best=False):
+        """Save model checkpoint"""
+        checkpoint = {
+            'epoch': epoch,
+            'model_state_dict': self.model.state_dict(),
+            'optimizer_state_dict': self.optimizer.state_dict(),
+            'scheduler_state_dict': self.scheduler.state_dict(),
+            'warmup_scheduler_state_dict': self.warmup_scheduler.state_dict(),
+            'val_loss': val_loss,
+            'val_cer': val_cer,
+            'char_to_idx': self.train_dataset.char_to_idx,
+            'idx_to_char': self.train_dataset.idx_to_char,
+            'config': self.config,
+            'history': self.history
+        }
+        
+        # Save latest checkpoint
+        checkpoint_path = self.checkpoint_dir / 'latest_checkpoint.pth'
+        torch.save(checkpoint, checkpoint_path)
+        
+        # Save best checkpoint
+        if is_best:
+            best_path = self.checkpoint_dir / 'best_model.pth'
+            torch.save(checkpoint, best_path)
+            print(f"  ✓ Best model saved (Val Loss: {val_loss:.4f}, CER: {val_cer:.2f}%)")
+        
+        # Save epoch checkpoint (history omitted to save disk space — it's in latest_checkpoint.pth)
+        if epoch % self.config.get('save_freq', 10) == 0:
+            epoch_path = self.checkpoint_dir / f'checkpoint_epoch_{epoch}.pth'
+            epoch_ckpt = {k: v for k, v in checkpoint.items() if k != 'history'}
+            torch.save(epoch_ckpt, epoch_path)
+    
+    def save_history(self):
+        """Save training history"""
+        history_path = self.log_dir / 'training_history.json'
+        with open(history_path, 'w') as f:
+            json.dump(self.history, f, indent=2)
+        print(f"\n✓ Training history saved to {history_path}")
+
+
+def main():
+    """Main training function"""
+    
+    # Configuration
+    config = {
+        # Data
+        'train_data_dir': 'data/train',
+        'train_annotations': 'data/train_annotations.json',
+        'val_data_dir': 'data/val',
+        'val_annotations': 'data/val_annotations.json',
+        'form_type': 'all',  # 'all', 'form1a', 'form2a', 'form3a', 'form90'
+        
+        # Model
+        'model_type': 'standard',  # 'standard', 'ensemble', 'lightweight'
+        'img_height': 64,
+        'img_width': 512,
+        'hidden_size': 128,
+        'num_lstm_layers': 1,
+        
+        # Training
+        'batch_size': 32,
+        'epochs': 100,
+        'learning_rate': 0.0001,
+        'weight_decay': 1e-4,   # FIXED: was 1e-5 — stronger L2 regularisation to reduce overfitting
+        'num_workers': 0,
+        'warmup_epochs': 5,        # Ramp LR gradually for first 5 epochs
+
+        # Scheduling & Early Stopping
+        'lr_patience': 5,          # FIXED: was 3 — give model more time before halving LR
+        'early_stopping_patience': 20,  # FIXED: was 10 — more patience during zoom training
+        'min_delta': 0.001,
+        
+        # Saving
+        'checkpoint_dir': 'checkpoints',
+        'log_dir': 'logs',
+        'save_freq': 10,
+    }
+    
+    # Initialize trainer
+    trainer = CRNNTrainer(config)
+    
+    # Start training
+    trainer.train()
+
+
+if __name__ == "__main__":
+    main()

train_emnist.py

@@ -0,0 +1,15 @@
+import torchvision
+import torchvision.transforms as transforms
+
+print("Loading EMNIST dataset...")
+
+train_data = torchvision.datasets.EMNIST(
+    root='datasets/emnist',
+    split='byclass',
+    train=True,
+    download=False,
+    transform=transforms.ToTensor()
+)
+
+print(f"Training samples: {len(train_data)}")
+print("EMNIST loaded successfully!")

train_mnist.py

@@ -0,0 +1,42 @@
+import tensorflow as tf
+import numpy as np
+from tensorflow.keras import layers, models
+
+# Load MNIST dataset
+(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()
+
+# Normalize pixel values to 0-1
+x_train = x_train / 255.0
+x_test = x_test / 255.0
+
+# Add channel dimension (28, 28) -> (28, 28, 1)
+x_train = x_train[..., tf.newaxis]
+x_test = x_test[..., tf.newaxis]
+
+# Build simple CNN model
+model = models.Sequential([
+    layers.Conv2D(32, (3,3), activation='relu', input_shape=(28,28,1)),
+    layers.MaxPooling2D(2,2),
+    layers.Conv2D(64, (3,3), activation='relu'),
+    layers.MaxPooling2D(2,2),
+    layers.Flatten(),
+    layers.Dense(128, activation='relu'),
+    layers.Dense(10, activation='softmax')  # 10 digits (0-9)
+])
+
+model.compile(optimizer='adam',
+              loss='sparse_categorical_crossentropy',
+              metrics=['accuracy'])
+
+model.summary()
+
+# Train
+model.fit(x_train, y_train, epochs=5, validation_split=0.1)
+
+# Evaluate
+test_loss, test_acc = model.evaluate(x_test, y_test)
+print(f"\nTest accuracy: {test_acc:.4f}")
+
+# Save model
+model.save("mnist_model.h5")
+print("Model saved as mnist_model.h5")

train_with_emnist.py

@@ -0,0 +1,169 @@
+"""
+train_with_emnist.py
+====================
+Fine-tune the CRNN model with EMNIST character data.
+
+FIXES vs old version:
+  - Phase 1: CNN FROZEN — only RNN+FC trained (prevents catastrophic forgetting)
+  - Phase 2: Full model at 10x lower LR for final polish
+  - log_softmax applied before CTCLoss (was missing — caused garbage loss)
+  - Loads from best_model.pth (synthetic, 0.12% CER baseline)
+  - Saves best_model_emnist.pth only when val improves
+"""
+
+import os
+import sys
+import torch
+import torch.nn.functional as F
+import torch.optim as optim
+from torch.utils.data import DataLoader, ConcatDataset
+
+sys.path.append('.')
+from crnn_model import get_crnn_model
+from dataset import CivilRegistryDataset, collate_fn
+
+print("=" * 55)
+print("Fine-tuning CRNN with EMNIST dataset")
+print("=" * 55)
+
+DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+print(f"Device: {DEVICE}")
+
+emnist_dataset = CivilRegistryDataset(
+    data_dir='data/train',
+    annotations_file='data/emnist_train_annotations.json',
+    img_height=64, img_width=512, augment=True
+)
+# FIXED: mix synthetic data in so the model never forgets multi-word sequences
+synth_dataset = CivilRegistryDataset(
+    data_dir='data/train',
+    annotations_file='data/train_annotations.json',
+    img_height=64, img_width=512, augment=True
+)
+train_dataset = emnist_dataset  # keep reference for char_to_idx / num_chars
+mixed_train   = ConcatDataset([emnist_dataset, synth_dataset])
+val_dataset = CivilRegistryDataset(
+    data_dir='data/val',
+    annotations_file='data/val_annotations.json',  # FIXED: was emnist_val — must match real task
+    img_height=64, img_width=512, augment=False
+)
+print(f"EMNIST train  : {len(emnist_dataset)}")
+print(f"Synthetic train: {len(synth_dataset)}")
+print(f"Mixed train   : {len(mixed_train)}")
+print(f"Val           : {len(val_dataset)}")
+
+train_loader = DataLoader(mixed_train, batch_size=32, shuffle=True,
+                          num_workers=0, collate_fn=collate_fn)
+val_loader   = DataLoader(val_dataset,   batch_size=32, shuffle=False,
+                          num_workers=0, collate_fn=collate_fn)
+
+# ── Load best synthetic checkpoint ───────────────────────────
+BASE = 'checkpoints/best_model.pth'
+if not os.path.exists(BASE):
+    print(f"ERROR: {BASE} not found. Run: python train.py")
+    sys.exit(1)
+
+ckpt   = torch.load(BASE, map_location=DEVICE, weights_only=False)
+config = ckpt.get('config', {})
+
+model = get_crnn_model(
+    model_type      = config.get('model_type', 'standard'),
+    img_height      = config.get('img_height', 64),
+    num_chars       = train_dataset.num_chars,
+    hidden_size     = config.get('hidden_size', 128),
+    num_lstm_layers = config.get('num_lstm_layers', 1),
+).to(DEVICE)
+
+missing, _ = model.load_state_dict(ckpt['model_state_dict'], strict=False)
+if missing:
+    print(f"  Note: {len(missing)} layers re-initialized (expected for fc layer)")
+print(f"  Loaded epoch {ckpt.get('epoch')} "
+      f"(val_loss={ckpt.get('val_loss', ckpt.get('val_cer', 0)):.4f})")
+
+criterion = torch.nn.CTCLoss(blank=0, reduction='mean', zero_infinity=True)
+
+
+def run_epoch(loader, training, optimizer=None):
+    model.train() if training else model.eval()
+    total, n = 0, 0
+    ctx = torch.enable_grad() if training else torch.no_grad()
+    with ctx:
+        for images, targets, target_lengths, _ in loader:
+            images        = images.to(DEVICE)
+            batch_size    = images.size(0)
+            if training:
+                optimizer.zero_grad()
+            # CRITICAL: log_softmax before CTCLoss
+            outputs       = F.log_softmax(model(images), dim=2)
+            seq_len       = outputs.size(0)
+            input_lengths = torch.full((batch_size,), seq_len, dtype=torch.long)
+            loss = criterion(outputs, targets, input_lengths, target_lengths)
+            if not torch.isnan(loss) and not torch.isinf(loss):
+                if training:
+                    loss.backward()
+                    torch.nn.utils.clip_grad_norm_(model.parameters(), 5)
+                    optimizer.step()
+                total += loss.item()
+                n     += 1
+    return total / max(n, 1)
+
+
+def run_phase(num, epochs, lr, freeze_cnn, patience):
+    print(f"\n{'='*55}")
+    print(f"  PHASE {num} — "
+          f"{'CNN FROZEN  (RNN+FC only)' if freeze_cnn else 'FULL MODEL  (all layers)'}"
+          f"   LR={lr}")
+    print(f"{'='*55}")
+
+    # Freeze or unfreeze CNN
+    for name, param in model.named_parameters():
+        param.requires_grad = not (freeze_cnn and 'cnn' in name)
+
+    trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
+    print(f"  Trainable params : {trainable:,}")
+
+    opt      = optim.Adam(
+        filter(lambda p: p.requires_grad, model.parameters()), lr=lr)
+    sched    = optim.lr_scheduler.ReduceLROnPlateau(opt, patience=3, factor=0.5)
+    best     = float('inf')
+    counter  = 0
+
+    for epoch in range(1, epochs + 1):
+        tr = run_epoch(train_loader, True,  opt)
+        vl = run_epoch(val_loader,   False, None)
+        sched.step(vl)
+
+        if vl < best:
+            best    = vl
+            counter = 0
+            torch.save({
+                'model_state_dict': model.state_dict(),
+                'config':           config,
+                'char_to_idx':      train_dataset.char_to_idx,
+                'idx_to_char':      train_dataset.idx_to_char,
+                'epoch':            epoch,
+                'val_loss':         vl,   # FIXED: renamed from val_cer — this is val loss, not CER%
+            }, 'checkpoints/best_model_emnist.pth')
+            print(f"  Epoch {epoch:02d}/{epochs}  Train={tr:.4f}  Val={vl:.4f}  <- saved")
+        else:
+            counter += 1
+            print(f"  Epoch {epoch:02d}/{epochs}  Train={tr:.4f}  Val={vl:.4f}"
+                  f"  (patience {counter}/{patience})")
+            if counter >= patience:
+                print(f"  Early stopping at epoch {epoch}.")
+                break
+    return best
+
+
+# ── Phase 1: Freeze CNN — teach RNN+FC to handle EMNIST chars ─
+p1_best = run_phase(1, epochs=30, lr=1e-4, freeze_cnn=True, patience=7)
+
+# ── Phase 2: Unfreeze all — gentle full-model polish ──────────
+p2_best = run_phase(2, epochs=20, lr=1e-6, freeze_cnn=False, patience=5)
+
+print(f"\n{'='*55}")
+print(f"EMNIST fine-tuning complete!")
+print(f"  Phase 1 best val loss : {p1_best:.4f}")
+print(f"  Phase 2 best val loss : {p2_best:.4f}")
+print(f"  Saved : checkpoints/best_model_emnist.pth")
+print(f"\nNext step: python IAM_train.py --prepare --train")

utils.py

@@ -0,0 +1,397 @@
+"""
+Utility Functions for CRNN+CTC Civil Registry OCR
+Includes CTC decoding, metrics calculation, and helper functions
+"""
+
+import torch
+import numpy as np
+def _editdistance(a, b):
+    """Pure-Python Levenshtein distance — replaces the editdistance C extension."""
+    m, n = len(a), len(b)
+    dp = list(range(n + 1))
+    for i in range(1, m + 1):
+        prev, dp[0] = dp[0], i
+        for j in range(1, n + 1):
+            prev, dp[j] = dp[j], prev if a[i-1] == b[j-1] else 1 + min(prev, dp[j], dp[j-1])
+    return dp[n]
+from typing import List, Dict, Tuple
+
+
+def decode_ctc_predictions(outputs, idx_to_char, method='greedy'):
+    """
+    Decode CTC predictions to text
+    
+    Args:
+        outputs: Model outputs [seq_len, batch, num_chars]
+        idx_to_char: Dictionary mapping indices to characters
+        method: 'greedy' or 'beam_search'
+        
+    Returns:
+        List of decoded strings
+    """
+    if method == 'greedy':
+        return greedy_decode(outputs, idx_to_char)
+    elif method == 'beam_search':
+        return beam_search_decode(outputs, idx_to_char)
+    else:
+        raise ValueError(f"Unknown decoding method: {method}")
+
+
+def greedy_decode(outputs, idx_to_char):
+    """
+    Greedy CTC decoding - fast but less accurate
+    """
+    # Get most probable characters
+    pred_indices = torch.argmax(outputs, dim=2)  # [seq_len, batch]
+    pred_indices = pred_indices.permute(1, 0)  # [batch, seq_len]
+    
+    decoded_texts = []
+    
+    for sequence in pred_indices:
+        chars = []
+        prev_idx = -1
+        
+        for idx in sequence:
+            idx = idx.item()
+            # Skip blank (0) and consecutive duplicates
+            if idx != 0 and idx != prev_idx:
+                if idx in idx_to_char:
+                    chars.append(idx_to_char[idx])
+            prev_idx = idx
+        
+        decoded_texts.append(''.join(chars))
+    
+    return decoded_texts
+
+
+def beam_search_decode(outputs, idx_to_char, beam_width=10):
+    """
+    Beam search CTC decoding - slower but more accurate.
+
+    FIXED Bug 6: previous code mixed list-of-chars and string representations.
+    After sorting new_beams (a dict keyed by strings), it did `list(seq)` on the
+    string key — which splits a string like "AB" into ['A','B'] accidentally works
+    for ASCII but is fragile and confusing. Rewritten to use strings throughout:
+    beams are now List[Tuple[str, float]] with the sequence always kept as a plain
+    string, eliminating the list/string ambiguity entirely.
+    """
+    outputs = torch.nn.functional.softmax(outputs, dim=2)
+    outputs = outputs.permute(1, 0, 2).cpu().numpy()  # [batch, seq_len, num_chars]
+
+    decoded_texts = []
+
+    for output in outputs:
+        # Each beam is (sequence_string, cumulative_probability)
+        beams: list = [('', 1.0)]
+
+        for timestep in output:
+            new_beams: dict = {}
+
+            for sequence, prob in beams:
+                for idx, char_prob in enumerate(timestep):
+                    if idx == 0:  # blank token — sequence unchanged
+                        new_seq = sequence
+                    elif idx in idx_to_char:
+                        char = idx_to_char[idx]
+                        # CTC rule: merge consecutive duplicate characters
+                        if sequence and sequence[-1] == char:
+                            new_seq = sequence        # duplicate — stay the same
+                        else:
+                            new_seq = sequence + char # append directly to string
+                    else:
+                        continue
+
+                    new_prob = prob * char_prob
+                    # Merge beams that produce the same string
+                    if new_seq in new_beams:
+                        new_beams[new_seq] = max(new_beams[new_seq], new_prob)
+                    else:
+                        new_beams[new_seq] = new_prob
+
+            # Keep top-k beams; keys are already strings — no list() conversion needed
+            beams = sorted(new_beams.items(), key=lambda x: x[1], reverse=True)[:beam_width]
+
+        # Best sequence is the string with highest probability
+        best_sequence = max(beams, key=lambda x: x[1])[0]
+        decoded_texts.append(best_sequence)
+
+    return decoded_texts
+
+
+def calculate_cer(predictions: List[str], ground_truths: List[str]) -> float:
+    """
+    Calculate Character Error Rate (CER)
+    
+    CER = (Substitutions + Deletions + Insertions) / Total Characters
+    """
+    if len(predictions) != len(ground_truths):
+        raise ValueError("Predictions and ground truths must have same length")
+    
+    total_distance = 0
+    total_length = 0
+    
+    for pred, gt in zip(predictions, ground_truths):
+        distance = _editdistance(pred, gt)
+        total_distance += distance
+        total_length += len(gt)
+    
+    cer = (total_distance / total_length * 100) if total_length > 0 else 0
+    return cer
+
+
+def calculate_wer(predictions: List[str], ground_truths: List[str]) -> float:
+    """
+    Calculate Word Error Rate (WER)
+    
+    WER = (Substitutions + Deletions + Insertions) / Total Words
+    """
+    if len(predictions) != len(ground_truths):
+        raise ValueError("Predictions and ground truths must have same length")
+    
+    total_distance = 0
+    total_length = 0
+    
+    for pred, gt in zip(predictions, ground_truths):
+        pred_words = pred.split()
+        gt_words = gt.split()
+        
+        distance = _editdistance(pred_words, gt_words)
+        total_distance += distance
+        total_length += len(gt_words)
+    
+    wer = (total_distance / total_length * 100) if total_length > 0 else 0
+    return wer
+
+
+def calculate_accuracy(predictions: List[str], ground_truths: List[str]) -> float:
+    """
+    Calculate exact match accuracy
+    """
+    if len(predictions) != len(ground_truths):
+        raise ValueError("Predictions and ground truths must have same length")
+    
+    correct = sum(1 for pred, gt in zip(predictions, ground_truths) if pred == gt)
+    accuracy = (correct / len(predictions) * 100) if len(predictions) > 0 else 0
+    
+    return accuracy
+
+
+class EarlyStopping:
+    """
+    Early stopping to stop training when validation loss stops improving
+    """
+    
+    def __init__(self, patience=10, min_delta=0.001):
+        self.patience = patience
+        self.min_delta = min_delta
+        self.counter = 0
+        self.best_loss = None
+        self.early_stop = False
+    
+    def __call__(self, val_loss):
+        if self.best_loss is None:
+            self.best_loss = val_loss
+        elif val_loss > self.best_loss - self.min_delta:
+            self.counter += 1
+            if self.counter >= self.patience:
+                self.early_stop = True
+        else:
+            self.best_loss = val_loss
+            self.counter = 0
+        
+        return self.early_stop
+
+
+class AverageMeter:
+    """
+    Computes and stores the average and current value
+    """
+    
+    def __init__(self):
+        self.reset()
+    
+    def reset(self):
+        self.val = 0
+        self.avg = 0
+        self.sum = 0
+        self.count = 0
+    
+    def update(self, val, n=1):
+        self.val = val
+        self.sum += val * n
+        self.count += n
+        self.avg = self.sum / self.count
+
+
+def calculate_confusion_matrix(predictions: List[str], ground_truths: List[str], char_set: List[str]) -> np.ndarray:
+    """
+    Calculate character-level confusion matrix
+    
+    Args:
+        predictions: List of predicted strings
+        ground_truths: List of ground truth strings
+        char_set: List of all possible characters
+        
+    Returns:
+        Confusion matrix [num_chars, num_chars]
+    """
+    char_to_idx = {char: idx for idx, char in enumerate(char_set)}
+    n_chars = len(char_set)
+    
+    confusion = np.zeros((n_chars, n_chars), dtype=np.int64)
+    
+    for pred, gt in zip(predictions, ground_truths):
+        # Align sequences (simple alignment)
+        max_len = max(len(pred), len(gt))
+        pred_padded = pred + ' ' * (max_len - len(pred))
+        gt_padded = gt + ' ' * (max_len - len(gt))
+        
+        for p_char, g_char in zip(pred_padded, gt_padded):
+            if p_char in char_to_idx and g_char in char_to_idx:
+                confusion[char_to_idx[g_char], char_to_idx[p_char]] += 1
+    
+    return confusion
+
+
+def extract_form_fields(text: str, form_type: str) -> Dict[str, str]:
+    """
+    Extract specific fields from recognized text based on form type
+    
+    Args:
+        text: Recognized text
+        form_type: 'form1a', 'form2a', 'form3a', 'form90'
+        
+    Returns:
+        Dictionary of extracted fields
+    """
+    fields = {}
+    
+    if form_type == 'form1a':  # Birth Certificate
+        # Extract common fields (simplified)
+        # In practice, use NER or regex patterns
+        fields['type'] = 'Birth Certificate'
+        # Add more field extraction logic
+        
+    elif form_type == 'form2a':  # Death Certificate
+        fields['type'] = 'Death Certificate'
+        
+    elif form_type == 'form3a':  # Marriage Certificate
+        fields['type'] = 'Marriage Certificate'
+        
+    elif form_type == 'form90':  # Marriage License Application
+        fields['type'] = 'Marriage License Application'
+    
+    return fields
+
+
+def validate_extracted_data(data: Dict[str, str], form_type: str) -> Tuple[bool, List[str]]:
+    """
+    Validate extracted data for completeness and format
+    
+    Args:
+        data: Extracted data dictionary
+        form_type: Form type
+        
+    Returns:
+        (is_valid, list_of_errors)
+    """
+    errors = []
+    
+    # Define required fields per form type
+    required_fields = {
+        'form1a': ['name', 'date_of_birth', 'place_of_birth'],
+        'form2a': ['name', 'date_of_death', 'place_of_death'],
+        'form3a': ['husband_name', 'wife_name', 'date_of_marriage'],
+        'form90': ['husband_name', 'wife_name', 'date_of_application']
+    }
+    
+    # Check required fields
+    for field in required_fields.get(form_type, []):
+        if field not in data or not data[field]:
+            errors.append(f"Missing required field: {field}")
+    
+    # Additional validation can be added here
+    # - Date format validation
+    # - Name format validation
+    # - etc.
+    
+    is_valid = len(errors) == 0
+    return is_valid, errors
+
+
+def load_checkpoint(checkpoint_path, model, optimizer=None, device='cpu'):
+    """
+    Load model checkpoint
+    
+    Args:
+        checkpoint_path: Path to checkpoint file
+        model: Model instance
+        optimizer: Optimizer instance (optional)
+        device: Device to load to
+        
+    Returns:
+        (model, optimizer, checkpoint_dict)
+    """
+    checkpoint = torch.load(checkpoint_path, map_location=device)
+    
+    model.load_state_dict(checkpoint['model_state_dict'])
+    
+    if optimizer is not None and 'optimizer_state_dict' in checkpoint:
+        optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
+    
+    print(f"✓ Loaded checkpoint from {checkpoint_path}")
+    print(f"  Epoch: {checkpoint.get('epoch', 'N/A')}")
+    if 'val_cer' in checkpoint:
+        print(f"  Val CER  : {checkpoint['val_cer']:.4f}%")
+    elif 'val_loss' in checkpoint:
+        print(f"  Val Loss : {checkpoint['val_loss']:.4f}  (run compare_live_cer.py for true CER)")
+    else:
+        print(f"  Val CER  : N/A  (run compare_live_cer.py for true CER)")
+    
+    return model, optimizer, checkpoint
+
+
+def save_predictions_to_file(predictions: List[str], ground_truths: List[str], output_file: str):
+    """
+    Save predictions and ground truths to file for analysis
+    """
+    with open(output_file, 'w', encoding='utf-8') as f:
+        f.write("Ground Truth\tPrediction\tMatch\n")
+        f.write("=" * 80 + "\n")
+        
+        for gt, pred in zip(ground_truths, predictions):
+            match = "✓" if gt == pred else "✗"
+            f.write(f"{gt}\t{pred}\t{match}\n")
+    
+    print(f"✓ Predictions saved to {output_file}")
+
+
+if __name__ == "__main__":
+    # Test utility functions
+    print("=" * 60)
+    print("Testing Utility Functions")
+    print("=" * 60)
+    
+    # Test CER calculation
+    predictions = ["Hello World", "Test", "Sample Text"]
+    ground_truths = ["Hello World", "Tset", "Sample Txt"]
+    
+    cer = calculate_cer(predictions, ground_truths)
+    wer = calculate_wer(predictions, ground_truths)
+    accuracy = calculate_accuracy(predictions, ground_truths)
+    
+    print(f"\nMetrics:")
+    print(f"  CER: {cer:.2f}%")
+    print(f"  WER: {wer:.2f}%")
+    print(f"  Accuracy: {accuracy:.2f}%")
+    
+    # Test early stopping
+    print("\nTesting Early Stopping:")
+    early_stopping = EarlyStopping(patience=3, min_delta=0.001)
+    
+    val_losses = [1.0, 0.9, 0.85, 0.84, 0.84, 0.84, 0.84]
+    for epoch, loss in enumerate(val_losses, 1):
+        should_stop = early_stopping(loss)
+        print(f"  Epoch {epoch}: Loss = {loss:.2f}, Stop = {should_stop}")
+        if should_stop:
+            break