Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,11 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+model_hindi/snapshot-1.data-00000-of-00001 filter=lfs diff=lfs merge=lfs -text
+model_hindi/snapshot-1.meta filter=lfs diff=lfs merge=lfs -text
+model_hindi/snapshot-2.data-00000-of-00001 filter=lfs diff=lfs merge=lfs -text
+model_hindi/snapshot-2.meta filter=lfs diff=lfs merge=lfs -text
+model_hindi/snapshot-4.data-00000-of-00001 filter=lfs diff=lfs merge=lfs -text
+model_hindi/snapshot-4.meta filter=lfs diff=lfs merge=lfs -text
+model_hindi/snapshot-5.data-00000-of-00001 filter=lfs diff=lfs merge=lfs -text
+model_hindi/snapshot-5.meta filter=lfs diff=lfs merge=lfs -text

DataLoader_Hindi_v2.py

@@ -0,0 +1,107 @@
+# -*- coding: utf-8 -*-
+from __future__ import division
+from __future__ import print_function
+
+import os
+import gc
+import numpy as np
+import cv2
+from collections import namedtuple
+from datasets import load_dataset
+
+from SamplePreprocessor_Hindi_v2 import preprocess
+
+Batch = namedtuple("Batch", ["imgs", "gtTexts"])
+
+
+class DataLoader:
+
+    def __init__(self,
+                 dataset_name: str,
+                 images_dir: str,
+                 batchSize: int,
+                 imgSize: tuple,
+                 maxTextLen: int,
+                 charList: str,
+                 dataAugmentation: bool = False,
+                 val_split: float = 0.1):
+
+        self.batchSize        = batchSize
+        self.imgSize          = imgSize
+        self.maxTextLen       = maxTextLen
+        self.charList         = charList
+        self.dataAugmentation = dataAugmentation
+        self.images_dir       = images_dir
+        self.charSet          = set(charList)
+
+        print(f"[DataLoader] Loading dataset: {dataset_name}")
+
+        dataset = load_dataset(dataset_name, split='train')
+        dataset = dataset.shuffle(seed=42)
+
+        split_idx = int(len(dataset) * (1 - val_split))
+
+        self.train_dataset = dataset.select(range(split_idx))
+        self.val_dataset   = dataset.select(range(split_idx, len(dataset)))
+
+        print(f"[DataLoader] Train: {len(self.train_dataset)}")
+        print(f"[DataLoader] Val:   {len(self.val_dataset)}")
+
+        self._current_dataset = None
+        self._idx = 0
+
+    def _load_and_preprocess_image(self, img_path, augment):
+        img = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)
+        if img is None:
+            img = np.zeros([self.imgSize[1], self.imgSize[0]], dtype=np.uint8)
+        return preprocess(img, self.imgSize, dataAugmentation=augment)
+
+    def trainSet(self, subset_size=None):
+        dataset = self.train_dataset.shuffle(seed=None)
+
+        if subset_size is not None:
+            subset_size = min(subset_size, len(dataset))
+            indices = np.random.choice(len(dataset), subset_size, replace=False)
+            dataset = dataset.select(indices)
+            print(f"[DataLoader] Training on {subset_size} samples")
+        else:
+            print(f"[DataLoader] Training on FULL dataset")
+
+        self._current_dataset = dataset
+        self._idx = 0
+
+    def validationSet(self):
+        self._current_dataset = self.val_dataset
+        self._idx = 0
+
+    def hasNext(self):
+        return self._idx < len(self._current_dataset)
+
+    def getIteratorInfo(self):
+      """Return current batch index and total batches."""
+      total = max(1, (len(self._current_dataset) + self.batchSize - 1) // self.batchSize)
+      current = self._idx // self.batchSize + 1
+      return (current, total)
+      
+    def getNext(self):
+        end = min(self._idx + self.batchSize, len(self._current_dataset))
+        batch_indices = range(self._idx, end)
+        self._idx = end
+
+        imgs = []
+        gtTexts = []
+
+        for idx in batch_indices:
+          sample = self._current_dataset[idx]
+
+          augment = (self._current_dataset != self.val_dataset)
+
+          img = np.array(sample["image"].convert("L"))
+          img = preprocess(img, self.imgSize, dataAugmentation=augment)
+
+          imgs.append(img)
+          gtTexts.append(sample['text'])
+        return Batch(imgs=imgs, gtTexts=gtTexts)
+
+    def cleanup(self):
+        gc.collect()

Model_Hindi_v2.py

@@ -0,0 +1,364 @@
+# -*- coding: utf-8 -*-
+"""
+Model_Hindi_v2.py  —  FAST & BALANCED VERSION
+--------------------------------------------------------------
+Optimized for speed with balanced regularization.
+
+KEY CHANGES:
+  ✅ Larger batch size (16-32) - No gradient accumulation
+  ✅ Label smoothing (0.05 instead of 0.1) - Gentler
+  ✅ Moderate dropout (0.3) - Not too strong
+  ✅ Faster training
+"""
+
+from __future__ import division
+from __future__ import print_function
+
+import sys
+import numpy as np
+import tensorflow as tf
+from tensorflow.keras.layers import LSTM, Dense, Bidirectional
+
+tf.compat.v1.disable_eager_execution()
+
+
+class DecoderType:
+    BestPath       = 0
+    BeamSearch     = 1
+
+
+class Model:
+    """CRNN+CTC optimized for fast training."""
+
+    # ✅ LARGER batch size for speed
+    batchSize      = 16  # Can increase to 32 if GPU allows
+    
+    imgSize        = (900, 64)
+    maxTextLen     = 128
+    CNN_TIME_STEPS = 225
+
+    def __init__(self, charList, decoderType=DecoderType.BestPath,
+                 mustRestore=False, lastEpoch=0):
+
+        self.charList    = charList
+        self.decoderType = decoderType
+        self.mustRestore = mustRestore
+        self.snapID      = 0
+        self.lastEpoch   = lastEpoch
+
+        self.SPACE_IDX = charList.index(' ') if ' ' in charList else 1
+
+        # Placeholders
+        self.inputImgs = tf.compat.v1.placeholder(
+            tf.float32,
+            shape=(None, Model.imgSize[0], Model.imgSize[1]),
+            name='input_imgs')
+
+        self.isTraining = tf.compat.v1.placeholder_with_default(
+            False, shape=(), name='is_training')
+
+        self.dropout_rate = tf.compat.v1.placeholder_with_default(
+            0.0, shape=(), name='dropout_rate')
+
+        self.setupCNN()
+        self.setupRNN()
+        self.setupCTC()
+
+        self.learningRate = tf.compat.v1.placeholder(
+            tf.float32, shape=[], name='learning_rate')
+
+        self.setupOptimizer()
+        (self.sess, self.saver) = self.setupTF()
+
+    # ── BatchNorm ─────────────────────────────────────────────────────────────
+    def _batch_norm(self, x, name):
+        """BatchNorm layer."""
+        channels = x.shape[-1]
+
+        gamma       = tf.Variable(tf.ones([channels]),  trainable=True,  name=f'{name}_gamma')
+        beta        = tf.Variable(tf.zeros([channels]), trainable=True,  name=f'{name}_beta')
+        moving_mean = tf.Variable(tf.zeros([channels]), trainable=False, name=f'{name}_moving_mean')
+        moving_var  = tf.Variable(tf.ones([channels]),  trainable=False, name=f'{name}_moving_var')
+
+        decay = 0.99
+        eps   = 1e-5
+
+        def train_mode():
+            mean, var = tf.nn.moments(x, axes=[0, 1, 2])
+            update_mean = tf.compat.v1.assign(
+                moving_mean, moving_mean * decay + mean * (1 - decay))
+            update_var  = tf.compat.v1.assign(
+                moving_var,  moving_var  * decay + var  * (1 - decay))
+            with tf.control_dependencies([update_mean, update_var]):
+                return tf.nn.batch_normalization(x, mean, var, beta, gamma, eps)
+
+        def infer_mode():
+            return tf.nn.batch_normalization(
+                x, moving_mean, moving_var, beta, gamma, eps)
+
+        return tf.cond(self.isTraining, train_mode, infer_mode)
+
+    # ── CNN ───────────────────────────────────────────────────────────────────
+    def setupCNN(self):
+        """5-block CNN with BatchNorm."""
+        cnnIn4d = tf.expand_dims(self.inputImgs, axis=3)
+
+        kernelVals  = [5, 5, 3, 3, 3]
+        featureVals = [1, 64, 128, 256, 256, 384]
+        poolVals    = [(2, 2), (2, 2), (1, 2), (1, 2), (1, 4)]
+        strideVals  = poolVals
+
+        pool = cnnIn4d
+        for i in range(len(poolVals)):
+            kernel = tf.Variable(
+                tf.random.truncated_normal(
+                    [kernelVals[i], kernelVals[i],
+                     featureVals[i], featureVals[i + 1]],
+                    stddev=0.1),
+                name=f'cnn_kernel_{i}')
+
+            conv = tf.nn.conv2d(pool, kernel, padding='SAME', strides=(1, 1, 1, 1))
+            conv = self._batch_norm(conv, name=f'bn_{i}')
+            relu = tf.nn.relu(conv)
+            
+            pool = tf.nn.max_pool(
+                relu,
+                ksize   = (1, poolVals[i][0],  poolVals[i][1],  1),
+                strides = (1, strideVals[i][0], strideVals[i][1], 1),
+                padding = 'VALID')
+
+        self.cnnOut4d = pool
+        print('[Model] CNN output shape:', pool.get_shape())
+
+    # ── RNN ───────────────────────────────────────────────────────────────────
+    def setupRNN(self):
+        """
+        Two stacked BiLSTMs with MODERATE dropout.
+        
+        ✅ Dropout 0.3 (not 0.5) - Balanced regularization
+        """
+        rnnIn3d = tf.squeeze(self.cnnOut4d, axis=[2])
+
+        numHidden = 384
+        
+        lstm1 = Bidirectional(
+            LSTM(numHidden, return_sequences=True),
+            name='bilstm_1')
+        
+        lstm2 = Bidirectional(
+            LSTM(numHidden, return_sequences=True),
+            name='bilstm_2')
+
+        x = lstm1(rnnIn3d)
+        # ✅ Moderate dropout (0.3, not 0.5)
+        x = tf.nn.dropout(x, rate=self.dropout_rate)
+        x = lstm2(x)
+        x = tf.nn.dropout(x, rate=self.dropout_rate)
+
+        dense = Dense(len(self.charList) + 1, name='output_dense')
+        self.rnnOut3d = dense(x)
+        print('[Model] RNN output shape:', self.rnnOut3d.shape)
+
+    # ── CTC ───────────────────────────────────────────────────────────────────
+    def setupCTC(self):
+        """CTC loss with GENTLE label smoothing."""
+        num_classes = len(self.charList) + 1
+        init_bias = np.zeros(num_classes, dtype=np.float32)
+        init_bias[self.SPACE_IDX] = 1.0
+
+        self.space_logit_bias = tf.Variable(
+            initial_value=init_bias,
+            trainable=True,
+            name='space_logit_bias',
+            dtype=tf.float32)
+
+        self.ctcIn3dTBC_raw = tf.transpose(self.rnnOut3d, [1, 0, 2])
+        self.ctcIn3dTBC = self.ctcIn3dTBC_raw + self.space_logit_bias
+
+        self.gtTexts = tf.SparseTensor(
+            tf.compat.v1.placeholder(tf.int64, shape=[None, 2], name='gt_indices'),
+            tf.compat.v1.placeholder(tf.int32, shape=[None],    name='gt_values'),
+            tf.compat.v1.placeholder(tf.int64, shape=[2],       name='gt_shape'))
+
+        self.seqLen = tf.compat.v1.placeholder(
+            tf.int32, shape=[None], name='seq_len')
+
+        # ✅ GENTLE label smoothing (0.05 instead of 0.1)
+        self.label_smoothing = 0.05
+        
+        ctc_loss_per_sample = tf.compat.v1.nn.ctc_loss(
+            labels             = self.gtTexts,
+            inputs             = self.ctcIn3dTBC,
+            sequence_length    = self.seqLen,
+            ctc_merge_repeated = True,
+            ignore_longer_outputs_than_inputs = True)
+
+        # Space weighting
+        SPACE_WEIGHT = 2.0
+        batch_size_dyn = tf.shape(ctc_loss_per_sample)[0]
+        
+        gt_values  = self.gtTexts.values
+        gt_indices = self.gtTexts.indices
+        batch_idx  = gt_indices[:, 0]
+        
+        is_space = tf.cast(tf.equal(gt_values, self.SPACE_IDX), tf.float32)
+
+        spaces_per_sample = tf.math.unsorted_segment_sum(
+            is_space, batch_idx, num_segments=batch_size_dyn)
+        labels_per_sample = tf.math.unsorted_segment_sum(
+            tf.ones_like(is_space), batch_idx, num_segments=batch_size_dyn)
+
+        space_fraction  = spaces_per_sample / tf.maximum(labels_per_sample, 1.0)
+        sample_weights  = 1.0 + (SPACE_WEIGHT - 1.0) * space_fraction
+
+        # Apply gentle label smoothing
+        smoothed_loss = ctc_loss_per_sample * (1.0 - self.label_smoothing) + \
+                        self.label_smoothing * tf.ones_like(ctc_loss_per_sample)
+        
+        self.loss = tf.reduce_mean(smoothed_loss * sample_weights)
+
+        # Saved-CTC path
+        self.savedCtcInput = tf.compat.v1.placeholder(
+            tf.float32,
+            shape=[Model.CNN_TIME_STEPS, None, len(self.charList) + 1],
+            name='saved_ctc_input')
+
+        self.lossPerElement = tf.compat.v1.nn.ctc_loss(
+            labels             = self.gtTexts,
+            inputs             = self.savedCtcInput,
+            sequence_length    = self.seqLen,
+            ctc_merge_repeated = True)
+
+        # Decoder
+        if self.decoderType == DecoderType.BestPath:
+            self.decoder = tf.nn.ctc_greedy_decoder(
+                inputs=self.ctcIn3dTBC, sequence_length=self.seqLen)
+        elif self.decoderType == DecoderType.BeamSearch:
+            self.decoder = tf.nn.ctc_beam_search_decoder(
+                inputs         = self.ctcIn3dTBC,
+                sequence_length= self.seqLen,
+                beam_width     = 100,
+                merge_repeated = False)
+
+    # ── Optimizer ─────────────────────────────────────────────────────────────
+    def setupOptimizer(self):
+        """Simple Adam optimizer (no gradient accumulation needed)."""
+        optimizer = tf.compat.v1.train.AdamOptimizer(self.learningRate)
+        
+        # Get gradients
+        tvars = tf.compat.v1.trainable_variables()
+        grads = tf.gradients(self.loss, tvars)
+        
+        # Clip gradients
+        grads, _ = tf.clip_by_global_norm(grads, 5.0)
+        
+        # Apply gradients
+        update_ops = tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.UPDATE_OPS)
+        with tf.control_dependencies(update_ops):
+            self.optimizer = optimizer.apply_gradients(zip(grads, tvars))
+
+    # ── TF session ────────────────────────────────────────────────────────────
+    def setupTF(self):
+        print('Python:     ', sys.version)
+        print('TensorFlow: ', tf.__version__)
+
+        config = tf.compat.v1.ConfigProto()
+        config.gpu_options.allow_growth = True
+        
+        sess  = tf.compat.v1.Session(config=config)
+        saver = tf.compat.v1.train.Saver(max_to_keep=5)
+
+        modelDir       = 'model_hindi/'
+        latestSnapshot = tf.train.latest_checkpoint(modelDir)
+
+        if self.mustRestore and not latestSnapshot:
+            raise Exception('No saved model found in: ' + modelDir)
+
+        if latestSnapshot:
+            print('[Model] Restoring from:', latestSnapshot)
+            saver.restore(sess, latestSnapshot)
+            self.lastEpoch = int(latestSnapshot.split('-')[-1])
+        else:
+            print('[Model] Initialising fresh weights')
+            sess.run(tf.compat.v1.global_variables_initializer())
+
+        return (sess, saver)
+
+    # ── Helpers ───────────────────────────────────────────────────────────────
+    def toSparse(self, texts):
+        indices = []
+        values  = []
+        shape   = [len(texts), 0]
+        for (batchElement, text) in enumerate(texts):
+            labelStr = [self.charList.index(c) for c in text]
+            if len(labelStr) > shape[1]:
+                shape[1] = len(labelStr)
+            for (i, label) in enumerate(labelStr):
+                indices.append([batchElement, i])
+                values.append(label)
+        return (indices, values, shape)
+
+    def decoderOutputToText(self, ctcOutput, batchSize):
+        encodedLabelStrs = [[] for _ in range(batchSize)]
+        decoded = ctcOutput[0][0]
+        for (idx, idx2d) in enumerate(decoded.indices):
+            label        = decoded.values[idx]
+            batchElement = idx2d[0]
+            encodedLabelStrs[batchElement].append(label)
+        return [str().join([self.charList[c] for c in labelStr])
+                for labelStr in encodedLabelStrs]
+
+    # ── Train ─────────────────────────────────────────────────────────────────
+    def trainBatch(self, batch, learning_rate=0.0001):
+        """Train on one batch (NO gradient accumulation)."""
+        numBatchElements = len(batch.imgs)
+        sparse = self.toSparse(batch.gtTexts)
+        feedDict = {
+            self.inputImgs   : batch.imgs,
+            self.gtTexts     : sparse,
+            self.seqLen      : [Model.CNN_TIME_STEPS] * numBatchElements,
+            self.learningRate: learning_rate,
+            self.isTraining  : True,
+            self.dropout_rate: 0.3,  # ✅ Moderate dropout
+        }
+        
+        _, lossVal = self.sess.run([self.optimizer, self.loss], feedDict)
+        return lossVal
+
+    # ── Infer ─────────────────────────────────────────────────────────────────
+    def inferBatch(self, batch, calcProbability=False, probabilityOfGT=False):
+        """Inference with dropout disabled."""
+        numBatchElements = len(batch.imgs)
+        evalList = [self.decoder] + ([self.ctcIn3dTBC] if calcProbability else [])
+        feedDict = {
+            self.inputImgs : batch.imgs,
+            self.seqLen    : [Model.CNN_TIME_STEPS] * numBatchElements,
+            self.isTraining: False,
+            self.dropout_rate: 0.0,
+        }
+        evalRes = self.sess.run(evalList, feedDict)
+        decoded = evalRes[0]
+        texts   = self.decoderOutputToText(decoded, numBatchElements)
+
+        probs = None
+        if calcProbability:
+            sparse   = (self.toSparse(batch.gtTexts)
+                        if probabilityOfGT else self.toSparse(texts))
+            ctcInput = evalRes[1]
+            feedDict = {
+                self.savedCtcInput : ctcInput,
+                self.gtTexts       : sparse,
+                self.seqLen        : [Model.CNN_TIME_STEPS] * numBatchElements,
+                self.isTraining    : False,
+                self.dropout_rate  : 0.0,
+            }
+            lossVals = self.sess.run(self.lossPerElement, feedDict)
+            probs    = np.exp(-lossVals)
+
+        return (texts, probs)
+
+    # ── Save ───────────────────────────────────────────────────────────���──────
+    def save(self, epoch):
+        self.snapID += 1
+        self.saver.save(self.sess, 'model_hindi/snapshot', global_step=epoch)
+        print(f'[Model] Checkpoint saved (epoch {epoch})')

SamplePreprocessor_Hindi_v2.py

@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+"""
+SamplePreprocessor_Hindi.py  —  MODERATE AUGMENTATION VERSION
+----------------------------------------------------------
+Balanced augmentation that prevents overfitting WITHOUT making images unreadable.
+
+KEY PRINCIPLE: Model must be able to LEARN from augmented images!
+- Apply only 1-2 augmentations per image (not 5+)
+- Lower probabilities (20-30% instead of 40-60%)
+- Gentler strength (±3° instead of ±5°)
+"""
+
+from __future__ import division
+from __future__ import print_function
+
+import random
+import numpy as np
+import cv2
+
+
+def preprocess(img: np.ndarray,
+               imgSize: tuple,
+               dataAugmentation: bool = False) -> np.ndarray:
+    """
+    Preprocess with MODERATE augmentation.
+    
+    Strategy: Apply 1-2 augmentations per image, not 5+
+    This prevents overfitting while keeping images learnable.
+    """
+
+    if img is None:
+        img = np.zeros([imgSize[1], imgSize[0]], dtype=np.uint8)
+
+    # ── MODERATE DATA AUGMENTATION ───────────────────────────────────────────
+    if dataAugmentation:
+        
+        # Choose 1-2 augmentations randomly (not all of them!)
+        num_augs = random.choice([0, 1, 1, 2])  # Mostly 1, sometimes 2, rarely 0
+        available_augs = ['rotate', 'brightness', 'blur', 'noise', 'scale', 'contrast']
+        selected_augs = random.sample(available_augs, min(num_augs, len(available_augs)))
+        
+        # 1. ROTATION - More gentle (±3° instead of ±5°)
+        if 'rotate' in selected_augs:
+            angle = random.uniform(-3.0, 3.0)  # Reduced from ±5°
+            h, w  = img.shape
+            M     = cv2.getRotationMatrix2D((w // 2, h // 2), angle, 1.0)
+            img   = cv2.warpAffine(img, M, (w, h), borderMode=cv2.BORDER_REPLICATE)
+
+        # 2. BRIGHTNESS - Moderate variation
+        if 'brightness' in selected_augs:
+            delta = random.uniform(-20, 20)  # Reduced from ±30
+            img   = np.clip(img.astype(np.float32) + delta, 0, 255).astype(np.uint8)
+        
+        # 3. CONTRAST - Gentle variation
+        if 'contrast' in selected_augs:
+            factor = random.uniform(0.8, 1.2)  # Reduced from 0.7-1.3
+            img = np.clip((img.astype(np.float32) - 127.5) * factor + 127.5, 
+                          0, 255).astype(np.uint8)
+
+        # 4. BLUR - Mild blur only
+        if 'blur' in selected_augs:
+            kernel_size = 3  # Only size 3
+            sigma = random.uniform(0.3, 1.0)  # Reduced from 0.5-2.0
+            img = cv2.GaussianBlur(img, (kernel_size, kernel_size), sigma)
+
+        # 5. NOISE - Light noise
+        if 'noise' in selected_augs:
+            std = random.uniform(3, 8)  # Reduced from 5-15
+            noise = np.random.normal(0, std, img.shape).astype(np.float32)
+            img = np.clip(img.astype(np.float32) + noise, 0, 255).astype(np.uint8)
+
+        # 6. SCALING - Small scale variation
+        if 'scale' in selected_augs:
+            scale = random.uniform(0.9, 1.1)  # Reduced from 0.85-1.15
+            h, w = img.shape
+            new_h = max(1, int(h * scale))
+            new_w = max(1, int(w * scale))
+            img = cv2.resize(img, (new_w, new_h))
+
+    # ── Resize to target canvas ──────────────────────────────────────────────
+    (wt, ht) = imgSize
+    (h,  w)  = img.shape
+
+    f       = max(w / wt, h / ht)
+    new_w   = max(1, min(wt, int(w / f)))
+    new_h   = max(1, min(ht, int(h / f)))
+
+    img    = cv2.resize(img, (new_w, new_h), interpolation=cv2.INTER_AREA)
+
+    # White canvas
+    canvas = np.ones([ht, wt], dtype=np.uint8) * 255
+    canvas[0:new_h, 0:new_w] = img
+
+    # ── Transpose for TF ──────────────────────────────────────────────────────
+    transposed = cv2.transpose(canvas)
+
+    # ── Normalization ─────────────────────────────────────────────────────────
+    (m, s) = cv2.meanStdDev(transposed)
+    m = m[0][0]
+    s = s[0][0]
+    normalised = (transposed - m) / (s + 1e-8)
+
+    return normalised.astype(np.float32)

build_charlist_hindi.py

@@ -0,0 +1,119 @@
+# -*- coding: utf-8 -*-
+"""
+build_charlist_hindi.py
+-----------------------
+Build charList.txt from Hindi OCR dataset.
+
+Extracts all unique characters from the 'text' column in data.csv
+and creates a character list for CTC training.
+
+Usage:
+    python build_charlist_hindi.py --csv path/to/data.csv
+
+Output:
+    ../model_hindi/charList.txt
+"""
+
+import os
+import argparse
+import pandas as pd
+
+
+def build_charlist(csv_path: str, output_path: str):
+    """
+    Extract all unique characters from CSV and build charList.txt.
+    
+    Format:
+        Index 0: '-' (CTC blank token)
+        Index 1: ' ' (space - critical for word boundaries)
+        Index 2+: All other characters in unicode order
+    """
+    print(f"Loading dataset from: {csv_path}")
+    
+    # Load CSV
+    df = pd.read_csv(csv_path)
+    print(f"Total samples: {len(df)}")
+    
+    # Extract all unique characters
+    all_chars = set()
+    
+    for text in df['text']:
+        all_chars.update(set(text))
+    
+    # Remove empty string if present
+    all_chars.discard("")
+    
+    print(f"\nFound {len(all_chars)} unique characters")
+    
+    # Separate space from other characters
+    space_char = " "
+    other_chars = [c for c in all_chars if c != space_char]
+    
+    # Sort other characters by unicode value
+    other_chars.sort(key=lambda c: ord(c))
+    
+    # Build final charlist:
+    # Index 0 = '-' (CTC blank token) — MUST be first
+    # Index 1 = ' ' (space — critical for word boundaries)
+    # Index 2+ = all other characters in unicode order
+    final_charlist = "-" + space_char + "".join(other_chars)
+    
+    # Make output directory if needed
+    os.makedirs(os.path.dirname(output_path), exist_ok=True)
+    
+    # Save
+    with open(output_path, "w", encoding="utf-8") as f:
+        f.write(final_charlist)
+    
+    print(f"\n✅ charList.txt saved to: {output_path}")
+    print(f"   Total characters: {len(final_charlist)}")
+    print(f"   Index 0 (CTC blank): '-'")
+    print(f"   Index 1 (space):     ' '")
+    print(f"\n   First 50 characters:")
+    print(f"   {repr(final_charlist[:50])}")
+    print(f"\n   Sample Devanagari characters:")
+    
+    # Show some sample Devanagari characters
+    devanagari_chars = [c for c in other_chars if '\u0900' <= c <= '\u097F']
+    print(f"   {repr(''.join(devanagari_chars[:30]))}")
+    
+    # Statistics
+    print(f"\n📊 Character set statistics:")
+    print(f"   Devanagari consonants/vowels: {len(devanagari_chars)}")
+    print(f"   Digits: {sum(c.isdigit() for c in other_chars)}")
+    print(f"   Punctuation: {sum(not c.isalnum() for c in other_chars)}")
+    
+    # Show character distribution
+    print(f"\n   Unicode ranges:")
+    ranges = {
+        'Devanagari': (0x0900, 0x097F),
+        'ASCII': (0x0000, 0x007F),
+        'Latin Extended': (0x0080, 0x00FF),
+    }
+    
+    for name, (start, end) in ranges.items():
+        count = sum(start <= ord(c) <= end for c in other_chars)
+        if count > 0:
+            print(f"   {name}: {count} characters")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Build character list for Hindi OCR dataset")
+    parser.add_argument("--csv", type=str, required=True,
+                        help="Path to data.csv file")
+    parser.add_argument("--output", type=str, 
+                        default="../model_hindi/charList.txt",
+                        help="Output path for charList.txt")
+    
+    args = parser.parse_args()
+    
+    if not os.path.exists(args.csv):
+        print(f"❌ Error: CSV file not found: {args.csv}")
+        return
+    
+    build_charlist(args.csv, args.output)
+
+
+if __name__ == "__main__":
+    main()

main_hindi_v2.py

@@ -0,0 +1,337 @@
+# -*- coding: utf-8 -*-
+"""
+main_hindi_v2.py  —  FAST TRAINING VERSION
+-------------------------------------------------------------
+Optimized training with proper learning rate schedule.
+
+KEY IMPROVEMENTS:
+  ✅ Warmup + cosine decay learning rate
+  ✅ Larger batch size = faster training
+  ✅ Less verbose logging
+  ✅ Better early stopping
+"""
+
+from __future__ import division
+from __future__ import print_function
+
+import os
+import sys
+import argparse
+import codecs
+import gc
+import json
+import time
+from datetime import datetime
+import math
+
+import cv2
+import numpy as np
+import tensorflow as tf
+import editdistance
+
+from DataLoader_Hindi_v2 import DataLoader, Batch
+from Model_Hindi_v2 import Model, DecoderType
+from SamplePreprocessor_Hindi_v2 import preprocess
+
+tf.compat.v1.disable_eager_execution()
+
+# ── Paths ─────────────────────────────────────────────────────────────────────
+class FilePaths:
+    fnCharList  = "model_hindi/charList.txt"
+    fnAccuracy  = "model_hindi/accuracy.txt"
+    fnMetrics   = "model_hindi/metrics.json"
+
+
+# ── Learning rate schedule ────────────────────────────────────────────────────
+def get_learning_rate(epoch, total_epochs=50):
+    """
+    Warmup + Cosine decay learning rate.
+    
+    Better than step decay for convergence.
+    """
+    warmup_epochs = 3
+    base_lr = 0.001
+    min_lr = 0.00001
+    
+    if epoch < warmup_epochs:
+        # Linear warmup
+        return base_lr * (epoch + 1) / warmup_epochs
+    else:
+        # Cosine decay
+        progress = (epoch - warmup_epochs) / (total_epochs - warmup_epochs)
+        return min_lr + (base_lr - min_lr) * 0.5 * (1 + math.cos(math.pi * progress))
+
+
+# ── Metrics tracking ──────────────────────────────────────────────────────────
+class MetricsTracker:
+    """Track training metrics."""
+    
+    def __init__(self, filepath):
+        self.filepath = filepath
+        self.metrics = {
+            'train_loss': [],
+            'val_cer': [],
+            'val_word_acc': [],
+            'learning_rate': [],
+            'epochs': [],
+            'epoch_times': []
+        }
+        
+        if os.path.exists(filepath):
+            try:
+                with open(filepath, 'r') as f:
+                    self.metrics = json.load(f)
+            except:
+                pass
+    
+    def add_epoch(self, epoch, train_loss, val_cer, val_word_acc, lr, epoch_time):
+        """Add metrics for an epoch."""
+        self.metrics['epochs'].append(epoch)
+        self.metrics['train_loss'].append(float(train_loss))
+        self.metrics['val_cer'].append(float(val_cer))
+        self.metrics['val_word_acc'].append(float(val_word_acc))
+        self.metrics['learning_rate'].append(float(lr))
+        self.metrics['epoch_times'].append(float(epoch_time))
+        self.save()
+    
+    def save(self):
+        """Save metrics."""
+        with open(self.filepath, 'w') as f:
+            json.dump(self.metrics, f, indent=2)
+    
+    def get_best_cer(self):
+        """Get best CER."""
+        if not self.metrics['val_cer']:
+            return float('inf')
+        return min(self.metrics['val_cer'])
+
+
+# ── Training ──────────────────────────────────────────────────────────────────
+def train(model: Model, loader: DataLoader, total_epochs=50):
+    """Train with early stopping."""
+    epoch = model.lastEpoch
+    
+    metrics_tracker = MetricsTracker(FilePaths.fnMetrics)
+    bestCharErrorRate = metrics_tracker.get_best_cer()
+    
+    noImprovementSince = 0
+    earlyStopping = 10
+
+    while epoch < total_epochs:
+        epoch += 1
+        epoch_start = time.time()
+        
+        print(f"\n{'='*80}")
+        print(f"Epoch {epoch}/{total_epochs}")
+        print(f"{'='*80}")
+
+        lr = get_learning_rate(epoch, total_epochs)
+        print(f"Learning rate: {lr:.6f}")
+
+        # ── Train ─────────────────────────────────────────────────────────────
+        print("Training...")
+        loader.trainSet(subset_size=10000)
+        epoch_loss = []
+        
+        batch_count = 0
+        while loader.hasNext():
+            batch = loader.getNext()
+            loss = model.trainBatch(batch, learning_rate=lr)
+            epoch_loss.append(loss)
+            batch_count += 1
+            
+            # Print progress every 50 batches
+            if batch_count % 50 == 0:
+                iterInfo = loader.getIteratorInfo()
+                print(f"  Batch {iterInfo[0]:>4}/{iterInfo[1]}  loss={loss:.4f}")
+
+        mean_loss = np.mean(epoch_loss)
+        epoch_time = time.time() - epoch_start
+        print(f"  Epoch loss: {mean_loss:.4f}  Time: {epoch_time/60:.1f}min")
+        
+        loader.cleanup()
+        gc.collect()
+
+        # ── Validate ──────────────────────────────────────────────────────────
+        charErrorRate, wordAccuracy = validate(model, loader)
+        
+        # Track metrics
+        metrics_tracker.add_epoch(epoch, mean_loss, charErrorRate, 
+                                 wordAccuracy, lr, epoch_time)
+        
+        # Checkpoint
+        if charErrorRate < bestCharErrorRate:
+            improvement = (bestCharErrorRate - charErrorRate) * 100
+            print(f"✅ CER improved by {improvement:.2f}pp → Saving model")
+            
+            bestCharErrorRate = charErrorRate
+            noImprovementSince = 0
+            model.save(epoch)
+            
+            with open(FilePaths.fnAccuracy, "w", encoding="utf-8") as f:
+                f.write(f"Epoch: {epoch}\n")
+                f.write(f"CER: {charErrorRate*100:.2f}%\n")
+                f.write(f"Word Acc: {wordAccuracy*100:.2f}%\n")
+                f.write(f"Loss: {mean_loss:.4f}\n")
+        else:
+            print(f"  No improvement ({charErrorRate*100:.2f}% vs {bestCharErrorRate*100:.2f}%)")
+            noImprovementSince += 1
+            
+            # Save every 5 epochs
+            if epoch % 5 == 0:
+                model.save(epoch)
+
+        gc.collect()
+
+        # Early stopping
+        if noImprovementSince >= earlyStopping:
+            print(f"\nEarly stopping: No improvement for {earlyStopping} epochs")
+            print(f"Best CER: {bestCharErrorRate*100:.2f}%")
+            break
+
+
+# ── Validation ────────────────────────────────────────────────────────────────
+def validate(model: Model, loader: DataLoader):
+    """Run validation."""
+    print("Validating...")
+    loader.validationSet()
+
+    numCharErr = 0
+    numCharTotal = 0
+    numWordOK = 0
+    numWordTotal = 0
+    
+    batch_count = 0
+
+    while loader.hasNext():
+        batch = loader.getNext()
+        (recognized, _) = model.inferBatch(batch)
+
+        for i in range(len(recognized)):
+            gt = batch.gtTexts[i]
+            pred = recognized[i]
+
+            numWordOK += 1 if gt == pred else 0
+            numWordTotal += 1
+
+            dist = editdistance.eval(pred, gt)
+            numCharErr += dist
+            numCharTotal += len(gt)
+
+            # Print first few predictions
+            if batch_count == 0 and i < 3:
+                status = "[✓]" if dist == 0 else f"[✗ {dist}]"
+                print(f"  {status} GT:  '{gt[:60]}'")
+                print(f"       PRD: '{pred[:60]}'")
+        
+        batch_count += 1
+
+    charErrorRate = numCharErr / max(numCharTotal, 1)
+    wordAccuracy = numWordOK / max(numWordTotal, 1)
+    
+    print(f"\n{'='*80}")
+    print(f"VALIDATION: CER={charErrorRate*100:.2f}%  Word Acc={wordAccuracy*100:.2f}%")
+    print(f"{'='*80}")
+    
+    loader.cleanup()
+    gc.collect()
+    
+    return charErrorRate, wordAccuracy
+
+
+# ── Inference ─────────────────────────────────────────────────────────────────
+def infer(model: Model, img_path: str) -> str:
+    """Inference on single image."""
+    img = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)
+    if img is None:
+        raise FileNotFoundError(f"Cannot read: {img_path}")
+
+    processed = preprocess(img, Model.imgSize, dataAugmentation=False)
+    batch = Batch(imgs=[processed], gtTexts=None)
+
+    (recognized, probability) = model.inferBatch(batch, calcProbability=True)
+
+    print(f"Recognised: '{recognized[0]}'")
+    print(f"Probability: {probability[0]:.4f}")
+    
+    return recognized[0]
+
+
+# ── Helpers ───────────────────────────────────────────────────────────────────
+def load_char_list() -> str:
+    if not os.path.exists(FilePaths.fnCharList):
+        raise FileNotFoundError(f"charList.txt not found at {FilePaths.fnCharList}")
+    with codecs.open(FilePaths.fnCharList, encoding="utf-8") as f:
+        return f.read()
+
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+def main():
+    parser = argparse.ArgumentParser(description="Hindi OCR - Fast Training")
+    
+    parser.add_argument("--train", action="store_true", help="Train")
+    parser.add_argument("--validate", action="store_true", help="Validate")
+    parser.add_argument("--infer", action="store_true", help="Infer")
+    
+    parser.add_argument("--dataset", type=str, 
+                        default="rajesh-1902/hindi-ocr-dataset",
+                        help="HuggingFace dataset name")
+    parser.add_argument("--images", type=str, default="images/",
+                        help="Path to images directory")
+    parser.add_argument("--image", type=str, default=None,
+                        help="Image for inference")
+    
+    parser.add_argument("--epochs", type=int, default=50,
+                        help="Total epochs to train")
+    parser.add_argument("--beamsearch", action="store_true",
+                        help="Use beam search decoder")
+    
+    args = parser.parse_args()
+
+    decoderType = DecoderType.BeamSearch if args.beamsearch else DecoderType.BestPath
+    charList = load_char_list()
+    
+    print(f"\nCharacter list: {len(charList)} characters")
+    print(f"Index 0: {repr(charList[0])} (blank)")
+    print(f"Index 1: {repr(charList[1])} (space)")
+    
+    os.makedirs("model_hindi/", exist_ok=True)
+
+    if args.train or args.validate:
+        loader = DataLoader(
+            dataset_name=args.dataset,
+            images_dir=args.images,
+            batchSize=Model.batchSize,
+            imgSize=Model.imgSize,
+            maxTextLen=Model.maxTextLen,
+            charList=charList,
+            dataAugmentation=args.train,
+            val_split=0.1
+        )
+
+        # Save corpus
+        
+        if args.train:
+            model = Model(charList, decoderType, mustRestore=False, lastEpoch=0)
+            try:
+                train(model, loader, total_epochs=args.epochs)
+            except KeyboardInterrupt:
+                print(f"\nTraining interrupted at epoch {model.lastEpoch}")
+
+        elif args.validate:
+            model = Model(charList, decoderType, mustRestore=True)
+            validate(model, loader)
+
+    elif args.infer:
+        if not args.image:
+            print("Error: --infer requires --image")
+            sys.exit(1)
+        model = Model(charList, decoderType, mustRestore=True)
+        infer(model, args.image)
+
+    else:
+        parser.print_help()
+
+
+if __name__ == "__main__":
+    main()

model_hindi/accuracy.txt

@@ -0,0 +1,4 @@
+Epoch: 4
+CER: 0.62%
+Word Acc: 84.03%
+Loss: 1.6097

model_hindi/charList.txt

@@ -0,0 +1 @@
+- !"#$%'()*+,-./0123456789:;=?@_`|ँंःअआइईउऊऋएऐऑओऔकखगघचछजझञटठडढणतथदधनपफबभमयरऱलवशषसह़ािीुूृॅेैॉोौ्ख़ज़ड़ढ़फ़​‌‍‏–‘’“”…₹

model_hindi/checkpoint

@@ -0,0 +1,5 @@
+model_checkpoint_path: "snapshot-5"
+all_model_checkpoint_paths: "snapshot-1"
+all_model_checkpoint_paths: "snapshot-2"
+all_model_checkpoint_paths: "snapshot-4"
+all_model_checkpoint_paths: "snapshot-5"

model_hindi/metrics.json

@@ -0,0 +1,44 @@
+{
+  "train_loss": [
+    92.00146484375,
+    4.817220211029053,
+    2.3719639778137207,
+    1.6097475290298462,
+    1.2168760299682617
+  ],
+  "val_cer": [
+    0.0749026983742714,
+    0.012815881115104564,
+    0.018309465725618725,
+    0.006242201903201177,
+    0.007158420082310658
+  ],
+  "val_word_acc": [
+    0.1515625,
+    0.72609375,
+    0.62203125,
+    0.8403125,
+    0.81859375
+  ],
+  "learning_rate": [
+    0.0006666666666666666,
+    0.001,
+    0.001,
+    0.0009509795896116975,
+    0.0008136274519200731
+  ],
+  "epochs": [
+    1,
+    2,
+    3,
+    4,
+    5
+  ],
+  "epoch_times": [
+    1097.182646036148,
+    1089.5408215522766,
+    1088.228083372116,
+    1079.393277645111,
+    1083.2388920783997
+  ]
+}

model_hindi/snapshot-1.data-00000-of-00001

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4291c4d3442d1caa732a2509144b5843896b2e424855009a89813b5ce4a5801a
+size 95642112

model_hindi/snapshot-1.meta

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8513619f1dadb01b917437682f67dd532324fd8998745c5c658058d01cced60
+size 765975

model_hindi/snapshot-2.data-00000-of-00001

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02bffa6061034b19754288469c499d40338ecf59438f1ad33866309cd6a450e0
+size 95642112

model_hindi/snapshot-2.meta

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8513619f1dadb01b917437682f67dd532324fd8998745c5c658058d01cced60
+size 765975

model_hindi/snapshot-4.data-00000-of-00001

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd04849088828eb6dcdf214571cd6645feb10c3aece45d460403e1f0c70c7a7d
+size 95642112

model_hindi/snapshot-4.meta

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8513619f1dadb01b917437682f67dd532324fd8998745c5c658058d01cced60
+size 765975

model_hindi/snapshot-5.data-00000-of-00001

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9bc0e778b55ece9933bb335fa75c8933e9a224e58ec218ddd12b65f873619cf
+size 95642112

model_hindi/snapshot-5.meta

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8513619f1dadb01b917437682f67dd532324fd8998745c5c658058d01cced60
+size 765975