updated the code and model

app.py

@@ -0,0 +1,121 @@
+import gradio as gr
+from PIL import Image
+import torch
+from transformers import AutoFeatureExtractor, AutoTokenizer, TrOCRProcessor, VisionEncoderDecoderModel
+import os
+
+# def sauvola_thresholding(grayImage_, window_size=15):
+#     """
+#     Sauvola thresholds are local thresholding techniques that are 
+#     useful for images where the background is not uniform, especially for text recognition.
+    
+#     grayImage_ --- Input image should be in 2-Dimension Gray Scale format.
+#     window_size --- It represents the filter window size.
+#     """
+#     # Assert the input conditions
+#     assert len(grayImage_.shape) == 2, "Input image must be a 2-dimensional gray scale image."
+#     assert isinstance(window_size, int) and window_size > 0, "Window size must be a positive integer."
+    
+#     thresh_sauvolavalue = threshold_sauvola(grayImage_, window_size=window_size)
+#     thresholdImage_ = (grayImage_ > thresh_sauvolavalue)
+    
+#     return thresholdImage_
+
+class OCRModel:
+    def __init__(self, encoder_model, decoder_model, trained_model_path):
+        # Load processor and model
+        self.feature_extractor = AutoFeatureExtractor.from_pretrained(encoder_model)
+        self.decoder_tokenizer = AutoTokenizer.from_pretrained(decoder_model)
+        self.processor = TrOCRProcessor(feature_extractor=self.feature_extractor, tokenizer=self.decoder_tokenizer)
+        self.model = VisionEncoderDecoderModel.from_pretrained(trained_model_path)
+        
+        # Configure model settings
+        self.model.config.decoder_start_token_id = self.processor.tokenizer.cls_token_id
+        self.model.config.pad_token_id = self.processor.tokenizer.pad_token_id
+        self.model.config.vocab_size = self.model.config.decoder.vocab_size
+        self.model.config.eos_token_id = self.processor.tokenizer.sep_token_id
+        self.model.config.max_length = 64
+        self.model.config.early_stopping = True
+        self.model.config.no_repeat_ngram_size = 3
+        self.model.config.length_penalty = 2.0
+        self.model.config.num_beams = 4
+        
+    def read_and_show(self, image_path):
+        """
+        Reads an image from the provided path and converts it to RGB.
+        :param image_path: String, path to the input image.
+        :return: PIL Image object
+        """
+        image = Image.open(image_path).convert('RGB')
+        return image
+
+    def ocr(self, image):
+        """
+        Performs OCR on the given image.
+        :param image: PIL Image object.
+        :return: Extracted text from the image.
+        """
+        # Preprocess the image
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        pixel_values = self.processor(image, return_tensors='pt').pixel_values.to(device)
+
+        # Generate text
+        generated_ids = self.model.generate(pixel_values)
+        generated_text = self.processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
+        return generated_text
+
+# Initialize the OCR model
+ocr_model = OCRModel(encoder_model="google/vit-base-patch16-224-in21k",
+                     decoder_model="surajp/RoBERTa-hindi-guj-san",
+                     trained_model_path="./model/") #'sabaridsnfuji/Tamil_Offline_Handwritten_OCR')#"./model/")
+
+
+    
+def main(image_path):
+# Process the image and extract text
+    image = ocr_model.read_and_show(image_path)
+    text = ocr_model.ocr(image)
+    
+    return image, text
+
+# Gradio Interface function
+def gradio_interface(image):
+    # Save the uploaded image locally
+    image_path = "uploaded_image.png"
+    image.save(image_path)
+    
+    # Call the main function to process the image and get the result
+    processed_image, result_text = main(image_path)
+    
+    return processed_image, result_text
+
+# Sample images for demonstration (make sure these image paths exist)
+sample_images = [
+    "./sample/16.jpg",  # replace with actual image paths
+    "./sample/20.jpg",  # replace with actual image paths
+    "./sample/21.jpg",  # replace with actual image paths
+    "./sample/31.jpg",  # replace with actual image paths
+    "./sample/35.jpg",  # replace with actual image paths
+]
+
+# Ensure sample images directory exists
+os.makedirs("samples", exist_ok=True)
+# Save some dummy sample images if they don't exist (you should replace these with actual images)
+for i, sample in enumerate(sample_images):
+    if not os.path.exists(sample):
+        img = Image.new("RGB", (224, 224), color=(i * 50, i * 50, i * 50))
+        img.save(sample)
+
+# Gradio UI setup with examples
+gr_interface = gr.Interface(
+    fn=gradio_interface, 
+    inputs=gr.Image(type="pil"),  # Updated to gr.Image
+    outputs=[gr.Image(type="pil"), gr.Textbox()],  # Updated to gr.Image and gr.Textbox
+    title="Hindi Handwritten OCR Recognition", 
+    description="Upload a cropped image containing a word, or use the sample images below to recognize the text. This is a word recognition model. Currently, text detection is not supported.",
+    examples=sample_images  # Add the examples here
+)
+
+# Launch the Gradio interface
+if __name__ == "__main__":
+    gr_interface.launch()

model/config.json

@@ -0,0 +1,182 @@
+{
+  "_name_or_path": "/content/pre_trained_net/",
+  "architectures": [
+    "VisionEncoderDecoderModel"
+  ],
+  "decoder": {
+    "_attn_implementation_autoset": true,
+    "_name_or_path": "surajp/RoBERTa-hindi-guj-san",
+    "add_cross_attention": true,
+    "architectures": [
+      "RobertaForMaskedLM"
+    ],
+    "attention_probs_dropout_prob": 0.1,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": 0,
+    "chunk_size_feed_forward": 0,
+    "classifier_dropout": null,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 2,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "gradient_checkpointing": false,
+    "hidden_act": "gelu",
+    "hidden_dropout_prob": 0.1,
+    "hidden_size": 768,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "is_decoder": true,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layer_norm_eps": 1e-05,
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "max_position_embeddings": 514,
+    "min_length": 0,
+    "model_type": "roberta",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 12,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 6,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": 1,
+    "position_embedding_type": "absolute",
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": null,
+    "torchscript": false,
+    "type_vocab_size": 1,
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "use_cache": true,
+    "vocab_size": 30522
+  },
+  "decoder_start_token_id": 0,
+  "early_stopping": null,
+  "encoder": {
+    "_attn_implementation_autoset": true,
+    "_name_or_path": "google/vit-base-patch16-224-in21k",
+    "add_cross_attention": false,
+    "architectures": [
+      "ViTModel"
+    ],
+    "attention_probs_dropout_prob": 0.0,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": null,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "encoder_stride": 16,
+    "eos_token_id": null,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "gelu",
+    "hidden_dropout_prob": 0.0,
+    "hidden_size": 768,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "image_size": 224,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layer_norm_eps": 1e-12,
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "min_length": 0,
+    "model_type": "vit",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 12,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_channels": 3,
+    "num_hidden_layers": 12,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "patch_size": 16,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "qkv_bias": true,
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": null,
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false
+  },
+  "eos_token_id": 2,
+  "is_encoder_decoder": true,
+  "length_penalty": null,
+  "max_length": null,
+  "model_type": "vision-encoder-decoder",
+  "no_repeat_ngram_size": null,
+  "num_beams": null,
+  "pad_token_id": 1,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.46.3",
+  "vocab_size": 30522
+}

model/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 0,
+  "decoder_start_token_id": 0,
+  "early_stopping": true,
+  "eos_token_id": 2,
+  "length_penalty": 2.0,
+  "max_length": 64,
+  "no_repeat_ngram_size": 3,
+  "num_beams": 4,
+  "pad_token_id": 1,
+  "transformers_version": "4.46.3"
+}

model/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aff44c11a9f00953a66d58e95e327f7883df105aea2c2140ca80583dddd818d2
+size 670287760

requirements.txt

@@ -0,0 +1,4 @@
+torch
+transformers
+datasets
+Pillow