Update app/utils.py

app/utils.py

@@ -4,9 +4,7 @@ from transformers import AutoModel, AutoTokenizer
 from PIL import Image, ImageEnhance, ImageFilter
 import torch
 import logging
-import shutil
-import os
-
+from transformers import BertTokenizer
 
 logger = logging.getLogger(__name__)
 
@@ -22,19 +20,21 @@ class OCRModel:
     def initialize(self):
         try:
             logger.info("Initializing OCR model...")
-            # Try loading with use_fast=False if the fast tokenizer fails
+            
+            # Try different tokenizer approaches
             try:
+                # First try with the standard approach
                 self.tokenizer = AutoTokenizer.from_pretrained(
-                    'stepfun-ai/GOT-OCR-2.0-hf', 
+                    'stepfun-ai/GOT-OCR-2.0-hf',
                     trust_remote_code=True,
-                    use_fast=False  # Try with slow tokenizer
+                    use_fast=False
                 )
             except Exception as e:
-                logger.warning(f"Fast tokenizer failed, trying alternative: {str(e)}")
-                self.tokenizer = AutoTokenizer.from_pretrained(
-                    'stepfun-ai/GOT-OCR-2.0-hf', 
-                    trust_remote_code=True,
-                    use_fast=False
+                logger.warning(f"Standard tokenizer failed, trying BertTokenizer: {str(e)}")
+                # Fall back to BertTokenizer if AutoTokenizer fails
+                self.tokenizer = BertTokenizer.from_pretrained(
+                    'stepfun-ai/GOT-OCR-2.0-hf',
+                    trust_remote_code=True
                 )
                 
             self.model = AutoModel.from_pretrained(
@@ -42,8 +42,7 @@ class OCRModel:
                 trust_remote_code=True,
                 low_cpu_mem_usage=True,
                 device_map='auto',
-                use_safetensors=True,
-                pad_token_id=self.tokenizer.eos_token_id
+                use_safetensors=True
             )
             
             self.device = "cuda" if torch.cuda.is_available() else "cpu"
@@ -54,7 +53,7 @@ class OCRModel:
         except Exception as e:
             logger.error(f"Error initializing model: {str(e)}", exc_info=True)
             raise
-
+            
     def preprocess_image(self, image):
         """Image preprocessing to improve text recognition quality"""
         try: