Training in progress, step 1000

adapter_config.json

@@ -23,11 +23,11 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "query",
+    "output.dense",
+    "intermediate.dense",
     "value",
     "key",
-    "output.dense",
-    "intermediate.dense"
+    "query"
   ],
   "task_type": null,
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:680245107cdadb5055f67d67bf7d89865479e0428a3b75bd5bca72ec83d94725
+oid sha256:ace69a40479cb7818005942179ac01520eee1b042f570b77c58aa179b9f20d6d
 size 10637752

config.json

@@ -173,6 +173,5 @@
   "processor_class": "TrOCRProcessor",
   "tie_word_embeddings": false,
   "torch_dtype": "float32",
-  "transformers_version": "4.42.3",
-  "vocab_size": 50265
+  "transformers_version": "4.44.2"
 }

finetune_phi3_vision.py

@@ -8,10 +8,59 @@ from transformers import AutoProcessor, BitsAndBytesConfig
 from transformers import AutoModelForCausalLM, AutoModelForVision2Seq
 from datetime import datetime
 import evaluate
+from transformers import TrainingArguments, Trainer, Seq2SeqTrainer, Seq2SeqTrainingArguments
+from sklearn.model_selection import train_test_split
+
+import random
+
+class MyDataCollator:
+    def __init__(self, processor):
+        self.processor = processor
+        self.image_token_id = processor.tokenizer.additional_special_tokens_ids[
+            processor.tokenizer.additional_special_tokens.index("<image>")
+        ]
+
+    def __call__(self, examples):
+        texts = []
+        images = []
+        for example in examples:
+            image = example["image"]
+            # print(example["query"])
+            question = example["query"]
+            answer = example["answers"]
+            messages = [
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": "OCR the text in the image."},
+                        {"type": "image"},
+                        {"type": "text", "text": question}
+                    ]
+                },
+                {
+                    "role": "assistant",
+                    "content": [
+                        {"type": "text", "text": answer}
+                    ]
+                }
+            ]
+            text = processor.apply_chat_template(messages, add_generation_prompt=False)
+            texts.append(text.strip())
+            images.append([image])
+
+        batch = processor(text=texts, images=images, return_tensors="pt", padding=True)
+
+        labels = batch["input_ids"].clone()
+        # labels[labels == processor.tokenizer.pad_token_id] = self.image_token_id
+        batch["labels"] = labels
+
+        return batch
+        
 # Define train and test size.
 TRAIN_SAMPLES = 1000
 TEST_SAMPLES = 200
 TEST_SIZE = 0.166 #
+samp_list = [1, 15000, 30000, 45000, 60000, 70000]
 
 # Define the directory containing the images.
 df_path = "/mnt/data1/Datasets/AlphaPen/" + "training_data.csv"
@@ -19,9 +68,11 @@ df = pd.read_csv(df_path)
 df.dropna(inplace=True)
 df["id"] = range(df.shape[0])
 df["query"] = "What is shown in this image?"
+train_df, test_df = train_test_split(df, test_size=0.02, random_state=0)
 
 root_dir = "/mnt/data1/Datasets/OCR/Alphapen/clean_data/final_cropped_rotated_"
-image_paths = [root_dir + img for img in df.filename]
+image_paths_train = [root_dir + img for img in train_df.filename]
+image_paths_test = [root_dir + img for img in test_df.filename]
 
 # New batch
 df_path_2 = "/mnt/data1/Datasets/AlphaPen/" + "training_b2.csv"
@@ -29,38 +80,44 @@ df_2 = pd.read_csv(df_path_2)
 df_2.dropna(inplace=True)
 df_2["id"] = range(df_2.shape[0])
 df_2["query"] = "What is shown in this image?"
+train_df_b2, test_df_b2 = train_test_split(df_2, test_size=0.01, random_state=0)
 
 root_dir_2 = "/mnt/data1/Datasets/OCR/Alphapen/DataBatch2/clean_data/cropped_data/cropped_"
-image_paths_2 = [root_dir_2 + img for img in df_2.filename]
-# Create a list of other columns such as id, query, and answer.
-ids = range(df.shape[0] + df_2.shape[0])
-queries = df['query'].tolist() + df_2['query'].tolist()
-answers = df['text'].tolist() + df_2['text'].tolist()
+image_paths_2_train = [root_dir_2 + img for img in train_df_b2.filename]
+image_paths_2_test = [root_dir_2 + img for img in test_df_b2.filename]
+
+
+ids_test = range(test_df.shape[0] + test_df_b2.shape[0])
+queries_test = test_df['query'].tolist() + test_df_b2['query'].tolist()
+answers_test = test_df['text'].tolist() + test_df_b2['text'].tolist()
 
 # Create the dataset dictionary.
-dataset_dict = {
-    'id': ids,
-    'image': image_paths + image_paths_2,
-    'query': queries,
-    'answers': answers
+
+
+eval_dataset_dict = {
+    'id': ids_test,
+    'image': image_paths_test + image_paths_2_test,
+    'query': queries_test,
+    'answers': answers_test
 }
 
 # Create the dataset.
-dataset = Dataset.from_dict(dataset_dict)
+
+eval_dataset = Dataset.from_dict(eval_dataset_dict)
 
 # Cast the 'image' column to Image type.
-dataset = dataset.cast_column("image", Image())
+
+eval_dataset = eval_dataset.cast_column("image", Image())
 
 # Split the dataset into train and test.
-split_dataset = dataset.train_test_split(test_size=TEST_SIZE, shuffle=False)
+# split_dataset = dataset.train_test_split(test_size=TEST_SIZE, shuffle=False)
 
-train_dataset = split_dataset["train"]
-eval_dataset  = split_dataset["test"]
-print(len(train_dataset))
+# train_dataset = split_dataset["train"]
+# eval_dataset  = split_dataset["test"]
+print(len(eval_dataset))
 # Push the dataset on Hugging Face Hub.
 # split_dataset.push_to_hub("NSTiwari/DocumentIDEFICS_QA")
 
-os.environ["WANDB_PROJECT"]="Alphapen"
 
 # Define model ID
 # model_id = "microsoft/Phi-3-vision-128k-instruct"
@@ -121,112 +178,86 @@ else:
 
 
 
-import random
-
-class MyDataCollator:
-    def __init__(self, processor):
-        self.processor = processor
-        self.image_token_id = processor.tokenizer.additional_special_tokens_ids[
-            processor.tokenizer.additional_special_tokens.index("<image>")
-        ]
-
-    def __call__(self, examples):
-        texts = []
-        images = []
-        for example in examples:
-            image = example["image"]
-            # print(example["query"])
-            question = example["query"]
-            answer = example["answers"]
-            messages = [
-                {
-                    "role": "user",
-                    "content": [
-                        {"type": "text", "text": "OCR the text in the image."},
-                        {"type": "image"},
-                        {"type": "text", "text": question}
-                    ]
-                },
-                {
-                    "role": "assistant",
-                    "content": [
-                        {"type": "text", "text": answer}
-                    ]
-                }
-            ]
-            text = processor.apply_chat_template(messages, add_generation_prompt=False)
-            texts.append(text.strip())
-            images.append([image])
-
-        batch = processor(text=texts, images=images, return_tensors="pt", padding=True)
-
-        labels = batch["input_ids"].clone()
-        # labels[labels == processor.tokenizer.pad_token_id] = self.image_token_id
-        batch["labels"] = labels
 
-        return batch
 
 data_collator = MyDataCollator(processor)
 
-from transformers import TrainingArguments, Trainer, Seq2SeqTrainer, Seq2SeqTrainingArguments
 
-training_args = Seq2SeqTrainingArguments(
-    predict_with_generate=True,
-    output_dir = "idefics2",
-    learning_rate = 2e-4,
-    fp16 = True,
-    per_device_train_batch_size = 8,
-    per_device_eval_batch_size = 8,
-    gradient_accumulation_steps = 2,
-    dataloader_pin_memory = False,
-    save_total_limit = 3,
-    eval_strategy ="steps",
-    save_strategy = "steps",
-    eval_steps = 200,
-    save_steps = 10000,
-    max_steps = 50000,
-    logging_steps = 10,
-    remove_unused_columns = False,
-    push_to_hub=True,
-    label_names = ["labels"],
-    load_best_model_at_end = False,
-    report_to = "wandb",
-    optim = "paged_adamw_8bit",
-    run_name=f"idefics2-vision-LoRA-{datetime.now().strftime('%Y-%m-%d-%H-%M-%s')}",
-    hub_model_id="hadrakey/alphapen_idefics2_finetune_v1",
-)
+for samp in samp_list:
+    os.environ["WANDB_PROJECT"]="Alphapen"
+    # Create a list of other columns such as id, query, and answer.
+    ids_train = range(train_df.shape[0] + train_df_b2.shape[0])
+    queries_train = train_df['query'].tolist() + train_df_b2['query'].tolist()
+    answers_train = train_df['text'].tolist() + train_df_b2['text'].tolist()
+
+    train_dataset_dict = {
+    'id': ids_train,
+    'image': image_paths_train + image_paths_2_train,
+    'query': queries_train,
+    'answers': answers_train
+    }
+
+    train_dataset = Dataset.from_dict(train_dataset_dict)
+    train_dataset = train_dataset.cast_column("image", Image())
+
+    training_args = Seq2SeqTrainingArguments(
+        predict_with_generate=True,
+        output_dir = "idefics2",
+        learning_rate = 2e-4,
+        fp16 = True,
+        per_device_train_batch_size = 8,
+        per_device_eval_batch_size = 8,
+        gradient_accumulation_steps = 2,
+        dataloader_pin_memory = False,
+        save_total_limit = 3,
+        eval_strategy ="steps",
+        save_strategy = "steps",
+        eval_steps = 500,
+        save_steps = 1000,
+        max_steps = 5000,
+        logging_steps = 10,
+        remove_unused_columns = False,
+        push_to_hub=True,
+        label_names = ["labels"],
+        load_best_model_at_end = False,
+        report_to = "wandb",
+        optim = "paged_adamw_8bit",
+        # run_name=f"idefics2-vision-LoRA-{datetime.now().strftime('%Y-%m-%d-%H-%M-%s')}",
+        run_name="idefics2-vision-LoRA-" + str(samp),
+        hub_model_id="hadrakey/alphapen_idefics2_" + str(samp),
+    )
 
-def compute_metrics(pred):
-    # accuracy_metric = evaluate.load("precision")
-    cer_metric = evaluate.load("cer")
-
-    labels_ids = pred.label_ids
-    pred_ids = pred.predictions
-    # print(pred_ids)
-    # print(labels_ids)
-    # max_length = max(pred_ids.shape[1], labels_ids.shape[1])
-    # generated_texts = processor.batch_decode(generated_ids[:, inputs["input_ids"].size(1):], skip_special_tokens=True)
-    pred_str = processor.batch_decode(pred_ids, skip_special_tokens=True)
-    pred_str = [word.lower() for word in pred_str]
-    # print(pred_str)
-    # pred_str = processor.batch_decode(pred_ids, skip_special_tokens=True)
-    labels_ids[labels_ids == -100] = processor.tokenizer.pad_token_id
-    label_str = processor.batch_decode(labels_ids, skip_special_tokens=True)
-    label_str = [word.lower() for word in label_str]
-    # print(label_str)
-    cer = cer_metric.compute(predictions=pred_str, references=label_str)
-    # accuracy = accuracy_metric.compute(predictions=pred_ids.tolist(), references=labels_ids.tolist())
-
-    return {"cer": cer}
-
-
-trainer = Seq2SeqTrainer(
-    model = model,
-    args = training_args,
-    data_collator = data_collator,
-    train_dataset = train_dataset,
-    eval_dataset = eval_dataset,
-    compute_metrics=compute_metrics,
-)
+    def compute_metrics(pred):
+        # accuracy_metric = evaluate.load("precision")
+        cer_metric = evaluate.load("cer")
+
+        labels_ids = pred.label_ids
+        pred_ids = pred.predictions
+        # print(pred_ids)
+        # print(labels_ids)
+        # max_length = max(pred_ids.shape[1], labels_ids.shape[1])
+        # generated_texts = processor.batch_decode(generated_ids[:, inputs["input_ids"].size(1):], skip_special_tokens=True)
+        pred_str = processor.batch_decode(pred_ids, skip_special_tokens=True)
+        pred_str = [word.lower() for word in pred_str]
+        # print(pred_str)
+        # pred_str = processor.batch_decode(pred_ids, skip_special_tokens=True)
+        labels_ids[labels_ids == -100] = processor.tokenizer.pad_token_id
+        label_str = processor.batch_decode(labels_ids, skip_special_tokens=True)
+        label_str = [word.lower() for word in label_str]
+        # print(label_str)
+        cer = cer_metric.compute(predictions=pred_str, references=label_str)
+        # accuracy = accuracy_metric.compute(predictions=pred_ids.tolist(), references=labels_ids.tolist())
+
+        return {"cer": cer}
+
+
+    trainer = Seq2SeqTrainer(
+        model = model,
+        args = training_args,
+        data_collator = data_collator,
+        train_dataset = train_dataset,
+        eval_dataset = eval_dataset,
+        compute_metrics=compute_metrics,
+    )
 
-trainer.train()
+    trainer.train()

idefics2/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1a529961f33fd865061f2d504f10e9dbda5d36ac583ca54c807b178a3eef0a02
+oid sha256:7e38855b7b26c79a86d6bc42985348143f602714b85923b6fcf6793830f400de
 size 746528304

idefics2/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:42f9baac86ca26a549fd61c40400a3efd2e95f6a3486ca7a7482e10ccfbb4ac6
+oid sha256:c886ec66a448f0680d0a46cd28b697b6899ecc0627e105de6d1eac26f3c78140
 size 5368

inference.py

@@ -1,9 +1,15 @@
 from transformers import TrOCRProcessor, VisionEncoderDecoderModel
 import pandas as pd
 from PIL import Image
+from torchmetrics.text import CharErrorRate
 
 # Finetuned model
-model_finetune = VisionEncoderDecoderModel.from_pretrained("hadrakey/alphapen_trocr")
+model_finetune_1 = VisionEncoderDecoderModel.from_pretrained("hadrakey/alphapen_new_large_1")
+model_finetune_2 = VisionEncoderDecoderModel.from_pretrained("hadrakey/alphapen_new_large_15000")
+model_finetune_3 = VisionEncoderDecoderModel.from_pretrained("hadrakey/alphapen_new_large_30000")
+model_finetune_4 = VisionEncoderDecoderModel.from_pretrained("hadrakey/alphapen_new_large_45000")
+model_finetune_5 = VisionEncoderDecoderModel.from_pretrained("hadrakey/alphapen_new_large_60000")
+model_finetune_6 = VisionEncoderDecoderModel.from_pretrained("hadrakey/alphapen_new_large_70000")
 
 #Baseline
 model_base = VisionEncoderDecoderModel.from_pretrained("microsoft/trocr-base-handwritten")
@@ -15,24 +21,78 @@ df_path = "/mnt/data1/Datasets/AlphaPen/" + "testing_data.csv"
 data = pd.read_csv(df_path)
 data.dropna(inplace=True)
 data.reset_index(inplace=True)
+sample = data.iloc[:50,:]
 
 root_dir = "/mnt/data1/Datasets/OCR/Alphapen/clean_data/"
 
 inf_baseline = []
-inf_finetune = []
-for idx in range(len(data)):
-    image = Image.open(root_dir + "final_cropped_rotated_" + data.filename[idx]).convert("RGB")
+inf_finetune_1 = []
+inf_finetune_2 = []
+inf_finetune_3 = []
+inf_finetune_4 = []
+inf_finetune_5 = []
+inf_finetune_6 = []
+
+cer_fine_1 = []
+cer_fine_2 = []
+cer_fine_3 = []
+cer_fine_4 = []
+cer_fine_5 = []
+cer_fine_6 = []
+cer_base = []
+
+cer_metric = CharErrorRate()
+
+for idx in range(len(sample)):
+    image = Image.open(root_dir + "final_cropped_rotated_" + sample.filename[idx]).convert("RGB")
     
     pixel_values = processor(image, return_tensors="pt").pixel_values
     generated_ids_base = model_base.generate(pixel_values)
-    generated_ids_fine = model_finetune.generate(pixel_values)
+    generated_ids_fine_1 = model_finetune_1.generate(pixel_values)
+    generated_ids_fine_2= model_finetune_2.generate(pixel_values)
+    generated_ids_fine_3 = model_finetune_3.generate(pixel_values)
+    generated_ids_fine_4 = model_finetune_4.generate(pixel_values)
+    generated_ids_fine_5 = model_finetune_5.generate(pixel_values)
+    generated_ids_fine_6 = model_finetune_6.generate(pixel_values)
+
     generated_text_base = processor.batch_decode(generated_ids_base, skip_special_tokens=True)[0]
-    generated_text_fine= processor.batch_decode(generated_ids_fine, skip_special_tokens=True)[0]
+    generated_text_fine_1= processor.batch_decode(generated_ids_fine_1, skip_special_tokens=True)[0]
+    generated_text_fine_2= processor.batch_decode(generated_ids_fine_2, skip_special_tokens=True)[0]
+    generated_text_fine_3= processor.batch_decode(generated_ids_fine_3, skip_special_tokens=True)[0]
+    generated_text_fine_4= processor.batch_decode(generated_ids_fine_4, skip_special_tokens=True)[0]
+    generated_text_fine_5= processor.batch_decode(generated_ids_fine_5, skip_special_tokens=True)[0]
+    generated_text_fine_6= processor.batch_decode(generated_ids_fine_6, skip_special_tokens=True)[0]
+
+    cer_fine_1.append(cer_metric(generated_text_fine_1.lower(), sample.text[idx].lower()).detach().numpy())
+    cer_fine_2.append(cer_metric(generated_text_fine_2.lower(), sample.text[idx].lower()).detach().numpy())
+    cer_fine_3.append(cer_metric(generated_text_fine_3.lower(), sample.text[idx].lower()).detach().numpy())
+    cer_fine_4.append(cer_metric(generated_text_fine_4.lower(), sample.text[idx].lower()).detach().numpy())
+    cer_fine_5.append(cer_metric(generated_text_fine_5.lower(), sample.text[idx].lower()).detach().numpy())
+    cer_fine_6.append(cer_metric(generated_text_fine_6.lower(), sample.text[idx].lower()).detach().numpy())
+    cer_base.append(cer_metric(generated_text_base.lower(), sample.text[idx].lower()).detach().numpy())
     
     inf_baseline.append(generated_text_base)
-    inf_finetune.append(generated_text_fine)
+    inf_finetune_1.append(generated_text_fine_1)
+    inf_finetune_2.append(generated_text_fine_2)
+    inf_finetune_3.append(generated_text_fine_3)
+    inf_finetune_4.append(generated_text_fine_4)
+    inf_finetune_5.append(generated_text_fine_5)
+    inf_finetune_6.append(generated_text_fine_6)
+
+sample["Baseline"]=inf_baseline
+sample["Finetune_1"]=inf_finetune_1 
+sample["Finetune_2"]=inf_finetune_2  
+sample["Finetune_3"]=inf_finetune_3  
+sample["Finetune_4"]=inf_finetune_4  
+sample["Finetune_5"]=inf_finetune_5   
+sample["Finetune_6"]=inf_finetune_6  
 
-data["Baseline"]=inf_baseline
-data["Finetune"]=inf_finetune  
+sample["cer_1"]=cer_fine_1 
+sample["cer_2"]=cer_fine_2  
+sample["cer_3"]=cer_fine_3  
+sample["cer_4"]=cer_fine_4  
+sample["cer_5"]=cer_fine_5   
+sample["cer_6"]=cer_fine_6
+sample["cer_base"]=cer_base
 
-data.to_csv("/mnt/data1/Datasets/AlphaPen/" + "inference_data.csv")
+sample.to_csv("/mnt/data1/Datasets/AlphaPen/" + "inference_results.csv")

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4b5c9a57267f9760bd4a7647d8ba830dc2631f0d0cd17ce843b0c314a00d5e79
+oid sha256:b14472ca382e9d96ea7efd3c778cbf0b73a412e31bc41cfec8d97e8988e6063d
 size 1335747032

trainer_lora.py

@@ -18,8 +18,9 @@ from src.loss import MarginLoss, KLRegularization
 from src.similarity import CERSimilarity
 from datetime import datetime
 from torch.utils.data import ConcatDataset
+import wandb
+
 
-os.environ["WANDB_PROJECT"] = "Alphapen-TrOCR"
 
 # @dataclass
 # class ScriptArguments:
@@ -56,10 +57,10 @@ root_dir = "/mnt/data1/Datasets/OCR/Alphapen/clean_data/final_cropped_rotated_"
 root_dir_b2 = "/mnt/data1/Datasets/OCR/Alphapen/DataBatch2/clean_data/cropped_data/cropped_"
 processor = TrOCRProcessor.from_pretrained(model_name)
 
-train_dataset_b1 = AphaPenDataset(root_dir=root_dir, df=train_df,  processor=processor)
-eval_dataset_b1 = AphaPenDataset(root_dir=root_dir, df=test_df,  processor=processor)
+train_dataset_b1 = AphaPenDataset(root_dir=root_dir, df=train_df.iloc[:100,:],  processor=processor)
+eval_dataset_b1 = AphaPenDataset(root_dir=root_dir, df=test_df.iloc[:100,:],  processor=processor)
 
-eval_dataset_b2 = AphaPenDataset(root_dir=root_dir_b2, df=test_df_b2,  processor=processor)
+eval_dataset_b2 = AphaPenDataset(root_dir=root_dir_b2, df=test_df_b2.iloc[:100,:],  processor=processor)
 
 # train_dataset = ConcatDataset([train_dataset_b1, train_dataset_b2])
 eval_dataset = ConcatDataset([eval_dataset_b1, eval_dataset_b2])
@@ -119,6 +120,7 @@ model = get_peft_model(model, lora_config)
 
 # from transformers.trainer_utils import PREFIX_CHECKPOINT_DIR
 for samp in samp_list:
+    os.environ["WANDB_PROJECT"] = "Alphapen-TrOCR"
     train_dataset_b2 = AphaPenDataset(root_dir=root_dir_b2, df=train_df_b2.iloc[:samp,:],  processor=processor)
 
     train_dataset = ConcatDataset([train_dataset_b1, train_dataset_b2])
@@ -179,4 +181,5 @@ for samp in samp_list:
         # callbacks=[SavePeftModelCallback]
     )
 
-    trainer.train()
+    trainer.train()
+    wandb.finish()

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:47866b7d56154c5af2ead2dd9b39c99b09ed7fcd9d7ab3b817d6a685abc94835
+oid sha256:0dee27664eb5dbd1e3cc935d19708874101ea117d0baf6647d382db3446b7c24
 size 5368