v1

app.py

@@ -0,0 +1,108 @@
+import torch
+import gradio as gr
+from config import *
+from PIL import Image
+from utils.utils import *
+from threading import Thread
+import torch.nn.functional as F
+from meteor.load_mmamba import load_mmamba
+from meteor.load_meteor import load_meteor
+from transformers import TextIteratorStreamer
+from torchvision.transforms.functional import pil_to_tensor
+
+# A100 Zero GPU
+import spaces
+
+# loading meteor model
+mmamba = load_mmamba('BK-Lee/Meteor-Mamba').cuda()
+meteor, tok_meteor = load_meteor('BK-Lee/Meteor-MLM', bits=4)
+
+# device
+device = torch.cuda.current_device()
+
+# freeze model
+freeze_model(mmamba)
+freeze_model(meteor)
+
+# previous length
+previous_length = 0
+
+
+def threading_function(inputs, image_token_number, streamer):
+    # Meteor Mamba
+    mmamba_inputs = mmamba.eval_process(inputs=inputs, tokenizer=tok_meteor, device=device, img_token_number=image_token_number)
+    if 'image' in mmamba_inputs.keys():
+        clip_features = meteor.clip_features(mmamba_inputs['image'])
+        mmamba_inputs.update({"image_features": clip_features})
+    mmamba_outputs = mmamba(**mmamba_inputs)
+    
+    # Meteor
+    meteor_inputs = meteor.eval_process(inputs=inputs, data='demo', tokenizer=tok_meteor, device=device, img_token_number=image_token_number)
+    if 'image' in mmamba_inputs.keys():
+        meteor_inputs.update({"image_features": clip_features})
+    meteor_inputs.update({"tor_features": mmamba_outputs.tor_features})
+
+    generation_kwargs = meteor_inputs
+    generation_kwargs.update({'streamer': streamer})
+    generation_kwargs.update({'do_sample': True})
+    generation_kwargs.update({'max_new_tokens': 128})
+    generation_kwargs.update({'top_p': 0.95})
+    generation_kwargs.update({'temperature': 0.9})
+    generation_kwargs.update({'use_cache': True})
+    return meteor.generate(**generation_kwargs)
+
+def add_message(history, message):
+    for x in message["files"]:
+        history.append(((x,), None))
+    if message["text"] is not None:
+        history.append((message["text"], None))
+    return history, gr.MultimodalTextbox(value=None, interactive=False)
+
+@spaces.GPU(duration=120)
+def bot_streaming(message, history):
+
+    # prompt type -> input prompt
+    image_token_number = int((490/14)**2)
+    if len(message['files']) != 0:
+        # Image Load
+        image = F.interpolate(pil_to_tensor(Image.open(message['files'][0]).convert("RGB")).unsqueeze(0), size=(490, 490), mode='bicubic').squeeze(0)
+        inputs = [{'image': image, 'question': message['text']}]
+    else:
+        inputs = [{'question': message['text']}]
+
+    # [4] Meteor Generation
+    with torch.inference_mode():
+        # kwargs
+        streamer = TextIteratorStreamer(tok_meteor, skip_special_tokens=True)
+
+        # Threading generation
+        thread = Thread(target=threading_function, kwargs=dict(inputs=inputs, image_token_number=image_token_number, streamer=streamer))
+        thread.start()
+
+        # generated text
+        generated_text = ""
+        for new_text in streamer:
+            generated_text += new_text
+        generated_text
+
+    # Text decoding
+    response = generated_text.split('assistant\n')[-1].split('[U')[0].strip()
+
+    buffer = ""
+    for character in response:
+        buffer += character
+        yield buffer
+
+demo = gr.ChatInterface(fn=bot_streaming, title="Meteor", 
+                        description="Meteor",
+                        stop_btn="Stop Generation", multimodal=True)
+demo.launch(debug=True)
+
+
+
+
+
+
+
+
+

config.py

@@ -0,0 +1,57 @@
+# OpenAI Key
+OPENAI_KEY = ""
+
+# Dataset root
+DATASET_ROOT=""
+
+# Pre Meteor Dataset
+METEOR_DATASET= "Meteor.json" 
+
+# Various json and parquet files
+SHAREGPT4V_CAPTION = "sharegpt4v_instruct_gpt4-vision_cap100k.json"
+SHAREGPT4V_INSTRUCTION = "sharegpt4v_mix665k_cap23k_coco-ap9k_lcs3k_sam9k_div2k.json"
+MINIGEMINI_INSTRUCTION = "minigemini_instruction.json"
+DOCDOWNSTREAM = 'train.jsonl'
+DOCREASON = 'detailed_explanation.jsonl'
+GLLAVA_ALIGN = "gllava_align.parquet"
+GLLAVA_QA = "gllava_qa.parquet"
+MATHVISION = "mathvision.parquet"
+MATHINSTRUCT = "MathInstruct.json"
+MATHPLUS = "mathplus.parquet"
+
+# Json files for Evaluation
+VQAV2 = "VQAv2/v2_OpenEnded_mscoco_test2015_questions.json"
+GQA = "gqa/testdev_balanced_questions.json"
+SQA = "ScienceQA/problems.json"
+SQA_SPLIT = "ScienceQA/pid_splits.json"
+VIZWIZ = "VizWiz/test.json"
+TEXTVQA = "TextVQA/llava_textvqa_val_v051_ocr.json"
+TEXTVQA_ANNOTATIONS = "TextVQA/TextVQA_0.5.1_val.json"
+POPE_POPULAR = "POPE/coco_pope_popular.json"
+POPE_ADVERSARIAL = "POPE/coco_pope_adversarial.json"
+POPE_RANDOM = "POPE/coco_pope_random.json"
+MME = "MME_Benchmark_release_version/llava_mme.json"
+MME_DIR = "MME_Benchmark_release_version"
+MMBENCH = "MMBench/MMBench_TEST_EN_legacy.tsv"
+MMBENCH_CN = "MMBench/MMBench_TEST_CN_legacy.tsv"
+MMBENCH_DEV = "MMBench/mmbench_dev_20230712.tsv"
+MMBENCH_CN_DEV = "MMBench/mmbench_dev_cn_20231003.tsv"
+QBENCH = "LLVisionQA-QBench/llvisionqa_dev.json"
+QBENCH_CN = "LLVisionQA-QBench/质衡-问答-验证集.json"
+MMVET = "mm-vet/mm-vet.json"
+MMMU = "MMMU/*/validation*"
+MATHVISTA = "MathVista/testmini-00000-of-00001-725687bf7a18d64b.parquet"
+AI2D = "ai2d/ai2d_test.json"
+HALLUSIONBENCH = "HallusionBench/HallusionBench.json"
+CHARTQA = "chartqa/test/test_augmented.json"
+SEED = "SEED-Bench/SEED-Bench.json"
+LLAVA = "llava-bench-in-the-wild/questions.jsonl"
+# BLINK =
+MATHVERSE = "MathVerse/testmini.json"
+MATHVERSE_TEXT_ONLY = "MathVerse/testmini_text_only.json"
+MMSTAR = "MMStar/mmstar.parquet"
+
+# Available evaluation datasets
+EVAL_DATASETS = ["qbench", "sqa", "ai2d", "chartqa", "seed", "pope", "hallusionbench", "mme", \
+                 "mathvista", "mmbench", "mmbench_cn", "mmvet", "llava", "mmstar", "mathverse"]
+

eval/create_evaluator.py

@@ -0,0 +1,529 @@
+import os
+import re
+import json
+import shortuuid
+import numpy as np
+import pandas as pd
+from config import *
+from collections import defaultdict
+from eval.utils import *
+
+class BaseEvaluator:
+    def __init__(self):
+        super(BaseEvaluator, self).__init__()
+
+        # Create evaluation results folder
+        self.save_dir = os.path.join(DATASET_ROOT, "eval_results")
+        if not os.path.exists(self.save_dir):
+            os.makedirs(self.save_dir)
+
+    def reset(self):
+        # Reset results for new dataset evaluation
+        self.gen_answers = []
+        self.inputs = []
+    
+    def process(self, inputs, outputs):
+        # Merge results
+        self.inputs.extend(inputs)
+        self.gen_answers.extend(outputs)
+
+class Evaluator(BaseEvaluator):
+    def __init__(self):
+        """
+        Eval Datasets
+
+        - VQAv2
+        - GQA
+        - SQA-IMG
+        - VizWiz
+        - TextVQA
+        - POPE
+        - MME
+        - MMBench
+        - MMBench-CN
+        - QBench
+        - MM-Vet
+        - MMMU
+        - MathVista
+        - AI2D
+        - HallusionBench
+        - ChartQA
+        - SEED
+        - LLaVA Wild
+        - BLINK
+        - MathVerse
+
+        """
+              
+        super().__init__()
+    
+    def evaluate(self, model, dataset, accel):
+
+        # gathering all gpu to one device
+        self.inputs = accel.gather_for_metrics(self.inputs)
+        self.gen_answers = accel.gather_for_metrics(self.gen_answers)
+        
+        if accel.is_main_process:
+            # check for duplicates
+            self.inputs, self.gen_answers = remove_duplicate(dataset, self.inputs, self.gen_answers)
+
+            # Select evaluation for dataset
+            if dataset == "vqav2":
+                return self.evaluate_vqa(model, accel)
+            elif dataset == "gqa":
+                return self.evaluate_gqa(model, accel)
+            elif dataset == "sqa":
+                return self.evaluate_sqa(model, accel)
+            elif dataset == "vizwiz":
+                return self.evaluate_vizwiz(model, accel)
+            elif dataset == "textvqa":
+                return self.evaluate_textvqa(model, accel)
+            elif dataset == "pope":
+                return self.evaluate_pope(model, accel)
+            elif dataset == "mme":
+                return self.evaluate_mme(model, accel)
+            elif dataset == "mmbench":
+                return self.evaluate_mmbench(model, accel)
+            elif dataset == "mmbench_dev":
+                return self.evaluate_mmbench_dev(model, accel)
+            elif dataset == "mmbench_cn":
+                return self.evaluate_mmbench_cn(model, accel)
+            elif dataset == "mmbench_cn_dev":
+                return self.evaluate_mmbench_cn_dev(model, accel)
+            elif dataset == "qbench":
+                return self.evaluate_qbench(model, accel)
+            elif dataset == "mm-vet":
+                return self.evaluate_mmvet(model, accel)
+            elif dataset == "mmmu":
+                return self.evaluate_mmmu(model, accel)
+            elif dataset == "mathvista":
+                return self.evaluate_mathvista(model, accel)
+            elif dataset == "ai2d":
+                return self.evaluate_ai2d(model, accel)
+            elif dataset == "hallusionbench":
+                return self.evaluate_hallusionbench(model, accel)
+            elif dataset == "chartqa":
+                return self.evaluate_chartqa(model, accel)
+            elif dataset == "seed":
+                return self.evaluate_seed(model, accel)
+            elif dataset == "llava":
+                return self.evaluate_llava(model, accel)
+            elif dataset == "blink":
+                return self.evaluate_blink(model, accel)
+            elif dataset == "mathverse":
+                return self.evaluate_mathverse(model, accel)
+            elif dataset == "mmstar":
+                return self.evaluate_mmstar(model, accel)
+            else:
+                raise ValueError(
+                    f'{dataset} is not an available dataset.')
+        else:
+            return None
+        
+    def evaluate_vqa(self, model, accel):
+        # VQAv2 Evaluation for EvalAI server
+        pred_answers = [{'question_id': inputs['id'], 'answer': answer} for inputs, answer in zip(self.inputs, self.gen_answers)]
+        pred_pth = os.path.join(self.save_dir, f"{model}_vqav2_results.json")
+        json.dump(pred_answers, open(pred_pth, "w"))
+        accel.print(f"Finished evaluating VQAv2. Evaluate the result file saved to {pred_pth} on EvalAI server.")
+        return 
+    
+    def evaluate_gqa(self, model, accel):
+        # GQA Evaluation
+        pred_answers = {inputs['id']: answer for inputs, answer in zip(self.inputs, self.gen_answers)}
+        # pred_answers = [{'question_id': inputs['id'], 'answer': answer} for inputs, answer in zip(self.inputs, self.gen_answers)]
+        pred_pth = os.path.join(self.save_dir, f"{model}_gqa_results.json")
+        json.dump(pred_answers, open(pred_pth, "w"))
+        accel.print("GQA Results:")
+        results = eval_gqa(pred_answers, json.load(open(os.path.join(DATASET_ROOT, GQA))))
+        return results['accuracy']
+    
+    def evaluate_sqa(self, model, accel):
+        # SQA Evaluation
+        pred_answers = [{'question_id': inputs['id'], 'answer': convert_to_choice(answer, inputs['candidates']), 'gt': inputs['gt']} for inputs, answer in zip(self.inputs, self.gen_answers)]
+        pred_pth = os.path.join(self.save_dir, f"{model}_sqa_results.json")
+        json.dump(pred_answers, open(pred_pth, "w"))
+
+        # Compute accuracy
+        results = [(answer['answer'] == answer['gt']) for answer in pred_answers]
+        accel.print (f"SQA Accuracy: {np.mean(results)*100} %")
+        return np.mean(results)*100
+    
+    def evaluate_vizwiz(self, model, accel):
+        # VizWiz Evaluation
+        evaluator = EvalAIAnswerProcessor()
+        pred_answers = [{'image': inputs['id'], 'answer': evaluator(answer)} for inputs, answer in zip(self.inputs, self.gen_answers)]
+        pred_pth = os.path.join(self.save_dir, f"{model}_vizwiz_results.json")
+        json.dump(pred_answers, open(pred_pth, "w"))
+        accel.print(f"Finished evaluating VizWiz. Evaluate the result file saved to {pred_pth} on EvalAI server.")
+        return 
+    
+    def evaluate_textvqa(self, model, accel):
+        # TextVQA Evaluation
+        pred_answers = [{'question_id': inputs['id'], 'pred_answer': answer, 'question': inputs['question'], 'gt_answers': inputs['gt']} for inputs, answer in zip(self.inputs, self.gen_answers)]
+        pred_pth = os.path.join(self.save_dir, f"{model}_textvqa_results.json")
+        json.dump(pred_answers, open(pred_pth, "w"))
+            
+        evaluator = TextVQAAccuracyEvaluator()
+        results = evaluator.eval_pred_list(pred_answers)*100
+        accel.print (f"TextVQA Accuracy: {results} %")
+        return results
+    
+    def evaluate_pope(self, model, accel):
+        # POPE Evaluation
+        pred_answers = [{'question_id': inputs['id'], 'answer': answer, 'question': inputs['question'], 'category': inputs['category']} for inputs, answer in zip(self.inputs, self.gen_answers)]
+        pred_pth = os.path.join(self.save_dir, f"{model}_pope_results.json")
+        json.dump(pred_answers, open(pred_pth, "w"))
+
+        pope_results = {}
+        pope_results['adversarial'] = None
+        pope_results['popular'] = None
+        pope_results['random'] = None
+
+        categories = ['adversarial', 'popular', 'random']
+        files = [POPE_ADVERSARIAL, POPE_POPULAR, POPE_RANDOM]
+
+        for category, file in zip(categories, files):
+            cur_answers = [x for x in pred_answers if x['category'] == category]
+            cur_answers = sorted(cur_answers, key=lambda x:x["question_id"])
+            pope_results[category] = eval_pope(cur_answers, os.path.join(DATASET_ROOT, file))
+        accel.print (f"POPE Adversarial Accuracy: {pope_results['adversarial']} %")
+        accel.print (f"POPE Popular Accuracy: {pope_results['popular']} %")
+        accel.print (f"POPE Random Accuracy: {pope_results['random']} %")
+        return pope_results
+    
+    def evaluate_mme(self, model, accel):
+        # MME Evaluation
+        pred_answers = [{'question_id': inputs['id'], 'answer': answer,  "question": inputs['question'], 'category': inputs['category']} for inputs, answer in zip(self.inputs, self.gen_answers)]
+        pred_pth = os.path.join(self.save_dir, f"{model}_mme_results.json")
+        json.dump(pred_answers, open(pred_pth, "w"))
+
+        ground_truth = get_gt(data_path=os.path.join(DATASET_ROOT, MME_DIR))
+        result_dir = os.path.join(self.save_dir, 'mme')
+        os.makedirs(result_dir, exist_ok=True)
+        results = defaultdict(list)
+
+        for answer in pred_answers:
+            file = answer['question_id'].split('/')[-1].split('.')[0] + '.txt'
+            results[answer['category']].append((file, answer['question'], answer['answer']))
+
+        for category, cate_tups in results.items():
+            with open(os.path.join(result_dir, f'{category}.txt'), 'w') as fp:
+                questions = set() # check for duplicates
+                for file, prompt, answer in cate_tups:
+                    if 'Answer the question using a single word or phrase.' in prompt:
+                        prompt = prompt.replace('Answer the question using a single word or phrase.', '').strip()
+                    if 'Please answer yes or no.' not in prompt:
+                        prompt = prompt + ' Please answer yes or no.'
+                        if (category, file, prompt) not in ground_truth:
+                            prompt = prompt.replace(' Please answer yes or no.', '  Please answer yes or no.')
+                    gt_ans = ground_truth[category, file, prompt]
+                    dup = file, prompt, gt_ans
+                    tup = file, prompt, gt_ans, answer
+                    if dup in questions:
+                        continue
+                    questions.add(dup)
+                    fp.write('\t'.join(tup) + '\n')
+
+        evaluator = MMEEvaluator()
+        scores = evaluator.process_result(result_dir)
+        accel.print("MME Scores:")
+        accel.print(scores)
+        for eval_type, eval_scores in scores.items():
+            accel.print("===========", eval_type, "===========")
+            accel.print("total score:", eval_scores['total'], "\n")
+            for task_name, score in eval_scores.items():
+                accel.print("\t", task_name, " score:", score)
+            accel.print("\n")
+        return scores
+    
+    def evaluate_mmbench(self, model, accel):
+        # MMBench Evaluation
+        df = pd.read_table(os.path.join(DATASET_ROOT, MMBENCH))
+        cur_df = df.copy()
+        cur_df = cur_df.drop(columns=['hint', 'category', 'source', 'image', 'comment', 'l2-category'])
+        cur_df.insert(6, 'prediction', None)
+        for inputs, answer in zip(self.inputs, self.gen_answers):
+            cur_df.loc[df['index'] == inputs['id'], 'prediction'] = answer
+        pred_pth = os.path.join(self.save_dir, f"{model}_mmbench_results.xlsx")
+        cur_df.to_excel(pred_pth, index=False, engine='openpyxl')
+        accel.print(f"Finished evaluating MMBench. Change {pred_pth} name to submission.xlsx and evaluate the result file saved to {pred_pth} on OpenCompass server.")
+        return 
+    
+    def evaluate_mmbench_dev(self, model, accel):
+        # MMBench Dev Evaluation
+        df = pd.read_table(os.path.join(DATASET_ROOT, MMBENCH_DEV))
+        cur_df = df.copy()
+        cur_df = cur_df.drop(columns=['hint', 'category', 'source', 'image', 'comment', 'l2-category'])
+        cur_df.insert(6, 'prediction', None)
+        for inputs, answer in zip(self.inputs, self.gen_answers):
+            cur_df.loc[df['index'] == inputs['id'], 'prediction'] = answer[0]
+        pred_pth = os.path.join(self.save_dir, f"{model}_mmbench_dev_results.xlsx")
+        cur_df.to_excel(pred_pth, index=False, engine='openpyxl')
+        accuracy = (cur_df['prediction'] == cur_df['answer']).mean()
+        accel.print(f'MMBench_dev Accuracy: {accuracy:.2%}')
+        return 
+    
+    def evaluate_mmbench_cn(self, model, accel):
+        # MMBench_CN Evaluation
+        df = pd.read_table(os.path.join(DATASET_ROOT, MMBENCH_CN))
+        cur_df = df.copy()
+        cur_df = cur_df.drop(columns=['hint', 'category', 'source', 'image', 'comment', 'l2-category'])
+        cur_df.insert(6, 'prediction', None)
+        for inputs, answer in zip(self.inputs, self.gen_answers):
+            cur_df.loc[df['index'] == inputs['id'], 'prediction'] = answer
+        pred_pth = os.path.join(self.save_dir, f"{model}_mmbench_cn_results.xlsx")
+        cur_df.to_excel(pred_pth, index=False, engine='openpyxl')
+        accel.print(f"Finished evaluating MMBench_CN. Change {pred_pth} name to submission.xlsx and evaluate the result file saved to {pred_pth} on OpenCompass server.")
+        return 
+    
+    def evaluate_mmbench_cn_dev(self, model, accel):
+        # MMBench_CN Dev Evaluation
+        df = pd.read_table(os.path.join(DATASET_ROOT, MMBENCH_CN_DEV))
+        cur_df = df.copy()
+        cur_df = cur_df.drop(columns=['hint', 'category', 'source', 'image', 'comment', 'l2-category'])
+        cur_df.insert(6, 'prediction', None)
+        for inputs, answer in zip(self.inputs, self.gen_answers):
+            cur_df.loc[df['index'] == inputs['id'], 'prediction'] = answer[0]
+        pred_pth = os.path.join(self.save_dir, f"{model}_mmbench_cn_dev_results.xlsx")
+        cur_df.to_excel(pred_pth, index=False, engine='openpyxl')
+        accuracy = (cur_df['prediction'] == cur_df['answer']).mean()
+        accel.print(f'MMBench_CN_dev Accuracy: {accuracy:.2%}')
+        return 
+    
+    def evaluate_qbench(self, model, accel):
+        # QBench Evaluation
+        pred_answers = [{'id': inputs['id'], 'answer': convert_to_choice(answer, inputs['candidates']), 'gt': inputs['gt'], 'candidates': inputs['candidates']} for inputs, answer in zip(self.inputs, self.gen_answers)]
+        pred_pth = os.path.join(self.save_dir, f'{model}_qbench_results.jsonl')
+        with open(pred_pth, "w") as pf:
+            pf.write(json.dumps(pred_answers) + "\n")
+
+        results = [(pred['candidates'][pred['answer']] == pred['gt']) for pred in pred_answers]
+        accel.print (f"QBench Accuracy: {np.mean(results)*100} %")
+        return np.mean(results)*100
+    
+    def evaluate_mmvet(self, model, accel):
+        # MM-Vet Evaluation
+        cur_result = {f"{inputs['id']}": answer for inputs, answer in zip(self.inputs, self.gen_answers)}
+        pred_pth = os.path.join(self.save_dir, f'{model}_mmvet_results.json')
+        with open(pred_pth, 'w') as f:
+            json.dump(cur_result, f, indent=2)
+        
+        accel.print(f"Finished evaluating MM-Vet. Evaluate the result file saved to {pred_pth}.")
+        return 
+    
+    def evaluate_mmmu(self, model, accel):
+        # MMMU Evaluation
+        predictions = {inputs['id']: answer for inputs, answer in zip(self.inputs, self.gen_answers)}
+        answers = {inputs['id']: {'ground_truth': inputs['gt'], 'question_type': inputs['question_type']} for inputs, answer in zip(self.inputs, self.gen_answers)}
+        pred_pth = os.path.join(self.save_dir, f'{model}_mmmu_results.json')
+        with open(pred_pth, "w") as f:
+            json.dump(predictions, f, indent=2)
+        ans_pth = os.path.join(self.save_dir, 'mmmu_answers.json')
+        with open(ans_pth, "w") as pf:
+            json.dump(answers, pf, indent=2)
+
+        # group by category
+        output_dict_w_cat = {}
+        for data_id, parsed_pred in predictions.items():
+            category = "_".join(data_id.split("_")[1:-1])
+            if category not in output_dict_w_cat:
+                output_dict_w_cat.update({category: {}})
+            output_dict_w_cat[category].update({data_id: parsed_pred})
+
+        # group by category
+        answer_dict_w_cat = {}
+        for data_id, parsed_pred in answers.items():
+            category = "_".join(data_id.split("_")[1:-1])
+            if category not in answer_dict_w_cat:
+                answer_dict_w_cat.update({category: {}})
+            answer_dict_w_cat[category].update({data_id: parsed_pred})
+
+        evaluation_result = {}
+
+        for category in CAT_SHORT2LONG.values():
+            accel.print("Evaluating: {}".format(category))
+            # get cat_outputs and cat_answers
+            try:
+                cat_outputs = output_dict_w_cat[category]
+                cat_answers = answer_dict_w_cat[category]
+            except KeyError:
+                accel.print("Skipping {} for not found".format(category))
+                continue
+
+            exampels_to_eval = []
+            for data_id, parsed_pred in cat_outputs.items():
+                question_type = cat_answers[data_id]['question_type']
+                if question_type != 'multiple-choice':
+                    parsed_pred = parse_open_response(parsed_pred) # mainly for type consistency (make it number, etc.)
+                else:
+                    parsed_pred = parsed_pred
+
+                exampels_to_eval.append({
+                    "id": data_id,
+                    "question_type": question_type,
+                    "answer": cat_answers[data_id]['ground_truth'],
+                    "parsed_pred": parsed_pred
+                })
+
+            judge_dict, metric_dict = evaluate(exampels_to_eval)
+            metric_dict.update({"num_example": len(exampels_to_eval)})
+
+            evaluation_result[category] = metric_dict
+
+        printable_results = {}
+        # add domain Subject
+        for domain, in_domain_cats in DOMAIN_CAT2SUB_CAT.items():
+            in_domain_cat_results = {}
+            for cat_name in in_domain_cats: # use the order in DOMAIN_CAT2SUB_CAT
+                if cat_name in evaluation_result.keys():
+                    in_domain_cat_results[cat_name] = evaluation_result[cat_name]
+                else:
+                    pass
+            in_domain_ins_acc = calculate_ins_level_acc(in_domain_cat_results)
+            in_domain_data_num = sum([cat_results['num_example'] for cat_results in in_domain_cat_results.values()])
+            printable_results['Overall-' + domain] = {"num": int(in_domain_data_num),
+                                                    "acc": round(in_domain_ins_acc, 3)
+                                                    }
+            # add sub category
+            for cat_name, cat_results in in_domain_cat_results.items():
+                printable_results[cat_name] = {"num": int(cat_results['num_example']),
+                                            "acc": round(cat_results['acc'], 3)
+                                            }
+            
+        # table.append(["-----------------------------", "-----", "----"])
+        all_ins_acc = calculate_ins_level_acc(evaluation_result)
+        printable_results['Overall'] = {"num": sum([cat_results['num_example'] for cat_results in evaluation_result.values()]),
+                                        "acc": round(all_ins_acc, 3)
+                                        }
+
+        accel.print(printable_results)
+        return 
+    
+    def evaluate_mathvista(self, model, accel):
+        # MathVista Evaluation
+        pred_answers = [{'pid': inputs['id'], 'image': inputs['id'], 'response': answer, 
+                         'question_type': inputs['question_type'], 'answer_type': inputs['answer_type'], 'metadata': inputs['metadata'], 
+                         'choices': inputs['choices'], 'query': inputs['question'], 'precision': inputs['precision'],} for inputs, answer in zip(self.inputs, self.gen_answers)]
+        predictions = {pred['pid']: pred for pred in pred_answers}
+        pred_pth = os.path.join(self.save_dir, f"{model}_mathvista_results.json")
+        json.dump(predictions, open(pred_pth, "w"))
+        
+        accel.print(f"Finished evaluating MathVista. Evaluate the result file saved to {pred_pth}.")
+        return 
+    
+    def evaluate_ai2d(self, model, accel):
+        # AI2D Evaluation
+        pred_answers = [{'question_id': inputs['id'], 'answer': answer, 'gt': inputs['gt']} for inputs, answer in zip(self.inputs, self.gen_answers)]
+        pred_pth = os.path.join(self.save_dir, f"{model}_ai2d_results.json")
+        json.dump(pred_answers, open(pred_pth, "w"))
+
+        # Compute accuracy
+        pattern = re.compile(r'[A-Z]')
+        results = [(char_to_int(pattern.findall(answer)[0]) == inputs['gt']) for inputs, answer in zip(self.inputs, self.gen_answers)]
+
+        accel.print(f"AI2D Accuracy: {np.mean(results)*100} %")
+        return np.mean(results)*100
+    
+    def evaluate_hallusionbench(self, model, accel):
+        # HallusionBench Evaluation
+        pred_answers = [{'answer': '1' if answer.lower().find('yes') != -1 else '0', 'question': inputs['question'], 'gt': inputs['gt']} for inputs, answer in zip(self.inputs, self.gen_answers)]
+        pred_pth = os.path.join(self.save_dir, f"{model}_hallusionbench_results.json")
+        json.dump(pred_answers, open(pred_pth, "w"))
+
+        # Compute accuracy
+        results = [(answer['answer'] == answer['gt']) for answer in pred_answers]
+        accel.print(f"HallusionBench Accuracy: {np.mean(results)*100} %")
+        return np.mean(results)*100
+    
+    def evaluate_chartqa(self, model, accel):
+        # ChartQA Evaluation
+        # post processing
+        processed_answers = []
+        for x in self.gen_answers:
+            if any(i.isdigit() for i in x):
+                processed_answers.append(x.split(" ")[0])
+            else:
+                processed_answers.append(x)
+        pred_answers = [{'answer': answer, 'question': inputs['question'], 'annotation': inputs['gt']} for inputs, answer in zip(self.inputs, processed_answers)]
+        pred_pth = os.path.join(self.save_dir, f"{model}_chartqa_results.json")
+        json.dump(pred_answers, open(pred_pth, "w"))
+
+        # Compute accuracy
+        acc = evaluate_relaxed_accuracy(pred_answers)
+        accel.print(f"ChartQA Accuracy: {acc*100}%")
+        return acc
+    
+    def evaluate_seed(self, model, accel):
+        # SEED Evaluation
+        pred_answers = [{'answer': answer, 'question': inputs['question'], 'question_id': inputs['id'], 'gt': inputs['gt'], 'question_type': inputs['question_type']} for inputs, answer in zip(self.inputs, self.gen_answers)]
+        pred_pth = os.path.join(self.save_dir, f"{model}_seed_results.json")
+        json.dump(pred_answers, open(pred_pth, "w"))
+
+        # Compute accuracy
+        results = [(answer['answer'] == answer['gt']) for answer in pred_answers]
+        accel.print (f"SEED Accuracy: {np.mean(results)*100} %")
+
+        # Per question type accuracy
+        for k, v in SEED_TYPES.items():
+            sub_results = []
+            for pred in pred_answers:
+                if pred['question_type'] == k:
+                    sub_results.append(pred['answer'] == pred['gt'])
+            accel.print (f"{v}: {np.mean(sub_results)*100} %")
+        
+        return np.mean(results)*100
+    
+    def evaluate_llava(self, model, accel):
+        # LLaVA-in-the-Wild Evaluation
+        pred_answers = [{'question_id': inputs['id'],  'prompt': inputs['question'], 'text': answer, "answer_id": shortuuid.uuid()} for inputs, answer in zip(self.inputs, self.gen_answers)]
+        sorted_answers = sorted(pred_answers, key=lambda x: x['question_id'])
+        pred_pth = os.path.join(self.save_dir, f'{model}_llava_results.jsonl')
+        ans_file = open(pred_pth, "w")
+        for pred in sorted_answers:
+            ans_file.write(json.dumps(pred) + "\n")
+            ans_file.flush()
+        ans_file.close()
+
+        accel.print(f"Finished evaluating LLaVA-in-the-wild. Evaluate the result file saved to {pred_pth}.")
+        return 
+    
+    def evaluate_blink(self, model, accel):
+        # BLINK Evaluation
+        # TODO
+        return 
+    
+    def evaluate_mathverse(self, model, accel):
+        # Mathverse Evaluation
+        pred_answers = [{'sample_index' : inputs['id'], 'problem_index' : inputs['problem_index'], 'problem_version' : inputs['problem_version'],
+                        'question' : inputs['origin_question'], 'answer' : inputs['gt'],
+                        'question_type': inputs['question_type'], 'question_type': inputs['question_type'], 
+                        'metadata': inputs['metadata'], 'query_wo': inputs['question'], 'query_cot' : inputs['query_cot'], 'model_answer' : answer} for inputs, answer in zip(self.inputs, self.gen_answers)]
+
+        # answers = [item for item in pred_answers if item['problem_version'] != 'Text_Only']
+        # text_only_answers = [item for item in pred_answers if item['problem_version'] == 'Text_Only']
+        
+        pred_pth = os.path.join(self.save_dir, f'{model}_mathverse_results.json')
+        json.dump(pred_answers, open(pred_pth, "w"))
+        pred_pth = os.path.join(self.save_dir, f'{model}_mathverse_scores.json')
+        eval_mathverse(self.save_dir, pred_answers,f'{model}_mathverse_extracts.json', f'{model}_mathverse_scores.json')
+        accel.print(f"Finished evaluating MathVerse. Evaluate the result file saved to {pred_pth}.")
+        # TODO
+        return 
+
+    def evaluate_mmstar(self, model, accel):
+        pred_answers = [{'question': inputs['question'],
+                        'answer': inputs['answer'],
+                        'category': inputs['category'],
+                        'l2_category': inputs['l2_category'],
+                        # 'bench': inputs['bench'],
+                        'prediction' : answer} for inputs, answer in zip(self.inputs, self.gen_answers)]
+        
+        pred_pth = os.path.join(self.save_dir, f'{model}_mmstar_results.json')
+        json.dump(pred_answers, open(pred_pth, "w"))
+        
+        df = pd.DataFrame(pred_answers)
+                        
+        eval_mmstar(df, self.save_dir, f'{model}_mmstar_scores.json')
+        pred_pth = os.path.join(self.save_dir, f'{model}_mmstar_scores.json')
+        accel.print(f"Finished evaluating MMStar. Evaluate the result file saved to {pred_pth}.")

eval/llavabench/eval.sh

@@ -0,0 +1,11 @@
+python eval/llavabench/eval_gpt_review_bench.py \
+    --question /mnt/ssd/lbk-cvpr/dataset/llava-bench-in-the-wild/questions.jsonl \
+    --context /mnt/ssd/lbk-cvpr/dataset/llava-bench-in-the-wild/context.jsonl \
+    --rule eval/llavabench/rule.json \
+    --answer-list \
+        /mnt/ssd/lbk-cvpr/dataset/llava-bench-in-the-wild/answers_gpt4.jsonl \
+        /mnt/ssd/lbk-cvpr/dataset/eval_results/Meteor_llava_results.jsonl \
+    --output \
+        /mnt/ssd/lbk-cvpr/dataset/eval_results/reviews_meteor_llava_results_step3.jsonl
+
+python eval/llavabench/summarize_gpt_review.py -f /mnt/ssd/lbk-cvpr/dataset/eval_results/reviews_meteor_llava_results_step3.jsonl

eval/llavabench/eval_gpt_review_bench.py

@@ -0,0 +1,122 @@
+import argparse
+import json
+import os
+
+import openai
+import time
+
+NUM_SECONDS_TO_SLEEP = 0.5
+
+openai.api_key= ""
+
+def get_eval(content: str, max_tokens: int):
+    while True:
+        try:
+            response = openai.ChatCompletion.create(
+                model='gpt-4-0613',
+                messages=[{
+                    'role': 'system',
+                    'content': 'You are a helpful and precise assistant for checking the quality of the answer.'
+                }, {
+                    'role': 'user',
+                    'content': content,
+                }],
+                temperature=0.2,  # TODO: figure out which temperature is best for evaluation
+                max_tokens=max_tokens,
+            )
+            break
+        except openai.error.RateLimitError:
+            pass
+        except Exception as e:
+            print(e)
+        time.sleep(NUM_SECONDS_TO_SLEEP)
+
+    return response['choices'][0]['message']['content']
+
+
+def parse_score(review):
+    try:
+        score_pair = review.split('\n')[0]
+        score_pair = score_pair.replace(',', ' ')
+        sp = score_pair.split(' ')
+        if len(sp) == 2:
+            return [float(sp[0]), float(sp[1])]
+        else:
+            print('error', review)
+            return [-1, -1]
+    except Exception as e:
+        print(e)
+        print('error', review)
+        return [-1, -1]
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='ChatGPT-based QA evaluation.')
+    parser.add_argument('-q', '--question')
+    parser.add_argument('-c', '--context')
+    parser.add_argument('-a', '--answer-list', nargs='+', default=[])
+    parser.add_argument('-r', '--rule')
+    parser.add_argument('-o', '--output')
+    parser.add_argument('--max-tokens', type=int, default=1024, help='maximum number of tokens produced in the output')
+    args = parser.parse_args()
+
+    f_q = open(os.path.expanduser(args.question))
+    f_ans1 = open(os.path.expanduser(args.answer_list[0]))
+    f_ans2 = open(os.path.expanduser(args.answer_list[1]))
+    rule_dict = json.load(open(os.path.expanduser(args.rule), 'r'))
+
+    if os.path.isfile(os.path.expanduser(args.output)):
+        cur_reviews = [json.loads(line) for line in open(os.path.expanduser(args.output))]
+    else:
+        cur_reviews = []
+
+    review_file = open(f'{args.output}', 'a')
+
+    context_list = [json.loads(line) for line in open(os.path.expanduser(args.context))]
+    image_to_context = {context['image']: context for context in context_list}
+
+    handles = []
+    idx = 0
+    for ques_js, ans1_js, ans2_js in zip(f_q, f_ans1, f_ans2):
+        ques = json.loads(ques_js)
+        ans1 = json.loads(ans1_js)
+        ans2 = json.loads(ans2_js)
+
+        inst = image_to_context[ques['image']]
+
+        if isinstance(inst['caption'], list):
+            cap_str = '\n'.join(inst['caption'])
+        else:
+            cap_str = inst['caption']
+
+        category = 'llava_bench_' + json.loads(ques_js)['category']
+        if category in rule_dict:
+            rule = rule_dict[category]
+        else:
+            assert False, f"Visual QA category not found in rule file: {category}."
+        prompt = rule['prompt']
+        role = rule['role']
+        content = (f'[Context]\n{cap_str}\n\n'
+                   f'[Question]\n{ques["text"]}\n\n'
+                   f'[{role} 1]\n{ans1["text"]}\n\n[End of {role} 1]\n\n'
+                   f'[{role} 2]\n{ans2["text"]}\n\n[End of {role} 2]\n\n'
+                   f'[System]\n{prompt}\n\n')
+        cur_js = {
+            'id': idx+1,
+            'question_id': ques['question_id'],
+            'answer1_id': ans1.get('answer_id', ans1['question_id']),
+            'answer2_id': ans2.get('answer_id', ans2['answer_id']),
+            'category': category
+        }
+        if idx >= len(cur_reviews):
+            review = get_eval(content, args.max_tokens)
+            scores = parse_score(review)
+            cur_js['content'] = review
+            cur_js['tuple'] = scores
+            review_file.write(json.dumps(cur_js) + '\n')
+            review_file.flush()
+        else:
+            print(f'Skipping {idx} as we already have it.')
+        idx += 1
+        print(idx)
+    review_file.close()

eval/llavabench/rule.json

@@ -0,0 +1,11 @@
+{
+    "coding": {"role": "Assistant", "prompt": "Your task is to evaluate the coding abilities of the above two assistants. They have been asked to implement a program to solve a given problem. Please review their code submissions, paying close attention to their problem-solving approach, code structure, readability, and the inclusion of helpful comments.\n\nPlease ensure that the assistants' submissions:\n\n1. Correctly implement the given problem statement.\n2. Contain accurate and efficient code.\n3. Include clear and concise comments that explain the code's logic and functionality.\n4. Adhere to proper coding standards and best practices.\n\nOnce you have carefully reviewed both submissions, provide detailed feedback on their strengths and weaknesses, along with any suggestions for improvement. You should first output a single line containing two scores on the scale of 1-10 (1: no code/no sense; 10: perfect) for Assistant 1 and 2, respectively. Then give extra comments starting from the next line."},
+    "math":  {"role": "Assistant", "prompt": "We would like to request your feedback on the mathematical proficiency of two AI assistants regarding the given user question.\nFirstly, please solve the problem independently, without referring to the answers provided by Assistant 1 and Assistant 2.\nAfterward, please examine the problem-solving process of Assistant 1 and Assistant 2 step-by-step to ensure their correctness, identifying any incorrect steps if present. Your evaluation should take into account not only the answer but also the problem-solving steps.\nFinally, please output a Python tuple containing two numerical scores for Assistant 1 and Assistant 2, ranging from 1 to 10, respectively. If applicable, explain the reasons for any variations in their scores and determine which assistant performed better."},
+    "default":  {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above.\nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."},
+    "conv":  {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with five descriptive sentences describing the same image and the bounding box coordinates of each object in the scene. These coordinates are in the form of bounding boxes, represented as (x1, y1, x2, y2) with floating numbers ranging from 0 to 1. These values correspond to the top left x, top left y, bottom right x, and bottom right y. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."},
+    "detail":  {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with five descriptive sentences describing the same image and the bounding box coordinates of each object in the scene. These coordinates are in the form of bounding boxes, represented as (x1, y1, x2, y2) with floating numbers ranging from 0 to 1. These values correspond to the top left x, top left y, bottom right x, and bottom right y. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."},
+    "complex":  {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with five descriptive sentences describing the same image and the bounding box coordinates of each object in the scene. These coordinates are in the form of bounding boxes, represented as (x1, y1, x2, y2) with floating numbers ranging from 0 to 1. These values correspond to the top left x, top left y, bottom right x, and bottom right y. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."},
+    "llava_bench_conv":  {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with a few sentences describing the image. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."},
+    "llava_bench_detail":  {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with a few sentences describing the image. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."},
+    "llava_bench_complex":  {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with a few sentences describing the image. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."}
+}

eval/llavabench/summarize_gpt_review.py

@@ -0,0 +1,60 @@
+import json
+import os
+from collections import defaultdict
+
+import numpy as np
+
+import argparse
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='ChatGPT-based QA evaluation.')
+    parser.add_argument('-d', '--dir', default=None)
+    parser.add_argument('-v', '--version', default=None)
+    parser.add_argument('-s', '--select', nargs='*', default=None)
+    parser.add_argument('-f', '--files', nargs='*', default=[])
+    parser.add_argument('-i', '--ignore', nargs='*', default=[])
+    return parser.parse_args()
+
+
+if __name__ == '__main__':
+    args = parse_args()
+
+    if args.ignore is not None:
+        args.ignore = [int(x) for x in args.ignore]
+
+    if len(args.files) > 0:
+        review_files = args.files
+    else:
+        review_files = [x for x in os.listdir(args.dir) if x.endswith('.jsonl') and (x.startswith('gpt4_text') or x.startswith('reviews_') or x.startswith('review_') or 'review' in args.dir)]
+
+    for review_file in sorted(review_files):
+        config = os.path.basename(review_file).replace('gpt4_text_', '').replace('.jsonl', '')
+        if args.select is not None and any(x not in config for x in args.select):
+            continue
+        if '0613' in config:
+            version = '0613'
+        else:
+            version = '0314'
+        if args.version is not None and args.version != version:
+            continue
+        scores = defaultdict(list)
+        print(config)
+        with open(os.path.join(args.dir, review_file) if args.dir is not None else review_file) as f:
+            for review_str in f:
+                review = json.loads(review_str)
+                if review['question_id'] in args.ignore:
+                    continue
+                if 'category' in review:
+                    scores[review['category']].append(review['tuple'])
+                    scores['all'].append(review['tuple'])
+                else:
+                    if 'tuple' in review:
+                        scores['all'].append(review['tuple'])
+                    else:
+                        scores['all'].append(review['score'])
+        for k, v in sorted(scores.items()):
+            stats = np.asarray(v).mean(0).tolist()
+            stats = [round(x, 3) for x in stats]
+            # print(k, stats, round(stats[1]/stats[0]*100, 1))
+            print(k, round(stats[1]/stats[0]*100, 1), round(stats[0] * 10, 1), round(stats[1] * 10, 1))
+        print('=================================')

eval/mathvista/calculate_score.py

@@ -0,0 +1,259 @@
+import os
+import re
+import argparse
+import pandas as pd
+
+# !pip install python-Levenshtein
+from Levenshtein import distance
+
+import sys
+sys.path.append('../')
+from utilities import *
+
+
+def get_most_similar(prediction, choices):
+    """
+    Use the Levenshtein distance (or edit distance) to determine which of the choices is most similar to the given prediction
+    """
+    distances = [distance(prediction, choice) for choice in choices]
+    ind = distances.index(min(distances))
+    return choices[ind]
+    # return min(choices, key=lambda choice: distance(prediction, choice))
+
+
+def normalize_extracted_answer(extraction, choices, question_type, answer_type, precision):
+    """
+    Normalize the extracted answer to match the answer type
+    """
+    if question_type == 'multi_choice':
+        # make sure the extraction is a string
+        if isinstance(extraction, str):
+            extraction = extraction.strip()
+        else:
+            try:
+                extraction = str(extraction)
+            except:
+                extraction = ""
+    
+        # extract "A" from "(A) text"
+        letter = re.findall(r'\(([a-zA-Z])\)', extraction)
+        if len(letter) > 0:
+            extraction = letter[0].upper()
+        
+        options = [chr(ord('A') + i) for i in range(len(choices))]
+            
+        if extraction in options:
+            # convert option letter to text, e.g. "A" -> "text"
+            ind = options.index(extraction)
+            extraction = choices[ind]
+        else:
+            # select the most similar option
+            extraction = get_most_similar(extraction, choices)
+        assert extraction in choices
+
+    elif answer_type == 'integer':
+        try:
+            extraction = str(int(float(extraction)))
+        except:
+            extraction = None
+
+    elif answer_type == 'float':
+        try:
+            extraction = str(round(float(extraction), precision))
+        except:
+            extraction = None
+        
+    elif answer_type == 'list':
+        try:
+            extraction = str(extraction)
+        except:
+            extraction = None
+
+    return extraction
+    
+
+def safe_equal(prediction, answer):
+    """
+    Check if the prediction is equal to the answer, even if they are of different types
+    """
+    try:
+        if prediction == answer:
+            return True
+        return False
+    except Exception as e:
+        print(e)
+        return False
+
+
+def get_acc_with_contion(res_pd, key, value):
+    if key == 'skills':
+        # if value in res_pd[key]:
+        total_pd = res_pd[res_pd[key].apply(lambda x: value in x)]
+    else:
+        total_pd = res_pd[res_pd[key] == value]
+
+    correct_pd = total_pd[total_pd['true_false'] == True]
+    acc = "{:.2f}".format(len(correct_pd) / len(total_pd) * 100)
+    return len(correct_pd), len(total_pd), acc
+        
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--output_dir', type=str, default='../results')
+    parser.add_argument('--output_file', type=str, default='output.json')
+    parser.add_argument('--score_file', type=str, default='scores.json')
+    parser.add_argument('--gt_file', type=str, default='../data/testmini.json', help='ground truth file')
+    parser.add_argument('--number', type=int, default=-1, help='number of problems to run')
+    parser.add_argument('--rerun', action='store_true', help='rerun the evaluation')
+    parser.add_argument('--caculate_gain', action='store_true', help='caculate the socre gains over random guess')
+    parser.add_argument('--random_file', type=str, default='score_random_guess.json')  
+    args = parser.parse_args()
+
+    # args
+    output_file = os.path.join(args.output_dir, args.output_file)
+
+    # # quick test
+    # output_file = '../results/llava-llama-2-13b/output_llava_llama_2_13b.json'
+
+    # read json
+    print(f"Reading {output_file}...")
+    results = read_json(output_file)
+
+    # read ground truth
+    print(f"Reading {args.gt_file}...")
+    gts = read_json(args.gt_file)
+
+    # full pids
+    full_pids = list(results.keys())
+    if args.number > 0:
+        full_pids = full_pids[:min(args.number, len(full_pids))]
+    print("Number of testing problems:", len(full_pids))
+    
+    ## [1] Evaluate if the prediction is true or false
+    print("\nEvaluating the predictions...")
+    update_json_flag = False
+    for pid in full_pids:
+        problem = results[pid]
+        # print(problem)
+
+        if args.rerun:
+            if 'prediction' in problem:
+                del problem['prediction']
+            if 'true_false' in problem:
+                del problem['true_false']
+
+        choices = problem['choices']
+        question_type = problem['question_type']
+        answer_type = problem['answer_type']
+        precision = problem['precision']
+        extraction = problem['extraction']
+
+        if 'answer' in problem:
+            answer = problem['answer']
+        else:
+            answer = gts[pid]['answer']
+            problem['answer'] = answer
+
+        # normalize the extracted answer to match the answer type
+        prediction = normalize_extracted_answer(extraction, choices, question_type, answer_type, precision)
+
+        # verify the prediction is true or false
+        true_false = safe_equal(prediction, answer)
+        
+        # update the problem
+        if "true_false" not in problem:
+            update_json_flag = True
+
+        elif true_false != problem['true_false']:
+            update_json_flag = True
+
+        if "prediction" not in problem:
+            update_json_flag = True
+
+        elif prediction !=  problem['prediction']:
+            update_json_flag = True
+            
+        problem['prediction'] = prediction
+        problem['true_false'] = true_false
+
+    # save the updated json
+    if update_json_flag:
+        print("\n!!!Some problems are updated.!!!")
+        print(f"\nSaving {output_file}...")
+        save_json(results, output_file)
+
+    ## [2] Calculate the average accuracy
+    total = len(full_pids)
+    correct = 0
+    for pid in full_pids:
+        if results[pid]['true_false']:
+            correct += 1
+    accuracy = str(round(correct / total * 100, 2))
+    print(f"\nCorrect: {correct}, Total: {total}, Accuracy: {accuracy}%")
+
+    scores = {"average": {"accuracy": accuracy, "correct": correct, "total": total}}
+    
+    ## [3] Calculate the fine-grained accuracy scores
+    
+    # merge the 'metadata' attribute into the data
+    for pid in results:
+        results[pid].update(results[pid].pop('metadata'))
+
+    # convert the data to a pandas DataFrame
+    df = pd.DataFrame(results).T
+
+    print(len(df))
+    print("Number of test problems:", len(df))
+    # assert len(df) == 1000 # Important!!!
+
+    # asign the target keys for evaluation
+    target_keys = ['question_type', 'answer_type', 'language', 'source', 'category', 'task', 'context', 'grade', 'skills']
+     
+    for key in target_keys:
+        print(f"\nType: [{key}]")
+        # get the unique values of the key
+        if key == 'skills':
+            # the value is a list
+            values = []
+            for i in range(len(df)):
+                values += df[key][i]
+            values = list(set(values))
+        else:
+            values = df[key].unique()
+        #print(values)
+
+        # calculate the accuracy for each value
+        scores[key] = {}
+        for value in values:
+            correct, total, acc = get_acc_with_contion(df, key, value)
+            if total > 0:
+                print(f"[{value}]: {acc}% ({correct}/{total})")
+                scores[key][value] = {"accuracy": acc, "correct": correct, "total": total}
+        
+        # sort the scores by accuracy
+        scores[key] = dict(sorted(scores[key].items(), key=lambda item: float(item[1]['accuracy']), reverse=True))
+
+    # save the scores
+    scores_file = os.path.join(args.output_dir, args.score_file)
+    print(f"\nSaving {scores_file}...")
+    save_json(scores, scores_file)
+    print("\nDone!")
+
+    # [4] Calculate the score gains over random guess
+    if args.caculate_gain:
+        random_file = os.path.join(args.output_dir, args.random_file)
+        random_scores = json.load(open(random_file))
+
+        print("\nCalculating the score gains...")
+        for key in scores:
+            if key == 'average':
+                gain = round(float(scores[key]['accuracy']) - float(random_scores[key]['accuracy']), 2)
+                scores[key]['acc_gain'] = gain
+            else:
+                for sub_key in scores[key]:
+                    gain = round(float(scores[key][sub_key]['accuracy']) - float(random_scores[key][sub_key]['accuracy']), 2)
+                    scores[key][sub_key]['acc_gain'] = str(gain)
+
+        # save the score gains
+        print(f"\nSaving {scores_file}...")    
+        save_json(scores, scores_file)
+        print("\nDone!")

eval/mathvista/eval.sh

@@ -0,0 +1,9 @@
+# python eval/mathvista/extract_answer.py \
+# --output_dir /mnt/ssd/lbk-cvpr/dataset/eval_results \
+# --output_file Meteor_mathvista_results.json 
+
+python eval/mathvista/calculate_score.py \
+--output_dir /mnt/ssd/lbk-cvpr/dataset/eval_results \
+--output_file Meteor_mathvista_results_refixed.json  \
+--score_file Meteor_mathvista_scores.json \
+--gt_file /mnt/ssd/lbk-cvpr/dataset/MathVista/annot_testmini.json \

eval/mathvista/extract_answer.py

@@ -0,0 +1,150 @@
+import os
+import re
+import time
+import argparse
+
+from tqdm import tqdm
+
+import sys
+sys.path.append('../')
+from utilities import *
+
+# OpenAI
+import openai
+openai.api_key = ""
+# print(openai.api_key)
+
+# load demo prompt
+from prompts.ext_ans import demo_prompt
+
+
+def verify_extraction(extraction):
+    extraction = extraction.strip()
+    if extraction == "" or extraction == None:
+        return False
+    return True
+
+
+def create_test_prompt(demo_prompt, query, response):
+    demo_prompt = demo_prompt.strip()
+    test_prompt = f"{query}\n\n{response}"
+    full_prompt = f"{demo_prompt}\n\n{test_prompt}\n\nExtracted answer: "
+    return full_prompt
+
+
+def extract_answer(response, problem, quick_extract=False):
+    question_type = problem['question_type']
+    answer_type = problem['answer_type']
+    choices = problem['choices']
+    query = problem['query']
+    pid = problem['pid']
+
+    if response == "":
+        return ""
+    
+    if question_type == 'multi_choice' and response in choices:
+        return response
+    
+    if answer_type == "integer":
+        try:
+            extraction = int(response)
+            return str(extraction)
+        except:
+            pass
+
+    if answer_type == "float":
+        try:
+            extraction = str(float(response))
+            return extraction
+        except:
+            pass
+
+    # quick extraction
+    if quick_extract:
+        print("Quickly extracting answer...")
+        # The answer is "text". -> "text"
+        try:
+            result = re.search(r'The answer is "(.*)"\.', response)
+            if result:
+                extraction = result.group(1)
+                return extraction
+        except:
+            pass
+
+    # general extraction
+    try:
+        full_prompt = create_test_prompt(demo_prompt, query, response)
+        extraction = get_chat_response(full_prompt, openai.api_key)
+        return extraction
+    except Exception as e:
+        print(e)
+        print(f"Error in extracting answer for {pid}")
+
+    return ""
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    # input
+    parser.add_argument('--output_dir', type=str, default='../results')
+    parser.add_argument('--output_file', type=str, default='answer.json')
+    parser.add_argument('--response_label', type=str, default='response', help='response label for the input file')
+    # model
+    parser.add_argument('--llm_engine', type=str, default='gpt-4-0613', help='llm engine',
+                        choices = ['gpt-3.5-turbo', 'gpt-3.5', 'gpt-4', 'gpt-4-0314', 'gpt-4-0613'])
+    parser.add_argument('--number', type=int, default=-1, help='number of problems to run')
+    parser.add_argument('--quick_extract', action='store_true', help='use rules to extract answer for some problems')
+    parser.add_argument('--rerun', action='store_true', help='rerun the answer extraction')
+    # output
+    parser.add_argument('--save_every', type=int, default=10, help='save every n problems')
+    parser.add_argument('--output_label', type=str, default='', help='label for the output file')
+    args = parser.parse_args()
+
+    # args
+    label = args.response_label
+    result_file = os.path.join(args.output_dir, args.output_file)
+
+    if args.output_label != '':
+        output_file = result_file.replace('.json', f'_{args.output_label}.json')
+    else:
+        output_file = result_file
+
+    # read results
+    print(f"Reading {result_file}...")
+    results = read_json(result_file)
+
+    # full pids
+    full_pids = list(results.keys())
+    if args.number > 0:
+        full_pids = full_pids[:min(args.number, len(full_pids))]
+    print("Number of testing problems:", len(full_pids))
+
+    # test pids
+    if args.rerun:
+        test_pids = full_pids
+    else:
+        test_pids = []
+        for pid in full_pids:
+            # print(pid)
+            if 'extraction' not in results[pid] or not verify_extraction(results[pid]['extraction']):
+                test_pids.append(pid)
+    
+    test_num = len(test_pids)
+    print("Number of problems to run:", test_num)
+    # print(test_pids)
+
+    # tqdm, enumerate results
+    for i, pid in enumerate(tqdm(test_pids)):
+        problem = results[pid]
+
+        assert label in problem
+        response = problem[label]       
+
+        
+        extraction  = extract_answer(response, problem, args.quick_extract)
+        results[pid]['extraction'] = extraction
+
+        if i % args.save_every == 0 or i == test_num - 1:
+            print(f"Saving results to {output_file}...")
+            save_json(results, output_file)
+            print(f"Results saved.")

eval/mathvista/prompts/ext_ans.py

@@ -0,0 +1,42 @@
+
+
+# pids = 852,  104,  824,  506,  540
+
+demo_prompt = """
+Please read the following example. Then extract the answer from the model response and type it at the end of the prompt.
+
+Hint: Please answer the question requiring an integer answer and provide the final value, e.g., 1, 2, 3, at the end.
+Question: Which number is missing?
+
+Model response: The number missing in the sequence is 14.
+
+Extracted answer: 14
+
+Hint: Please answer the question requiring a floating-point number with one decimal place and provide the final value, e.g., 1.2, 1.3, 1.4, at the end.
+Question: What is the fraction of females facing the camera?
+
+Model response: The fraction of females facing the camera is 0.6, which means that six out of ten females in the group are facing the camera.
+
+Extracted answer: 0.6
+
+Hint: Please answer the question requiring a floating-point number with two decimal places and provide the final value, e.g., 1.23, 1.34, 1.45, at the end.
+Question: How much money does Luca need to buy a sour apple candy and a butterscotch candy? (Unit: $)
+
+Model response: Luca needs $1.45 to buy a sour apple candy and a butterscotch candy.
+
+Extracted answer: 1.45
+
+Hint: Please answer the question requiring a Python list as an answer and provide the final list, e.g., [1, 2, 3], [1.2, 1.3, 1.4], at the end.
+Question: Between which two years does the line  graph saw its maximum peak?
+
+Model response: The line graph saw its maximum peak between 2007 and 2008.
+
+Extracted answer: [2007, 2008]
+
+Hint: Please answer the question and provide the correct option letter, e.g., A, B, C, D, at the end.
+Question: What fraction of the shape is blue?\nChoices:\n(A) 3/11\n(B) 8/11\n(C) 6/11\n(D) 3/5
+
+Model response: The correct answer is (B) 8/11.
+
+Extracted answer: B
+"""

eval/mathvista/utilities.py

@@ -0,0 +1,199 @@
+import os
+import cv2
+import json
+import time
+import pickle
+import openai
+import re
+from word2number import w2n
+
+
+def create_dir(output_dir):
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+    
+
+def read_csv(file):
+    data = []
+    with open(file, 'r') as f:
+        for line in f:
+            data.append(line.strip())
+    return data
+
+
+def read_pandas_csv(csv_path):
+    # read a pandas csv sheet 
+    import pandas as pd
+    df = pd.read_csv(csv_path)
+    return df
+
+
+def read_json(path):
+    with open(path, 'r', encoding='utf-8') as f:
+        return json.load(f)
+
+
+def read_jsonl(file):
+    with open(file, 'r') as f:
+        data = [json.loads(line) for line in f]
+    return data
+
+
+def read_pickle(path):
+    with open(path, 'rb') as f:
+        return pickle.load(f)
+
+
+def save_json(data, path):
+    with open(path, 'w') as f:
+        json.dump(data, f, indent=4)
+
+
+def save_array_img(path, image):
+    cv2.imwrite(path, image)
+
+
+def contains_digit(text):
+    # check if text contains a digit
+    if any(char.isdigit() for char in text):
+        return True
+    return False  
+    
+def contains_number_word(text):
+    # check if text contains a number word
+    ignore_words = ["a", "an", "point"]
+    words = re.findall(r'\b\w+\b', text)  # This regex pattern matches any word in the text
+    for word in words:
+        if word in ignore_words:
+            continue
+        try:
+            w2n.word_to_num(word)
+            return True  # If the word can be converted to a number, return True
+        except ValueError:
+            continue  # If the word can't be converted to a number, continue with the next word
+    
+    # check if text contains a digit
+    if any(char.isdigit() for char in text):
+        return True
+
+    return False  # If none of the words could be converted to a number, return False
+
+
+def contains_quantity_word(text, special_keep_words=[]):
+    # check if text contains a quantity word
+    quantity_words = ["most", "least", "fewest"
+                      "more", "less", "fewer", 
+                      "largest", "smallest", "greatest", 
+                      "larger", "smaller", "greater", 
+                      "highest", "lowest", "higher", "lower",
+                      "increase", "decrease",
+                      "minimum", "maximum", "max", "min",
+                      "mean", "average", "median",
+                      "total", "sum", "add", "subtract",
+                      "difference", "quotient", "gap",
+                      "half", "double", "twice", "triple",
+                      "square", "cube", "root",
+                      "approximate", "approximation",
+                      "triangle", "rectangle", "circle", "square", "cube", "sphere", "cylinder", "cone", "pyramid",
+                      "multiply", "divide",
+                      "percentage", "percent", "ratio", "proportion", "fraction", "rate", 
+                    ]
+    
+    quantity_words += special_keep_words # dataset specific words
+    
+    words = re.findall(r'\b\w+\b', text)  # This regex pattern matches any word in the text
+    if any(word in quantity_words for word in words):
+        return True
+
+    return False  # If none of the words could be converted to a number, return False
+
+
+def is_bool_word(text):
+    if text in ["Yes", "No", "True", "False", 
+                "yes", "no", "true", "false", 
+                "YES", "NO", "TRUE", "FALSE"]:
+        return True
+    return False
+
+
+def is_digit_string(text):
+    # remove ".0000"
+    text = text.strip()
+    text = re.sub(r'\.0+$', '', text)
+    try:
+        int(text)
+        return True
+    except ValueError:
+        return False
+   
+    
+def is_float_string(text):
+    # text is a float string if it contains a "." and can be converted to a float
+    if "." in text:
+        try:
+            float(text)
+            return True
+        except ValueError:
+            return False
+    return False
+
+
+def copy_image(image_path, output_image_path):
+    from shutil import copyfile
+    copyfile(image_path, output_image_path)
+
+
+def copy_dir(src_dir, dst_dir):
+    from shutil import copytree
+    # copy the source directory to the target directory
+    copytree(src_dir, dst_dir)
+
+
+import PIL.Image as Image
+def get_image_size(img_path):
+    img = Image.open(img_path)
+    width, height = img.size
+    return width, height
+
+
+def get_chat_response(promot, api_key, model="gpt-3.5-turbo", temperature=0, max_tokens=256, n=1, patience=10000000,
+ sleep_time=0):
+    messages = [
+        {"role": "user", "content": promot},
+    ]
+    # print("I am here")
+    while patience > 0:
+        patience -= 1
+        try:
+            response = openai.ChatCompletion.create(model=model,
+                                                messages=messages,
+                                                api_key=api_key,
+                                                temperature=temperature,
+                                                max_tokens=max_tokens,
+                                                n=n)
+            if n == 1:
+                prediction = response['choices'][0]['message']['content'].strip()
+                if prediction != "" and prediction != None:
+                    return prediction
+            else:
+                prediction = [choice['message']['content'].strip() for choice in response['choices']]
+                if prediction[0] != "" and prediction[0] != None:
+                    return prediction
+
+        except Exception as e:
+            if "Rate limit" not in str(e):
+                print(e)
+
+            if "Please reduce the length of the messages" in str(e):
+                print("!!Reduce promot size")
+                # reduce input prompt and keep the tail
+                new_size = int(len(promot) * 0.9)
+                new_start = len(promot) - new_size
+                promot = promot[new_start:]
+                messages = [
+                    {"role": "user", "content": promot},
+                ]
+                
+            if sleep_time > 0:
+                time.sleep(sleep_time)
+    return ""

eval/mm-vet/eval.sh

@@ -0,0 +1 @@
+python eval/mm-vet/evaluate_mmvet.py

eval/mm-vet/evaluate_mmvet.py

@@ -0,0 +1,250 @@
+import openai
+import json
+
+import os
+from tqdm import tqdm
+import pandas as pd
+import numpy as np
+from collections import Counter
+import time 
+
+gpt_model = "gpt-4-0613"
+openai.api_key= ""
+DATASET_ROOT=""
+MMVET = "mm-vet/mm-vet.json"
+
+prompt = """Compare the ground truth and prediction from AI models, to give a correctness score for the prediction. <AND> in the ground truth means it is totally right only when all elements in the ground truth are present in the prediction, and <OR> means it is totally right when any one element in the ground truth is present in the prediction. The correctness score is 0.0 (totally wrong), 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, or 1.0 (totally right). Just complete the last space of the correctness score.
+
+Question | Ground truth | Prediction | Correctness
+--- | --- | --- | ---
+What is x in the equation? | -1 <AND> -5 | x = 3 | 0.0
+What is x in the equation? | -1 <AND> -5 | x = -1 | 0.5
+What is x in the equation? | -1 <AND> -5 | x = -5 | 0.5
+What is x in the equation? | -1 <AND> -5 | x = -5 or 5 | 0.5
+What is x in the equation? | -1 <AND> -5 | x = -1 or x = -5 | 1.0
+Can you explain this meme? | This meme is poking fun at the fact that the names of the countries Iceland and Greenland are misleading. Despite its name, Iceland is known for its beautiful green landscapes, while Greenland is mostly covered in ice and snow. The meme is saying that the person has trust issues because the names of these countries do not accurately represent their landscapes. | The meme talks about Iceland and Greenland. It's pointing out that despite their names, Iceland is not very icy and Greenland isn't very green. | 0.4
+Can you explain this meme? | This meme is poking fun at the fact that the names of the countries Iceland and Greenland are misleading. Despite its name, Iceland is known for its beautiful green landscapes, while Greenland is mostly covered in ice and snow. The meme is saying that the person has trust issues because the names of these countries do not accurately represent their landscapes. | The meme is using humor to point out the misleading nature of Iceland's and Greenland's names. Iceland, despite its name, has lush green landscapes while Greenland is mostly covered in ice and snow. The text 'This is why I have trust issues' is a playful way to suggest that these contradictions can lead to distrust or confusion. The humor in this meme is derived from the unexpected contrast between the names of the countries and their actual physical characteristics. | 1.0
+"""
+
+# load metadata
+decimal_places = 1 # number of decimal places to round to
+
+
+sub_set = None
+sub_set_name = ''
+
+mmvet_metadata = os.path.join(DATASET_ROOT, MMVET)
+with open(mmvet_metadata, 'r') as f:
+    data = json.load(f)
+
+
+counter = Counter()
+cap_set_list = []
+cap_set_counter = []
+len_data = 0
+for id, value in data.items():
+    if sub_set is not None and id not in sub_set:
+        continue
+    question = value["question"]
+    answer = value["answer"]
+    cap = value["capability"]
+    cap = set(cap)
+    counter.update(cap)
+    if cap not in cap_set_list:
+        cap_set_list.append(cap)
+        cap_set_counter.append(1)
+    else:
+        cap_set_counter[cap_set_list.index(cap)] += 1
+    
+    len_data += 1
+
+sorted_list = counter.most_common()
+columns = [k for k, v in sorted_list]
+columns.append("total")
+columns.append("std")
+columns.append('runs')
+df = pd.DataFrame(columns=columns)
+
+
+cap_set_sorted_indices = np.argsort(-np.array(cap_set_counter))
+new_cap_set_list = []
+new_cap_set_counter = []
+for index in cap_set_sorted_indices:
+    new_cap_set_list.append(cap_set_list[index])
+    new_cap_set_counter.append(cap_set_counter[index])
+
+cap_set_list = new_cap_set_list
+cap_set_counter = new_cap_set_counter
+cap_set_names = ["_".join(list(cap_set)) for cap_set in cap_set_list]
+
+columns2 = cap_set_names
+columns2.append("total")
+columns2.append("std")
+columns2.append('runs')
+df2 = pd.DataFrame(columns=columns2)
+
+###### change your model name ######
+model = "Meteor"
+result_path = os.path.join(DATASET_ROOT, "eval_results")
+num_run = 1 # we set it as 5 in the paper
+model_results_file = os.path.join(result_path, f"{model}_mmvet_results.json")
+
+# grade results for each sample to svae
+grade_file = f'{model}_{gpt_model}-grade-{num_run}runs.json'
+grade_file = os.path.join(result_path, grade_file)
+
+# score results regarding capabilities/capability integration to save
+cap_score_file = f'{model}_{sub_set_name}{gpt_model}-cap-score-{num_run}runs.csv'
+cap_score_file = os.path.join(result_path, cap_score_file)
+cap_int_score_file = f'{model}_{sub_set_name}{gpt_model}-cap-int-score-{num_run}runs.csv'
+cap_int_score_file = os.path.join(result_path, cap_int_score_file)
+
+with open(model_results_file) as f:
+    results = json.load(f)
+if os.path.exists(grade_file):
+    with open(grade_file, 'r') as f:
+        grade_results = json.load(f)
+else:
+    grade_results = {}
+
+
+def need_more_runs():
+    need_more_runs = False
+    if len(grade_results) > 0:
+        for k, v in grade_results.items():
+            if len(v['score']) < num_run:
+                need_more_runs = True
+                break
+    return need_more_runs or len(grade_results) < len_data
+
+
+while need_more_runs():
+    for j in range(num_run):
+        print(f'eval run {j}')
+        for id, line in tqdm(data.items()):
+            if sub_set is not None and id not in sub_set:
+                continue
+            if id in grade_results and len(grade_results[id]['score']) >= (j + 1):
+                continue
+
+            model_pred = results[id]
+            
+            question = prompt + '\n' + ' | '.join([line['question'], line['answer'].replace("<AND>", " <AND> ").replace("<OR>", " <OR> "), model_pred, ""])
+            messages = [
+            {"role": "user", "content": question},
+            ]
+
+            if id not in grade_results:
+                sample_grade = {'model': [], 'content': [], 'score': []}
+            else:
+                sample_grade = grade_results[id]
+
+            
+            grade_sample_run_complete = False
+            temperature = 0.0
+
+            while not grade_sample_run_complete:
+                try:
+                    response = openai.ChatCompletion.create(
+                        model=gpt_model,
+                        max_tokens=3,
+                        temperature=temperature,
+                        messages=messages)
+                    content = response['choices'][0]['message']['content']
+                    flag = True
+                    try_time = 1
+                    while flag:
+                        try:
+                            content = content.split(' ')[0].strip()
+                            score = float(content)
+                            if score > 1.0 or score < 0.0:
+                                assert False
+                            flag = False
+                        except:
+                            question = prompt + '\n' + ' | '.join([line['question'], line['answer'].replace("<AND>", " <AND> ").replace("<OR>", " <OR> "), model_pred, ""]) + "\nPredict the correctness of the answer (digit): "
+                            messages = [
+                            {"role": "user", "content": question},
+                            ]
+                            response = openai.ChatCompletion.create(
+                                model=gpt_model,
+                                max_tokens=3,
+                                temperature=temperature,
+                                messages=messages)
+                            content = response['choices'][0]['message']['content']
+                            try_time += 1
+                            temperature += 0.5
+                            print(f"{id} try {try_time} times")
+                            print(content)
+                            if try_time > 5:
+                                score = 0.0
+                                flag = False
+                    grade_sample_run_complete = True
+                except:
+                    # gpt4 may have token rate limit
+                    print("sleep 30s")
+                    time.sleep(30)
+
+            if len(sample_grade['model']) >= j + 1:
+                sample_grade['model'][j] = response['model']
+                sample_grade['content'][j] = content
+                sample_grade['score'][j] = score
+            else:
+                sample_grade['model'].append(response['model'])
+                sample_grade['content'].append(content)
+                sample_grade['score'].append(score)
+            grade_results[id] = sample_grade
+
+            with open(grade_file, 'w') as f:
+                json.dump(grade_results, f, indent=4)
+
+
+assert not need_more_runs()
+cap_socres = {k: [0.0]*num_run for k in columns[:-2]}
+counter['total'] = len_data
+
+cap_socres2 = {k: [0.0]*num_run for k in columns2[:-2]}
+counter2 = {columns2[i]:cap_set_counter[i] for i in range(len(cap_set_counter))}
+counter2['total'] = len_data
+
+for k, v in grade_results.items():
+    if sub_set is not None and k not in sub_set:
+        continue
+    for i in range(num_run):
+        score = v['score'][i]
+        caps = set(data[k]['capability'])
+        for c in caps:
+            cap_socres[c][i] += score
+        
+        cap_socres['total'][i] += score
+
+        index = cap_set_list.index(caps)
+        cap_socres2[cap_set_names[index]][i] += score
+        cap_socres2['total'][i] += score
+
+for k, v in cap_socres.items():
+    cap_socres[k] = np.array(v) / counter[k] *100
+
+
+std = round(cap_socres['total'].std(), decimal_places)
+total_copy = cap_socres['total'].copy()
+runs = str(list(np.round(total_copy, decimal_places)))
+
+for k, v in cap_socres.items():
+    cap_socres[k] = round(v.mean(), decimal_places)
+
+cap_socres['std'] = std
+cap_socres['runs'] = runs
+df.loc[model] = cap_socres
+
+
+for k, v in cap_socres2.items():
+    cap_socres2[k] = round(np.mean(np.array(v) / counter2[k] *100), decimal_places)
+cap_socres2['std'] = std
+cap_socres2['runs'] = runs
+df2.loc[model] = cap_socres2
+
+df.to_csv(cap_score_file)
+df2.to_csv(cap_int_score_file)
+
+print(df)
+print(df2)

eval/utils.py

@@ -0,0 +1,1351 @@
+import os
+import re
+import json
+import openai
+from typing import Dict
+from tqdm import tqdm
+import random
+import numpy as np
+from sklearn.metrics import accuracy_score, precision_score, recall_score, confusion_matrix
+from typing import Optional
+from collections import defaultdict
+from eval.mathvista.utilities import get_chat_response
+from config import *
+from copy import deepcopy
+
+random.seed(42)
+
+# SEED Question types
+SEED_TYPES = {1: 'Scene Understanding', 2: 'Instance Identity', 3: 'Instance Location', 4: 'Instance Attributes', 5: 'Instances Counting', 6: 'Spatial Relation', 7: 'Instance Interaction', 8: 'Visual Reasoning', 9: 'Text Understanding'}
+
+# Check for duplicated questions, items
+def remove_duplicate(dataset, inputs, gen_answers):
+    if dataset == "mme": 
+        return inputs, gen_answers
+    elif dataset == "pope": 
+        questions = set()
+        new_inputs, new_answers = [], []
+        for i, a in zip(inputs, gen_answers):
+            dup = i['id'], i['category']
+            if dup in questions:
+                continue
+            questions.add(dup)
+            new_inputs.append(i)
+            new_answers.append(a)
+    else:
+        questions = set()
+        new_inputs, new_answers = [], []
+        for i, a in zip(inputs, gen_answers):
+            if i['id'] in questions:
+                continue
+            questions.add(i['id'])
+            new_inputs.append(i)
+            new_answers.append(a)
+    return new_inputs, new_answers
+
+class EvalAIAnswerProcessor:
+    """
+    Processes an answer similar to Eval AI
+        copied from
+        https://github.com/facebookresearch/mmf/blob/c46b3b3391275b4181567db80943473a89ab98ab/pythia/tasks/processors.py#L897
+    """
+
+    CONTRACTIONS = {
+        "aint": "ain't",
+        "arent": "aren't",
+        "cant": "can't",
+        "couldve": "could've",
+        "couldnt": "couldn't",
+        "couldn'tve": "couldn't've",
+        "couldnt've": "couldn't've",
+        "didnt": "didn't",
+        "doesnt": "doesn't",
+        "dont": "don't",
+        "hadnt": "hadn't",
+        "hadnt've": "hadn't've",
+        "hadn'tve": "hadn't've",
+        "hasnt": "hasn't",
+        "havent": "haven't",
+        "hed": "he'd",
+        "hed've": "he'd've",
+        "he'dve": "he'd've",
+        "hes": "he's",
+        "howd": "how'd",
+        "howll": "how'll",
+        "hows": "how's",
+        "Id've": "I'd've",
+        "I'dve": "I'd've",
+        "Im": "I'm",
+        "Ive": "I've",
+        "isnt": "isn't",
+        "itd": "it'd",
+        "itd've": "it'd've",
+        "it'dve": "it'd've",
+        "itll": "it'll",
+        "let's": "let's",
+        "maam": "ma'am",
+        "mightnt": "mightn't",
+        "mightnt've": "mightn't've",
+        "mightn'tve": "mightn't've",
+        "mightve": "might've",
+        "mustnt": "mustn't",
+        "mustve": "must've",
+        "neednt": "needn't",
+        "notve": "not've",
+        "oclock": "o'clock",
+        "oughtnt": "oughtn't",
+        "ow's'at": "'ow's'at",
+        "'ows'at": "'ow's'at",
+        "'ow'sat": "'ow's'at",
+        "shant": "shan't",
+        "shed've": "she'd've",
+        "she'dve": "she'd've",
+        "she's": "she's",
+        "shouldve": "should've",
+        "shouldnt": "shouldn't",
+        "shouldnt've": "shouldn't've",
+        "shouldn'tve": "shouldn't've",
+        "somebody'd": "somebodyd",
+        "somebodyd've": "somebody'd've",
+        "somebody'dve": "somebody'd've",
+        "somebodyll": "somebody'll",
+        "somebodys": "somebody's",
+        "someoned": "someone'd",
+        "someoned've": "someone'd've",
+        "someone'dve": "someone'd've",
+        "someonell": "someone'll",
+        "someones": "someone's",
+        "somethingd": "something'd",
+        "somethingd've": "something'd've",
+        "something'dve": "something'd've",
+        "somethingll": "something'll",
+        "thats": "that's",
+        "thered": "there'd",
+        "thered've": "there'd've",
+        "there'dve": "there'd've",
+        "therere": "there're",
+        "theres": "there's",
+        "theyd": "they'd",
+        "theyd've": "they'd've",
+        "they'dve": "they'd've",
+        "theyll": "they'll",
+        "theyre": "they're",
+        "theyve": "they've",
+        "twas": "'twas",
+        "wasnt": "wasn't",
+        "wed've": "we'd've",
+        "we'dve": "we'd've",
+        "weve": "we've",
+        "werent": "weren't",
+        "whatll": "what'll",
+        "whatre": "what're",
+        "whats": "what's",
+        "whatve": "what've",
+        "whens": "when's",
+        "whered": "where'd",
+        "wheres": "where's",
+        "whereve": "where've",
+        "whod": "who'd",
+        "whod've": "who'd've",
+        "who'dve": "who'd've",
+        "wholl": "who'll",
+        "whos": "who's",
+        "whove": "who've",
+        "whyll": "why'll",
+        "whyre": "why're",
+        "whys": "why's",
+        "wont": "won't",
+        "wouldve": "would've",
+        "wouldnt": "wouldn't",
+        "wouldnt've": "wouldn't've",
+        "wouldn'tve": "wouldn't've",
+        "yall": "y'all",
+        "yall'll": "y'all'll",
+        "y'allll": "y'all'll",
+        "yall'd've": "y'all'd've",
+        "y'alld've": "y'all'd've",
+        "y'all'dve": "y'all'd've",
+        "youd": "you'd",
+        "youd've": "you'd've",
+        "you'dve": "you'd've",
+        "youll": "you'll",
+        "youre": "you're",
+        "youve": "you've",
+    }
+
+    NUMBER_MAP = {
+        "none": "0",
+        "zero": "0",
+        "one": "1",
+        "two": "2",
+        "three": "3",
+        "four": "4",
+        "five": "5",
+        "six": "6",
+        "seven": "7",
+        "eight": "8",
+        "nine": "9",
+        "ten": "10",
+    }
+    ARTICLES = ["a", "an", "the"]
+    PERIOD_STRIP = re.compile(r"(?!<=\d)(\.)(?!\d)")
+    COMMA_STRIP = re.compile(r"(?<=\d)(\,)+(?=\d)")
+    PUNCTUATIONS = [
+        ";",
+        r"/",
+        "[",
+        "]",
+        '"',
+        "{",
+        "}",
+        "(",
+        ")",
+        "=",
+        "+",
+        "\\",
+        "_",
+        "-",
+        ">",
+        "<",
+        "@",
+        "`",
+        ",",
+        "?",
+        "!",
+    ]
+
+    def __init__(self, *args, **kwargs):
+        pass
+
+    def word_tokenize(self, word):
+        word = word.lower()
+        word = word.replace(",", "").replace("?", "").replace("'s", " 's")
+        return word.strip()
+
+    def process_punctuation(self, in_text):
+        out_text = in_text
+        for p in self.PUNCTUATIONS:
+            if (p + " " in in_text or " " + p in in_text) or (
+                re.search(self.COMMA_STRIP, in_text) is not None
+            ):
+                out_text = out_text.replace(p, "")
+            else:
+                out_text = out_text.replace(p, " ")
+        out_text = self.PERIOD_STRIP.sub("", out_text, re.UNICODE)
+        return out_text
+
+    def process_digit_article(self, in_text):
+        out_text = []
+        temp_text = in_text.lower().split()
+        for word in temp_text:
+            word = self.NUMBER_MAP.setdefault(word, word)
+            if word not in self.ARTICLES:
+                out_text.append(word)
+            else:
+                pass
+        for word_id, word in enumerate(out_text):
+            if word in self.CONTRACTIONS:
+                out_text[word_id] = self.CONTRACTIONS[word]
+        out_text = " ".join(out_text)
+        return out_text
+
+    def __call__(self, item):
+        item = self.word_tokenize(item)
+        item = item.replace("\n", " ").replace("\t", " ").strip()
+        item = self.process_punctuation(item)
+        item = self.process_digit_article(item)
+        return item
+
+
+class TextVQAAccuracyEvaluator:
+    def __init__(self):
+        self.answer_processor = EvalAIAnswerProcessor()
+
+    def _compute_answer_scores(self, raw_answers):
+        """
+        compute the accuracy (soft score) of human answers
+        """
+        answers = [self.answer_processor(a) for a in raw_answers]
+        assert len(answers) == 10
+        gt_answers = list(enumerate(answers))
+        unique_answers = set(answers)
+        unique_answer_scores = {}
+
+        for unique_answer in unique_answers:
+            accs = []
+            for gt_answer in gt_answers:
+                other_answers = [item for item in gt_answers if item != gt_answer]
+                matching_answers = [
+                    item for item in other_answers if item[1] == unique_answer
+                ]
+                acc = min(1, float(len(matching_answers)) / 3)
+                accs.append(acc)
+            unique_answer_scores[unique_answer] = sum(accs) / len(accs)
+
+        return unique_answer_scores
+
+    def eval_pred_list(self, pred_list):
+        pred_scores = []
+        for entry in pred_list:
+            pred_answer = self.answer_processor(entry["pred_answer"])
+            unique_answer_scores = self._compute_answer_scores(entry["gt_answers"])
+            score = unique_answer_scores.get(pred_answer, 0.0)
+            pred_scores.append(score)
+
+        accuracy = sum(pred_scores) / len(pred_scores)
+        return accuracy
+
+# MME
+class MMEEvaluator:
+    def divide_chunks(self, l, n=2):
+        # looping till length l
+        for i in range(0, len(l), n): 
+            yield l[i:i + n]
+        
+        return 
+
+    def parse_pred_ans(self, pred_ans):
+        pred_label = None
+        if pred_ans in ["yes", "no"]:
+            pred_label = pred_ans
+        else:
+            prefix_pred_ans = pred_ans[:4]
+
+            if "yes" in prefix_pred_ans:
+                pred_label = "yes"
+            elif "no" in prefix_pred_ans:
+                pred_label = "no"
+            else:
+                pred_label = "other"
+
+        return pred_label
+
+
+    def compute_metric(self, gts, preds):
+        assert len(gts) == len(preds)
+
+        label_map = {
+            "yes": 1,
+            "no": 0,
+            "other": -1,
+        }
+        
+        gts = [label_map[x] for x in gts]
+        preds = [label_map[x] for x in preds]
+
+        acc = accuracy_score(gts, preds) 
+
+        clean_gts = []
+        clean_preds = []
+        other_num = 0 
+        for gt, pred in zip(gts, preds):
+            if pred == -1:
+                other_num += 1
+                continue
+            clean_gts.append(gt)
+            clean_preds.append(pred)
+        
+
+        conf_mat = confusion_matrix(clean_gts, clean_preds, labels=[1,0])
+        precision = precision_score(clean_gts, clean_preds, average='binary')
+        recall = recall_score(clean_gts, clean_preds, average='binary')
+        tp, fn = conf_mat[0]
+        fp, tn = conf_mat[1]
+
+        metric_dict = dict()
+        metric_dict = {
+            "TP": tp,
+            "FN": fn,
+            "TN": tn,
+            "FP": fp,
+            "precision": precision,
+            "recall": recall,
+            "other_num": other_num,
+            "acc": acc,
+        }
+
+        return metric_dict
+
+
+    def process_result(self, results_dir):
+        eval_type_dict = {
+            "Perception": ["existence", "count", "position", "color", "posters", "celebrity", "scene", "landmark", "artwork", "OCR"],
+            "Cognition": ["commonsense_reasoning", "numerical_calculation", "text_translation", "code_reasoning"]
+        }
+
+        model_score_dict = dict()
+        for eval_type, task_name_list in eval_type_dict.items():
+           
+            scores = 0
+            task_score_dict = dict()
+
+            for task_name in task_name_list:
+                if not os.path.exists(results_dir):
+                    os.makedirs(results_dir)
+                task_txt = os.path.join(results_dir, task_name + ".txt")
+                lines = open(task_txt, 'r').readlines()
+                chunk_lines = list(self.divide_chunks(lines)) # one image corresponds to two questions
+                
+                img_num = len(chunk_lines)
+                task_other_ans_num = 0
+                task_score = 0
+                acc_plus_correct_num = 0
+                gts = []
+                preds = []
+
+                for img_items in chunk_lines:
+                    assert len(img_items) == 2
+                    img_correct_num = 0
+
+                    for img_item in img_items:
+                        img_name, question, gt_ans, pred_ans = img_item.split("\t")
+
+                        gt_ans = gt_ans.lower()
+                        pred_ans = pred_ans.lower()
+
+                        assert gt_ans in ["yes", "no"] # gt can only be yes or no.
+
+                        pred_ans = self.parse_pred_ans(pred_ans)
+                        assert pred_ans in ["yes", "no", "other"]
+
+                        gts.append(gt_ans)
+                        preds.append(pred_ans)
+                        
+                        if gt_ans == pred_ans:
+                            img_correct_num += 1
+                        
+                        if pred_ans not in ["yes", "no"]:
+                            task_other_ans_num += 1
+
+                    if img_correct_num == 2:
+                        acc_plus_correct_num += 1
+
+                # cal TP precision acc, etc.
+                metric_dict = self.compute_metric(gts, preds)
+                acc_plus = acc_plus_correct_num / img_num
+                metric_dict["acc_plus"] = acc_plus
+                
+                
+                for k, v in metric_dict.items():
+                    if k in ["acc", "acc_plus"]:
+                        task_score += v*100
+                
+                task_score_dict[task_name] = task_score
+                
+                scores += task_score
+            task_score_dict['total'] = scores
+            model_score_dict[eval_type] = task_score_dict
+        return model_score_dict
+
+# For MMMU, convert all <image #> tokens to <image>
+def replace_image_tokens(question):
+    replaced = set()
+    def replace_token(match):
+        token = match.group(0)
+        if token not in replaced:
+            replaced.add(token)
+            return '<image>'
+        return token
+
+    pattern = re.compile(r'<image\s\d+>')
+    return pattern.sub(replace_token, question)
+
+# For MMMU, count all <image #> tokens
+def count_unique_image_tokens(string):
+    pattern = r'<image\s\d+>'
+    matches = re.findall(pattern, string)
+    return len(set(matches))
+
+# TextVQA    
+def prompt_processor(self, prompt):
+    if prompt.startswith('OCR tokens: '):
+        pattern = r"Question: (.*?) Short answer:"
+        match = re.search(pattern, prompt, re.DOTALL)
+        question = match.group(1)
+    elif 'Reference OCR token: ' in prompt and len(prompt.split('\n')) == 3:
+        if prompt.startswith('Reference OCR token:'):
+            question = prompt.split('\n')[1]
+        else:
+            question = prompt.split('\n')[0]
+    elif len(prompt.split('\n')) == 2:
+        question = prompt.split('\n')[0]
+    else:
+        assert False
+
+    return question.lower()
+
+# Convert answer to integer
+def char_to_int(char):
+    return ord(char.upper()) - ord('A')
+
+# In case model does not output a single letter, find the choice in answer
+def convert_to_choice(answer, candidates):
+    options = ["A", "B", "C", "D", "E"]
+    if answer in options:
+        extracted_answer = answer
+    elif len(answer) >= 2 and answer[0] in options and "." in answer:
+        extracted_answer= answer[0]
+    else:
+        pattern = re.compile(r'The answer is ([A-Z]).')
+        res = pattern.findall(answer)
+        if len(res) == 1:
+            extracted_answer = res[0]  # 'A', 'B', ...
+        else:
+            extracted_answer = "FAILED"
+
+    if extracted_answer in options[:len(candidates)]:
+        return options.index(extracted_answer)
+    else:
+        return -1
+    
+def get_pred_idx(prediction, choices, options):
+    """
+    Get the index (e.g. 2) from the prediction (e.g. 'C')
+    """
+    if prediction in options[:len(choices)]:
+        return options.index(prediction)
+    else:
+        return -1
+
+# Chart QA
+def relaxed_correctness(target: str,
+                        prediction: str,
+                        max_relative_change: float = 0.05) -> bool:
+    """Calculates relaxed correctness.
+
+    The correctness tolerates certain error ratio defined by max_relative_change.
+    See https://arxiv.org/pdf/2203.10244.pdf, end of section 5.1:
+    “Following Methani et al. (2020), we use a relaxed accuracy measure for the
+    numeric answers to allow a minor inaccuracy that may result from the automatic
+    data extraction process. We consider an answer to be correct if it is within
+    5% of the gold answer. For non-numeric answers, we still need an exact match
+    to consider an answer to be correct.”
+
+    Args:
+    target: Target string.
+    prediction: Predicted string.
+    max_relative_change: Maximum relative change.
+
+    Returns:
+    Whether the prediction was correct given the specified tolerance.
+    """
+
+    def _to_float(text: str) -> Optional[float]:
+        try:
+            if text.endswith('%'):
+                # Convert percentages to floats.
+                return float(text.rstrip('%')) / 100.0
+            else:
+                return float(text)
+        except ValueError:
+            return None
+
+    prediction_float = _to_float(prediction)
+    target_float = _to_float(target)
+    if prediction_float is not None and target_float:
+        relative_change = abs(prediction_float -
+                            target_float) / abs(target_float)
+        return relative_change <= max_relative_change
+    else:
+        return prediction.lower() == target.lower()
+
+# MME
+def get_gt(data_path):
+    ground_truth = {}
+    for category in os.listdir(data_path):
+        category_dir = os.path.join(data_path, category)
+        if not os.path.isdir(category_dir):
+            continue
+        if os.path.exists(os.path.join(category_dir, 'images')):
+            image_path = os.path.join(category_dir, 'images')
+            qa_path = os.path.join(category_dir, 'questions_answers_YN')
+        else:
+            image_path = qa_path = category_dir
+        assert os.path.isdir(image_path), image_path
+        assert os.path.isdir(qa_path), qa_path
+        for file in os.listdir(qa_path):
+            if not file.endswith('.txt'):
+                continue
+            for line in open(os.path.join(qa_path, file)):
+                question, answer = line.strip().split('\t')
+                ground_truth[(category, file, question)] = answer
+    return ground_truth
+
+# Chart QA
+def relaxed_correctness(target: str,
+                        prediction: str,
+                        max_relative_change: float = 0.05) -> bool:
+    """Calculates relaxed correctness.
+
+    The correctness tolerates certain error ratio defined by max_relative_change.
+    See https://arxiv.org/pdf/2203.10244.pdf, end of section 5.1:
+    “Following Methani et al. (2020), we use a relaxed accuracy measure for the
+    numeric answers to allow a minor inaccuracy that may result from the automatic
+    data extraction process. We consider an answer to be correct if it is within
+    5% of the gold answer. For non-numeric answers, we still need an exact match
+    to consider an answer to be correct.”
+
+    Args:
+    target: Target string.
+    prediction: Predicted string.
+    max_relative_change: Maximum relative change.
+
+    Returns:
+    Whether the prediction was correct given the specified tolerance.
+    """
+
+    def _to_float(text: str) -> Optional[float]:
+        try:
+            if text.endswith('%'):
+                # Convert percentages to floats.
+                return float(text.rstrip('%'))
+            else:
+                return float(text)
+        except ValueError:
+            return None
+
+    prediction_float = _to_float(prediction)
+    target_float = _to_float(target)
+    if prediction_float is not None and target_float:
+        relative_change = abs(prediction_float -
+                            target_float) / abs(target_float)
+        return relative_change <= max_relative_change
+    else:
+        return prediction.lower() == target.lower()
+        
+def evaluate_relaxed_accuracy(entries):
+    scores = []
+    for elem in entries:
+        if isinstance(elem['annotation'], str):
+            elem['annotation'] = [elem['annotation']]
+        score = max([
+            relaxed_correctness(elem['answer'].strip(), ann)
+            for ann in elem['annotation']
+        ])
+        scores.append(score)
+    return sum(scores) / len(scores)
+
+# POPE
+def eval_pope(answers, label_file):
+    label_list = [json.loads(q)['label'] for q in open(label_file, 'r')]
+
+    for answer in answers:
+        text = answer['answer'].lower()
+
+        # Only keep the first sentence
+        if text.find('.') != -1:
+            text = text.split('.')[0]
+
+        text = text.replace(',', '')
+        words = text.split(' ')
+        if 'No' in words or 'not' in words or 'no' in words:
+            answer['answer'] = 'no'
+        else:
+            answer['answer'] = 'yes'
+
+    for i in range(len(label_list)):
+        if label_list[i] == 'no':
+            label_list[i] = 0
+        else:
+            label_list[i] = 1
+
+    pred_list = []
+    for answer in answers:
+        if answer['answer'] == 'no':
+            pred_list.append(0)
+        else:
+            pred_list.append(1)
+
+    pos = 1
+    neg = 0
+
+    TP, TN, FP, FN = 0, 0, 0, 0
+    for pred, label in zip(pred_list, label_list):
+        if pred == pos and label == pos:
+            TP += 1
+        elif pred == pos and label == neg:
+            FP += 1
+        elif pred == neg and label == neg:
+            TN += 1
+        elif pred == neg and label == pos:
+            FN += 1
+
+    acc = (TP + TN) / (TP + TN + FP + FN)
+    return acc
+
+# Eval GQA
+# book to float
+def toScore(b):
+    return float(1 if b else 0)
+
+# Compute average of a list
+def avg(l):
+    if len(l) == 0:
+        return 0
+    return float(sum(l)) / len(l)
+
+def eval_gqa(predictions, questions):
+    # Initialize data structure to track all metrics: e.g. accuracy, validity and plausibility, as well as 
+    # accuracy per question type, length and number of reasoning steps.
+    scores = {
+        "accuracy": [], # list of accuracies per question (1 if correct else 0). Will be averaged ultimately.
+        "binary": [], # list of accuracies per a binary question (1 if correct else 0). Will be averaged ultimately.
+        "open": [], # list of accuracies per an open question (1 if correct else 0). Will be averaged ultimately.
+        "validity": [], # list of validity per question (1 if valid else 0).
+        "plausibility": [], # list of plausibility per question (1 if plausible else 0). 
+        "consistency": [], # list of consistency scores for entailed questions.
+        "accuracyPerStructuralType": defaultdict(list), # list of question accuracies for each structural type (e.g. compare, logic questions).
+        "accuracyPerSemanticType": defaultdict(list), # list of question accuracies for each semantic type (e.g. questions about an object, an attribute, a relation).
+        "accuracyPerLength": defaultdict(list), # list of question accuracies per question's word number.
+        "accuracyPerSteps": defaultdict(list), # list of question accuracies per question's reasoning length (steps number).
+        "grounding": [] # list of grounding scores for each question.
+    }
+
+    # Initialize golden and predicted histograms per each question group. Used to compute the distribution metric.
+    dist = {
+        "gold": defaultdict(lambda: defaultdict(int)),
+        "predicted": defaultdict(lambda: defaultdict(int))
+    }
+
+    ##### Question lengths - words numbers and reasoning steps number
+    ##########################################################################################
+
+    # Compute question length (words number)
+    def getWordsNum(question):
+        return len(question["question"].split())
+
+    # Compute number of reasoning steps (excluding the final "querying" step which doesn't increase effective reasoning length)
+    def getStepsNum(question):
+        return len([c for c in question["semantic"] if not (any([o in "{}: {}".format(c["operation"], c["argument"]) 
+            for o in ["exist", "query: name", "choose name"]]))])
+
+    ##### Main score computation 
+    ##########################################################################################
+
+    # Loop over the questions and compute mterics
+    for qid, question in questions.items():
+        gold = question["answer"]
+        if qid not in predictions:
+            continue
+        predicted = predictions[qid].lower()
+
+        correct = (predicted == gold)
+        score = toScore(correct)
+
+        wordsNum = getWordsNum(question)
+        stepsNum = getStepsNum(question)
+        
+        # Compute scores over the balanced dataset (more robust against cheating by making educated guesses)
+        if question["isBalanced"]:
+            # Update accuracy
+            scores["accuracy"].append(score)
+            scores["accuracyPerLength"][wordsNum].append(score)
+            scores["accuracyPerSteps"][stepsNum].append(score)
+            scores["accuracyPerStructuralType"][question["types"]["structural"]].append(score)
+            scores["accuracyPerSemanticType"][question["types"]["semantic"]].append(score)
+            answerType = "open" if question["types"]["structural"] == "query" else "binary"
+            scores[answerType].append(score)
+    
+            # Update histograms for gold and predicted answers
+            globalGroup = question["groups"]["global"]
+            if globalGroup is not None:
+                dist["gold"][globalGroup][gold] += 1
+                dist["predicted"][globalGroup][predicted] += 1
+
+    # Average scores over all questions (in the balanced dataset) and print scores
+    metrics = [
+        "binary",
+        "open",
+        "accuracy",
+        "consistency",
+        "validity",
+        "plausibility",
+        "grounding",
+    ]
+
+    detailedMetrics = [
+        ("accuracyPerStructuralType", "Accuracy / structural type"), 
+        ("accuracyPerSemanticType", "Accuracy / semantic type"), 
+        ("accuracyPerSteps", "Accuracy / steps number"),
+        ("accuracyPerLength", "Accuracy / words number") 
+    ]
+
+    subMetrics = {
+        "attr": "attribute",
+        "cat": "category",
+        "global": "scene",
+        "obj": "object",
+        "rel": "relation" 
+    }
+    # average
+    for k in metrics:
+        if isinstance(scores[k], list):
+            scores[k] = avg(scores[k]) * 100
+
+    for k, _ in detailedMetrics:
+        for t in scores[k]:
+            scores[k][t] = avg(scores[k][t]) * 100, len(scores[k][t])
+
+    # print
+    print("")
+    for m in metrics:
+        # skip grounding and consistency scores if not requested
+        if m == "grounding":
+            continue
+        if m == "consistency":
+            continue
+
+        # print score
+        print("{title}: {score:.2f}{suffix}".format(title = m.capitalize(), score = scores[m], 
+            suffix = " (lower is better)" if m == "distribution" else "%"))
+
+    for m, mPrintName in detailedMetrics:
+        print("")
+        # print metric title
+        print("{}:".format(mPrintName))
+        
+        for t in sorted(list(scores[m].keys())):
+            # set sub-metric title
+            tName = t
+            if isinstance(scores[k], list):
+                tName = subMetrics.get(t, t).capitalize()
+
+            # print score
+            print("  {title}: {score:.2f}{suffix} ({amount} questions)".format(title = tName, 
+                score = scores[m][t][0], suffix = "%", amount = scores[m][t][1]))    
+            
+    return scores
+
+# MMMU
+DOMAIN_CAT2SUB_CAT = {
+  'Art and Design': ['Art', 'Art_Theory', 'Design', 'Music'],
+  'Business': ['Accounting', 'Economics', 'Finance', 'Manage','Marketing'],
+  'Science': ['Biology', 'Chemistry', 'Geography', 'Math', 'Physics',],
+  'Health and Medicine': ['Basic_Medical_Science', 'Clinical_Medicine', 'Diagnostics_and_Laboratory_Medicine', 'Pharmacy', 'Public_Health'],
+  'Humanities and Social Science': ['History', 'Literature', 'Sociology', 'Psychology'],
+  'Tech and Engineering': ['Agriculture', 'Architecture_and_Engineering', 'Computer_Science', 'Electronics', 'Energy_and_Power', 'Materials', 'Mechanical_Engineering'],
+}
+
+
+CAT_SHORT2LONG = {
+    'acc': 'Accounting',
+    'agri': 'Agriculture',
+    'arch': 'Architecture_and_Engineering',
+    'art': 'Art',
+    'art_theory': 'Art_Theory',
+    'bas_med': 'Basic_Medical_Science',
+    'bio': 'Biology',
+    'chem': 'Chemistry',
+    'cli_med': 'Clinical_Medicine',
+    'cs': 'Computer_Science',
+    'design': 'Design',
+    'diag_med': 'Diagnostics_and_Laboratory_Medicine',
+    'econ': 'Economics',
+    'elec': 'Electronics',
+    'ep': 'Energy_and_Power',
+    'fin': 'Finance',
+    'geo': 'Geography',
+    'his': 'History',
+    'liter': 'Literature',
+    'manage': 'Manage',
+    'mark': 'Marketing',
+    'mate': 'Materials',
+    'math': 'Math',
+    'mech': 'Mechanical_Engineering',
+    'music': 'Music',
+    'phar': 'Pharmacy',
+    'phys': 'Physics',
+    'psy': 'Psychology',
+    'pub_health': 'Public_Health',
+    'socio': 'Sociology'
+}
+
+"""Response Parsing and Evaluation for various models"""
+
+# ----------- Process Multi-choice -------------
+def parse_multi_choice_response(response, all_choices, index2ans):
+    """
+    Parse the prediction from the generated response.
+    Return the predicted index e.g., A, B, C, D.
+    """
+    for char in [',', '.', '!', '?', ';', ':', "'"]:
+        response = response.strip(char)
+    response = " " + response + " " # add space to avoid partial match
+
+    index_ans = True
+    ans_with_brack = False
+    candidates = []
+    for choice in all_choices:  # e.g., (A) (B) (C) (D)
+        if f'({choice})' in response:
+            candidates.append(choice)
+            ans_with_brack = True
+
+    if len(candidates) == 0:
+        for choice in all_choices: # e.g., A B C D
+            if f' {choice} ' in response:
+                candidates.append(choice)
+
+    # if all above doesn't get candidates, check if the content is larger than 5 tokens and try to parse the example
+    if len(candidates) == 0 and len(response.split()) > 5:
+        for index, ans in index2ans.items():
+            if ans.lower() in response.lower():
+                candidates.append(index)
+                index_ans = False # it's content ans.
+
+    if len(candidates) == 0:  # still not get answer, randomly choose one.
+        pred_index = random.choice(all_choices)
+    elif len(candidates) > 1:
+        start_indexes = []
+        if index_ans:
+            if ans_with_brack: 
+                for can in candidates:
+                    index = response.rfind(f'({can})')
+                    start_indexes.append(index) # -1 will be ignored anyway
+                # start_indexes = [generated_response.index(f'({can})') for can in candidates]
+            else:
+                for can in candidates:
+                    index = response.rfind(f" {can} ")
+                    start_indexes.append(index)
+        else:
+            for can in candidates:
+                index = response.lower().rfind(index2ans[can].lower())
+                start_indexes.append(index)
+        # get the last one
+        pred_index = candidates[np.argmax(start_indexes)]
+    else: # if only one candidate, use it.
+        pred_index = candidates[0]
+
+    return pred_index
+
+# ----------- Process Open -------------
+def check_is_number(string):
+    """
+    Check if the given string a number.
+    """
+    try:
+        float(string.replace(',', ''))
+        return True
+    except ValueError:
+        # check if there's comma inside
+        return False
+
+def normalize_str(string):
+    """
+    Normalize the str to lower case and make them float numbers if possible.
+    """
+    # check if characters in the string
+
+    # if number, numerize it.
+    string = string.strip()
+
+    is_number = check_is_number(string)
+
+    if is_number:
+        string = string.replace(',', '')
+        string = float(string)
+        # leave 2 decimal
+        string = round(string, 2)
+        return [string]
+    else: # it's likely to be a string
+        # lower it 
+        string = string.lower()
+        if len(string) == 1:
+            return [" " + string, string + " "] # avoid trivial matches
+        return [string]
+
+def extract_numbers(string):
+    """
+    Exact all forms of numbers from a string with regex.
+    """
+    # Pattern for numbers with commas
+    pattern_commas = r'-?\b\d{1,3}(?:,\d{3})+\b'
+    # Pattern for scientific notation
+    pattern_scientific = r'-?\d+(?:\.\d+)?[eE][+-]?\d+'
+    # Pattern for simple numbers without commas
+    pattern_simple = r'-?(?:\d+\.\d+|\.\d+|\d+\b)(?![eE][+-]?\d+)(?![,\d])'
+
+    # Extract numbers with commas
+    numbers_with_commas = re.findall(pattern_commas, string)
+    # Extract numbers in scientific notation
+    numbers_scientific = re.findall(pattern_scientific, string)
+    # Extract simple numbers without commas
+    numbers_simple = re.findall(pattern_simple, string)
+
+    # Combine all extracted numbers
+    all_numbers = numbers_with_commas + numbers_scientific + numbers_simple
+    return all_numbers
+
+def parse_open_response(response):
+    """
+    Parse the prediction from the generated response.
+    Return a list of predicted strings or numbers.
+    """
+    # content = content.strip("\n").strip(".").strip(" ")
+    def get_key_subresponses(response):
+        key_responses = []
+        response = response.strip().strip(".").lower()
+        sub_responses = re.split(r'\.\s(?=[A-Z])|\n', response)
+        indicators_of_keys = ['could be ', 'so ', 'is ',
+                            'thus ', 'therefore ', 'final ', 'answer ', 'result ']
+        key_responses = []
+        for index, resp in enumerate(sub_responses):
+            # if last one, accept it's an equation (the entire response can be just one sentence with equation)
+            if index == len(sub_responses) - 1:
+                indicators_of_keys.extend(['='])
+            shortest_key_response = None # the shortest response that may contain the answer (tail part of the response)
+            for indicator in indicators_of_keys:
+                if indicator in resp:
+                    if not shortest_key_response:
+                        shortest_key_response = resp.split(indicator)[-1].strip()
+                    else:
+                        if len(resp.split(indicator)[-1].strip()) < len(shortest_key_response):
+                            shortest_key_response = resp.split(indicator)[-1].strip()
+                    # key_responses.append(resp.split(indicator)[1].strip())
+            
+            if shortest_key_response:
+                # and it's not trivial
+                if shortest_key_response.strip() not in [":", ",", ".", "!", "?", ";", ":", "'"]:
+                    key_responses.append(shortest_key_response)
+        if len(key_responses) == 0: # did not found any
+            return [response]
+        return key_responses
+    # pdb.set_trace()
+    key_responses = get_key_subresponses(response)
+
+    pred_list = key_responses.copy() # keep the original string response
+    for resp in key_responses:
+        pred_list.extend(extract_numbers(resp))
+
+    tmp_pred_list = []
+    for i in range(len(pred_list)):
+        tmp_pred_list.extend(normalize_str(pred_list[i]))
+    pred_list = tmp_pred_list
+
+    # remove duplicates
+    pred_list = list(set(pred_list))
+
+    return pred_list
+
+# ----------- Evaluation -------------
+
+def eval_multi_choice(gold_i, pred_i):
+    """
+    Evaluate a multiple choice instance.
+    """
+    correct = False
+    # only they are exactly the same, we consider it as correct
+    if isinstance(gold_i, list):
+        for answer in gold_i:
+            if answer == pred_i:
+                correct = True
+                break
+    else: # gold_i is a string
+        if gold_i == pred_i:
+            correct = True
+    return correct
+
+def eval_open(gold_i, pred_i):
+    """
+    Evaluate an open question instance
+    """
+    correct = False
+    if isinstance(gold_i, list):
+        # use float to avoid trivial matches
+        norm_answers = []
+        for answer in gold_i:
+            norm_answers.extend(normalize_str(answer))
+    else:
+        norm_answers = normalize_str(gold_i)
+    for pred in pred_i: # pred is already normalized in parse response phase
+        if isinstance(pred, str): # if it's a string, then find if ans in the pred_i
+            for norm_ans in norm_answers:
+                # only see if the string answer in the string pred
+                if isinstance(norm_ans, str) and norm_ans in pred:
+                    if not correct:
+                        correct = True
+                    break
+        else: # it's a float number
+            if pred in norm_answers:
+                if not correct:
+                    correct = True
+                break
+    return correct
+
+# ----------- Batch Evaluation -------------
+def evaluate(samples):
+    """
+    Batch evaluation for multiple choice and open questions.
+    """
+    pred_correct = 0
+    judge_dict = dict()
+    for sample in samples:
+        gold_i = sample['answer']
+        pred_i = sample['parsed_pred']
+        if sample['question_type'] == 'multiple-choice':
+            correct = eval_multi_choice(gold_i, pred_i)
+        else: # open question
+            correct = eval_open(gold_i, pred_i)
+
+        if correct:
+            judge_dict[sample['id']] = 'Correct'
+            pred_correct += 1
+        else:
+            judge_dict[sample['id']] = 'Wrong'
+
+    if len(samples) == 0:
+        return {'acc': 0}
+    return judge_dict, {'acc': pred_correct / len(samples)}
+
+
+
+# ----------- Calculate Accuracy -------------
+def calculate_ins_level_acc(results: Dict):
+    """Calculate the instruction level accuracy for given Subject results"""
+    acc = 0
+    ins_num = 0
+    for cat_results in results.values():
+        acc += cat_results['acc'] * cat_results['num_example']
+        ins_num += cat_results['num_example']
+    if ins_num == 0:
+        return 0
+    return acc / ins_num
+    
+def eval_mathverse(output_dir, results, extract_file, score_file):
+    openai.api_key = OPENAI_KEY
+    
+    demo_prompt_extract = """
+I am providing you a response from a model to a math problem, termed 'Model Response'. You should extract the answer from the response as 'Extracted Answer'. Directly output the extracted answer with no explanation.
+
+1.
+Model response: 'Rounded to two decimal places, the perimeter of the sector is approximately:\n\n(-2, 1)'
+Extracted Answer: (-2, 1)
+
+2.
+Model response: 'at those points.\n\nTherefore, the correct option that represents the meaning of the intersection points of the graphs is:\n\nD. They give the solutions to the equation $f(t)=g(t)$.",'
+Extracted Answer: D
+
+3.
+Model response: ' at 1 (there's a closed circle at y = 1), the range in interval notation is \\((-4, 1]\\).\n\nFinal values:\nDomain: \\((-3, 3]\\)\nRange: \\((-4, 1]\\)'
+Extracted Answer: Domain: \\((-3, 3]\\)\nRange: \\((-4, 1]\\)
+
+4.
+Model response: 'As it stands, I cannot provide the correct option letter because there isn't enough information to solve for 'y'.'
+Extracted Answer: null
+
+5.
+Model response: 'Given that AB = 17.6 meters, we can now substitute into the equation:\n\nd = 17.6 / cos(38\u00b0)\n\nTherefore, to one decimal place, the distance d between Ned and Bart is approximately 22.3 meters.'
+Extracted answer: 22.3
+
+6.
+Model response:  have all the coefficients for the quadratic function:\n\\( f(x) = ax^2 + bx + c \\)\n\\( f(x) = -1x^2 - 2x + 1 \\)\n\nTherefore, the equation for the graphed function \\( f \\) is:\n\\( f(x) = -x^2 - 2x + 1 \\)"'
+Extracted answer: f(x) = -x^2 - 2x + 1
+
+7.
+"""
+    demo_prompt_score = """
+Below are two answers to a math question. Question is [Question], [Standard Answer] is the standard answer to the question, and [Model_answer] is the answer extracted from a model's output to this question.  Determine whether these two answers are consistent.
+Please note that only when the [Model_answer] completely matches the [Standard Answer] means they are consistent. For non-multiple-choice questions, if the meaning is expressed in the same way, it is also considered consistent, for example, 0.5m and 50cm.
+If they are consistent, Judement is 1; if they are different, Judement is 0.
+
+[Question]: Write the set of numbers represented on the number line in interval notation.
+[Standard Answer]: (-2,1]
+[Model_answer] : Extracted Answer: \\((-2, 1)\\)
+Judgement: 0
+
+[Question]: As shown in the figure, circle O has a radius 1.0, if angle BAC = 60.0, then the length of BC is ()\nChoices:\nA:2\nB:2\u221a{{3}}\nC:\u221a{{3}}\nD:2\u221a{{2}}
+[Standard Answer]: C
+[Model_answer] : B:2\u221a{{3}}
+Judgement: 0
+
+[Question]: Find the domain and range of the function f using interval notation.
+[Standard Answer]: domain: [-4, 0) and range: (-3, 1]
+[Model_answer] : Range: \\((-4, 1]\\)
+Judgement: 0
+
+[Question]: As shown in the figure, circle O has a radius 1.0, if angle BAC = 60.0, then the length of BC is ()\nChoices:\nA:2\nB:2\u221a{{3}}\nC:\u221a{{3}}\nD:2\u221a{{2}}
+[Standard Answer]: C
+[Model_answer] : null
+Judgement: 0
+
+[Question]: Given the graph of the ellipse that intersects with x-axis at 9 and -9 and with y-axis at 3 and -3, determine its equation.A. \\frac{{x^2}}{{81}} + \\frac{{y^2}}{{9}} = 1 B. Can not determine.\n
+[Standard Answer]: A
+[Model_answer] : \\frac{{x^2}}{{81}} + \\frac{{y^2}}{{9}} = 1
+Judgement: 1
+
+[Question]: {question}
+[Standard Answer]: {gt}
+[Model_answer] : {extraction}
+Judgement: """
+    
+    def create_extract_prompt(demo_prompt, response, inst):
+        demo_prompt = demo_prompt.strip()
+        test_prompt = f"Model response: '{response}'\nExtracted Answer: "
+        full_prompt = f"{demo_prompt}\n\n{test_prompt}"
+        return full_prompt
+    
+    def create_scoring_prompt(demo_prompt, inst):
+        demo_prompt = demo_prompt.strip()
+        full_prompt = demo_prompt.format(question = inst['question'], gt=inst['answer'], extraction=inst['extraction'])
+        return full_prompt
+
+
+    def extract_answer(response, inst, api_key):
+        # general extraction
+        try:
+            full_prompt = create_extract_prompt(demo_prompt_extract, response, inst)
+            extraction = get_chat_response(full_prompt, api_key)
+            return extraction
+        except Exception as e:
+            print(e)
+            print(f"Error in extracting answer for {response}")
+        return ""
+    
+    def match_answer(inst, api_key):
+        try:
+            full_prompt = create_scoring_prompt(demo_prompt_score, inst)
+            extraction = get_chat_response(full_prompt, api_key)
+            return extraction.replace("Judgement:", "").strip()
+        except Exception as e:
+            print(e)
+            print(f"Error in matching answer")
+
+        return ""
+    
+    save_results = []
+    score_dict = defaultdict(lambda: defaultdict(list))
+    score_version_dict = defaultdict(list)
+    
+    for i, inst in enumerate(tqdm(results)):
+        response = inst['model_answer']
+        extraction = extract_answer(response, inst, OPENAI_KEY)
+        inst['extraction'] = extraction.replace('Extracted Answer: ', '').strip()
+        
+        judgement = match_answer(inst, OPENAI_KEY)
+        while True:
+            if judgement.strip() not in ['0', '1']:
+                print('Wrong return format: ', judgement)
+                judgement = match_answer(inst, OPENAI_KEY)
+            else:
+                inst['judgement'] = int(judgement)
+                break
+
+        save_results.append(inst)
+
+        score_dict[inst['metadata']['subject']][inst['metadata']['subfield']].append(inst['judgement'])
+        score_version_dict[inst['problem_version']].append(inst['judgement'])
+    
+    results_file = os.path.join(output_dir, extract_file)
+    with open(results_file, 'w') as f:
+        json.dump(save_results, f, indent=4)
+        
+    print(f"Save MathVerse Results at {results_file}")
+    
+    save_json = {}
+    # version level acc
+    total_cnt, right_cnt = 0, 0
+    for version in score_version_dict:
+        version_total_cnt = len(score_version_dict[version])
+        version_right_cnt = len([inst for inst in score_version_dict[version] if inst == 1])
+        total_cnt += version_total_cnt
+        right_cnt += version_right_cnt
+        print(f"{version} Acc: {(version_right_cnt/version_total_cnt):.3f}")
+        save_json[version] = f"{(version_right_cnt/version_total_cnt):.3f}"
+
+    print(f"Acc: {(right_cnt/total_cnt):.3f}")
+    
+    save_json["Total Acc"] = f"{(right_cnt/total_cnt):.3f}"
+    
+    scores_file = os.path.join(output_dir, score_file)
+    with open(scores_file, 'w') as f:
+        json.dump(save_json, f, indent=4)
+    
+    
+def eval_mmstar(eval_file, output_dir, score_file):
+    MMStar_score_l2 = {
+        'coarse perception': {
+            'image scene and topic': 0,
+            'image style & quality': 0,
+            'image emotion': 0
+        },
+        'fine-grained perception': {
+            'object counting': 0,
+            'recognition': 0,
+            'localization': 0
+        },
+        'instance reasoning': {
+            'single-instance reasoning': 0,
+            'cross-instance attribute reasoning': 0,
+            'cross-instance relation reasoning': 0
+        },
+        'logical reasoning': {
+            'code & sequence reasoning': 0,
+            'diagram reasoning': 0,
+            'common reasoning': 0
+        },
+        'science & technology': {
+            'biology & chemistry & physics': 0,
+            'electronics & energy & mechanical eng.': 0,
+            'geography & earth science & agriculture': 0
+        },
+        'math': {
+            'geometry': 0,
+            'numeric commonsense and calculation': 0,
+            'statistical reasoning': 0
+        },
+    }
+    MMStar_counter = deepcopy(MMStar_score_l2)
+
+    data = eval_file
+    lt = len(data)
+    lines = [data.iloc[i] for i in range(lt)]
+    for i in tqdm(range(len(lines))):
+        line = lines[i]
+        predict = str(line['prediction'])
+        answers = str(line['answer'])
+        # ori_bench = str(line['bench'])
+        category = str(line['category'])
+        l2_category = str(line['l2_category'])
+        MMStar_counter[category][l2_category] += 1
+
+        answer = answers.lower().strip().replace('\n', ' ')
+        predict = predict.lower().strip().replace('\n', ' ')
+        # if ori_bench == 'MathVista' and answer not in ['a', 'b', 'c', 'd']:
+        #     if answer in predict:
+        #         MMStar_score_l2[category][l2_category] += 1
+        # else:
+        try:
+            if answer == predict[0]:
+                MMStar_score_l2[category][l2_category] += 1
+            elif predict[0] == '(' and answer == predict[1]:
+                MMStar_score_l2[category][l2_category] += 1
+            elif predict[0:7] == 'option ' and answer == predict[7]:
+                MMStar_score_l2[category][l2_category] += 1
+            elif predict[0:14] == 'the answer is ' and answer == predict[14]:
+                MMStar_score_l2[category][l2_category] += 1
+        except Exception as e:
+            pass
+
+    MMStar_score = {}
+    MMStar_score['final score'] = 0
+    for k, v in MMStar_score_l2.items():
+        MMStar_score[k] = 0
+        for l2_k, l2_v in v.items():
+            if float(MMStar_counter[k][l2_k]) == 0:
+                MMStar_score[f'{k}({l2_k})'] = 0
+            else:
+                MMStar_score[f'{k}({l2_k})'] = float(l2_v) / \
+                    float(MMStar_counter[k][l2_k])
+            MMStar_score[k] += l2_v
+        MMStar_score['final score'] += MMStar_score[k]
+        MMStar_score[k] = float(MMStar_score[k]) / 250.0
+    MMStar_score['final score'] = float(MMStar_score['final score']) / 1500.0
+
+    score_pth = os.path.join(output_dir, score_file)
+    with open(score_pth, 'w') as f:
+        json.dump(MMStar_score, f, indent=4)
+        
+    print(
+        f'MMStar_eval successfully finished evaluating {eval_file}, results saved in {score_pth}')
+    print('Score: ')
+    for key, value in MMStar_score.items():
+        print('{}:{}'.format(key, value))

loader/create_eval_dataset.py

@@ -0,0 +1,410 @@
+import os
+import json
+import math
+import glob
+from config import *
+from PIL import Image
+import pandas as pd
+import pyarrow.parquet as pq 
+import torch.nn.functional as F
+from eval.utils import *
+from torch.utils.data import Dataset
+from torchvision.transforms.functional import pil_to_tensor
+
+class CreateEvalDataset(Dataset):
+    def __init__(self):
+        super(CreateEvalDataset, self).__init__()
+
+        """
+        Eval Datasets
+
+        - VQAv2
+        - GQA
+        - SQA-IMG
+        - VizWiz
+        - TextVQA
+        - POPE
+        - MME
+        - MMBench
+        - MMBench-CN
+        - QBench
+        - MM-Vet
+        - MMMU
+        - MathVista
+        - AI2D
+        - HallusionBench
+        - ChartQA
+        - SEED
+        - LLaVA Wild
+        - BLINK
+        - MathVerse
+
+        """
+
+        # dataset root path
+        self.dataset_root_path = DATASET_ROOT
+
+        # load test data
+        pre_vqav2 = json.load(open(os.path.join(DATASET_ROOT, VQAV2)))
+        pre_gqa = json.load(open(os.path.join(DATASET_ROOT, GQA)))
+        pre_sqa = json.load(open(os.path.join(DATASET_ROOT, SQA)))
+        pre_sqa_split = json.load(open(os.path.join(DATASET_ROOT, SQA_SPLIT)))
+        pre_vizwiz = json.load(open(os.path.join(DATASET_ROOT, VIZWIZ)))
+        pre_textvqa = json.load(open(os.path.join(DATASET_ROOT, TEXTVQA)))
+        pre_textvqa_annotations = json.load(open(os.path.join(DATASET_ROOT, TEXTVQA_ANNOTATIONS)))
+        pre_pope_popular = pd.read_json(os.path.join(DATASET_ROOT, POPE_POPULAR), lines=True)
+        pre_pope_adversarial= pd.read_json(os.path.join(DATASET_ROOT, POPE_ADVERSARIAL), lines=True)
+        pre_pope_random = pd.read_json(os.path.join(DATASET_ROOT, POPE_RANDOM), lines=True)
+        pre_mme = json.load(open(os.path.join(DATASET_ROOT, MME)))
+        pre_mmbench = pd.read_table(os.path.join(DATASET_ROOT, MMBENCH))
+        pre_mmbench_dev = pd.read_table(os.path.join(DATASET_ROOT, MMBENCH_DEV))
+        pre_mmbench_cn = pd.read_table(os.path.join(DATASET_ROOT, MMBENCH_CN))
+        pre_mmbench_cn_dev = pd.read_table(os.path.join(DATASET_ROOT, MMBENCH_CN_DEV))
+        pre_qbench = json.load(open(os.path.join(DATASET_ROOT, QBENCH)))
+        pre_qbench_cn = json.load(open(os.path.join(DATASET_ROOT, QBENCH_CN)))
+        pre_mmvet = json.load(open(os.path.join(DATASET_ROOT, MMVET)))
+        mmmu_files = glob.glob(os.path.join(DATASET_ROOT, MMMU))
+        pre_mmmu = [pq.read_pandas(os.path.join(DATASET_ROOT, mf)).to_pandas() for mf in mmmu_files]
+        pre_mathvista1 = pq.read_pandas(os.path.join(DATASET_ROOT, MATHVISTA)).to_pandas()
+        pre_ai2d = json.load(open(os.path.join(DATASET_ROOT, AI2D)))
+        pre_hallusionbench = json.load(open(os.path.join(DATASET_ROOT, HALLUSIONBENCH)))
+        pre_chartqa = json.load(open(os.path.join(DATASET_ROOT, CHARTQA)))
+        pre_seed = json.load(open(os.path.join(DATASET_ROOT, SEED)))
+        pre_llava = pd.read_json(os.path.join(DATASET_ROOT, LLAVA), lines=True)
+        # pre_blink = json.load(open(os.path.join(DATASET_ROOT, BLINK)))
+        pre_mathverse = json.load(open(os.path.join(DATASET_ROOT, MATHVERSE)))
+        pre_mathverse_text_only = json.load(open(os.path.join(DATASET_ROOT, MATHVERSE_TEXT_ONLY)))
+        pre_mmstar = pq.read_pandas(os.path.join(DATASET_ROOT, MMSTAR)).to_pandas()
+        
+        # data filtering
+        vqav2 = self.vqav2_filtering(pre_vqav2)
+        gqa = self.gqa_filtering(pre_gqa)
+        sqa = self.sqa_filtering(pre_sqa, pre_sqa_split)
+        vizwiz = self.vizwiz_filtering(pre_vizwiz)
+        textvqa = self.textvqa_filtering(pre_textvqa, pre_textvqa_annotations)
+        pope = self.pope_filtering([pre_pope_popular, pre_pope_adversarial, pre_pope_random])
+        mme = self.mme_filtering(pre_mme)
+        mmbench = self.mmbench_filtering(pre_mmbench)
+        mmbench_dev = self.mmbench_filtering(pre_mmbench_dev)
+        mmbench_cn = self.mmbench_filtering(pre_mmbench_cn)
+        mmbench_cn_dev = self.mmbench_filtering(pre_mmbench_cn_dev)
+        qbench = self.qbench_filtering(pre_qbench)
+        qbench_cn = self.qbench_filtering(pre_qbench_cn)
+        mmvet = self.mmvet_filtering(pre_mmvet)
+        mmmu = self.mmmu_filtering(pre_mmmu)
+        mathvista = self.mathvista_filtering(pre_mathvista1)
+        ai2d = self.ai2d_filtering(pre_ai2d)
+        hallusionbench = self.hallusionbench_filtering(pre_hallusionbench)
+        chartqa = self.chartqa_filtering(pre_chartqa)
+        seed = self.seed_filtering(pre_seed)
+        llava = self.llava_filtering(pre_llava)
+        # blink = self.blink_filtering(pre_blink)
+        mathverse = self.mathverse_filtering(pre_mathverse, pre_mathverse_text_only)
+        mmstar = self.mmstar_filtering(pre_mmstar)
+        
+        # merging
+        self.data = {
+            'vqav2': vqav2,
+            'gqa': gqa,
+            'sqa':sqa,
+            'vizwiz': vizwiz,
+            'textvqa': textvqa,
+            'pope': pope,
+            'mme': mme,
+            'mmbench': mmbench,
+            'mmbench_dev': mmbench_dev,
+            'mmbench_cn': mmbench_cn,
+            'mmbench_cn_dev': mmbench_cn_dev,
+            'qbench': qbench,
+            'mm-vet': mmvet,
+            'mmmu': mmmu,
+            'mathvista': mathvista,
+            'ai2d': ai2d,
+            'hallusionbench': hallusionbench,
+            'chartqa': chartqa,
+            'seed': seed,
+            'llava': llava,
+            # 'blink': blink,
+            'mathverse' : mathverse,
+            'mmstar' : mmstar
+        }        
+
+
+    def vqav2_filtering(self, pre_data):
+        data = []
+        for x in pre_data['questions']:
+            data.append({'image': f"VQAv2/test2015/COCO_test2015_{x['image_id']:012d}.jpg",
+                        'question': x['question'],
+                        'id': x['question_id']})
+        return data
+    
+    def gqa_filtering(self, pre_data):
+        data = []
+        for qid, x in pre_data.items():
+            data.append({'image': f"gqa/images/{x['imageId']}.jpg",
+                        'question': x['question'],
+                        'id': qid})
+        return data
+    
+    def sqa_filtering(self, pre_data, pre_sqa_split):
+        data = []
+        questions = {idx: pre_data[idx] for idx in pre_sqa_split['test']}
+        for qid, x in questions.items():
+            if x['image'] is not None:
+                choices = '\n'.join(f"{chr(ord('A') + i)}. {choice}" for i, choice in enumerate(x['choices']))
+                question = '\n'.join([x['hint'], x['question'], choices])
+                data.append({'image': f"ScienceQA/images/test/{qid}/image.png",
+                            'question': question,
+                            'id': qid,
+                            'candidates': x['choices'],
+                            'gt': x['answer']})
+        return data
+    
+    def vizwiz_filtering(self, pre_data):
+        data = []
+        for qid, x in enumerate(pre_data):
+            data.append({'image': f"VizWiz/test/{x['image']}",
+                        'question': x['question'],
+                        'id': qid})
+        return data
+    
+    def textvqa_filtering(self, pre_data, annotations):
+        data = []
+        for x, answer in zip(pre_data, annotations['data']):
+            data.append({'image': f"TextVQA/train_images/{x['image']}",
+                        'question': x['text'],
+                        'id': x['question_id'],
+                        'gt': answer['answers']})
+        return data
+    
+    def pope_filtering(self, pre_data):
+        data = []
+        categories = ['adversarial', 'popular', 'random']
+        for category, split in zip(categories, pre_data):
+            for _, x in split.iterrows():
+                data.append({'image': f"coco2014/val2014/{x['image']}",
+                            'question': x['text'],
+                            'id': x['question_id'], 
+                            'category': category})
+        return data
+    
+    def mme_filtering(self, pre_data):
+        data = []
+        for x in pre_data:
+            data.append({'image': f"MME_Benchmark_release_version/{x['image']}",
+                        'question': x['text'],
+                        'id': x['question_id'],
+                        'category': x['category']})
+        return data
+    
+    def mmbench_filtering(self, pre_data):
+        data = []
+        for _, x in pre_data.iterrows():
+            options = ['A', 'B', 'C', 'D']
+            choice_list = [choice for choice in options if not self.is_none(x[choice])]
+            choices = '\n'.join(f"{chr(ord('A') + i)}. {x[choice]}" for i, choice in enumerate(choice_list))
+            question = '\n'.join([x['question'], choices])
+
+            if not self.is_none(x['hint']):
+                question = '\n'.join([x['hint'], question])
+        
+            data.append({'image': x['image'],
+                        'question': question,
+                        'id': x['index'],
+                        'answer': x['answer'] if 'answer' in x else None})
+        return data
+    
+    def qbench_filtering(self, pre_data):
+        data = []
+        for qid, x in enumerate(pre_data):
+            choices = '\n'.join(f"{chr(ord('A') + i)}. {choice}" for i, choice in enumerate(x['candidates']))
+            question = '\n'.join([x['question'], choices])
+            data.append({'image': f"LLVisionQA-QBench/images/{x['img_path']}",
+                        'question': question,
+                        'id': qid,
+                        'candidates': x['candidates'],
+                        'gt': x['correct_ans']})
+        return data
+    
+    def mmvet_filtering(self, pre_data):
+        data = []
+        for qid, x in pre_data.items():
+            data.append({'image': f"mm-vet/images/{x['imagename']}",
+                        'question': x['question'],
+                        'id': qid,
+                        'gt': x['answer'],
+                        'capability': x['capability']})
+        return data
+    
+    def mmmu_filtering(self, pre_data):
+        data = []
+        for split in pre_data:
+            for _, x in split.iterrows():
+                index2ans, all_choices = self.get_multi_choice_info(eval(x['options']))
+                choices = ' '.join([f"{k}. {v}" for k,v in index2ans.items()])
+                question = '\n'.join([x['question'], choices])
+                num_images = count_unique_image_tokens(question)
+                data.append({'images': [x[f"image_{i+1}"]['bytes'] for i in range(num_images)],
+                            'question': replace_image_tokens(question),
+                            'id': x['id'],
+                            'question_type': x['question_type'],
+                            'gt': x['answer'],
+                            'index2ans': index2ans,
+                            'all_choices': all_choices})
+        return data
+    
+    def mathvista_filtering(self, pre_data):
+        data = []
+        for _, x in pre_data.iterrows():
+            skills = x['metadata']['skills'].tolist()
+            x['metadata']['skills'] = skills
+            choices = x['choices'].tolist() if x['choices'] is not None else None
+            data.append({'image': f"MathVista/{x['image']}",
+                        'question': x['query'],
+                        'question_type': x['question_type'],
+                        'answer': x['answer'],
+                        'answer_type': x['answer_type'],
+                        'choices': choices,
+                        'metadata': x['metadata'],
+                        'precision': x['precision'],
+                        'id': x['pid']})
+        return data
+    
+    def ai2d_filtering(self, pre_data):
+        data = []
+        for x in pre_data:
+            choices = ' '.join(f"{chr(ord('A') + i)}. {choice}" for i, choice in enumerate(x["metadata"]["answerTexts"]))
+            question = '\n'.join([x['question'], choices])
+            image = f"ai2d/abc_images/{x['imageName']}" if x['metadata']['abcLabel'] else f"ai2d/images/{x['imageName']}"
+            data.append({'image': image,
+                         'question': question,
+                        'id': x['metadata']['questionId'],
+                        'gt': x['metadata']['correctAnswer']})
+        return data
+    
+    def hallusionbench_filtering(self, pre_data):
+        data = []
+        for qid, x in enumerate(pre_data):
+            if x['filename'] is None:
+                img_path = ""
+                question = x['question']
+            else:
+                img_path = f"HallusionBench/hallusion_bench/{x['filename'][2:]}".format()
+                question =  "<image>" + x['question']
+            data.append({'image': img_path,
+                        'question': question,
+                        'id': qid,
+                        'gt': x['gt_answer']})
+        return data
+
+    def chartqa_filtering(self, pre_data):
+        data = []
+        for qid, x in enumerate(pre_data):
+            data.append({'image': f"chartqa/test/png/{x['imgname']}",
+                        'question': x['query'],
+                        'id': x['imgname'],
+                        'gt': x['label']})
+        return data
+    
+    def seed_filtering(self, pre_data):
+        data = []
+        for x in pre_data['questions']:
+            if x['data_type'] != 'image':
+                continue
+            choice_list = [key for key in x.keys() if 'choice' in key]
+            choices = '\n'.join(f"{chr(ord('A') + i)}. {x[choice]}" for i, choice in enumerate(choice_list))
+            question = '\n'.join([x['question'], choices])
+            data.append({'image': f"SEED-Bench/SEED-Bench-image/{x['data_id']}",
+                        'question': question,
+                        'id': x['question_id'],
+                        'question_type': x['question_type_id'],
+                        'gt': x['answer']})
+        return data
+    
+    def llava_filtering(self, pre_data):
+        data = []
+        for _, x in pre_data.iterrows():
+            data.append({'image': f"llava-bench-in-the-wild/images/{x['image']}",
+                        'question': x['text'],
+                        'id': x['question_id'],
+                        "category": x['category']})
+        return data
+    
+    def blink_filtering(self, pre_data):
+        data = []
+        # TODO
+        return data
+    
+    def mathverse_filtering(self, pre_data, pre_data_text_only):
+        data = []
+        for x in pre_data:
+            data.append({'image': f"MathVerse/images/{x['image']}",
+                        'question': "<image>" + x['query_wo'],
+                        # 'question': "<image>" + x['query_cot'],
+                        'id': x['sample_index'],
+                        'problem_index': x['problem_index'],
+                        'problem_version': x['problem_version'],
+                        'gt' : x['answer'],
+                        'question_type': x['question_type'],
+                        'metadata' : x['metadata'],
+                        'query_cot' : x['query_cot'],
+                        'origin_question': x['question']
+                        })
+        offset = len(pre_data)
+        for x in pre_data_text_only:
+            data.append({'image': "",
+                        'question': x['query_wo'],
+                        # 'question': x['query_cot'],
+                        'id': str(int(x['sample_index']) + offset),
+                        'problem_index': x['problem_index'],
+                        'problem_version': x['problem_version'],
+                        'gt' : x['answer'],
+                        'question_type': x['question_type'],
+                        'metadata' : x['metadata'],
+                        'query_cot' : x['query_cot'],
+                        'origin_question': x['question']
+                        })
+            
+        return data
+    
+    def is_none(self, value):
+        return type(value) is float and math.isnan(value)
+
+    def get_options(self, row, options):
+        parsed_options = []
+        for option in options:
+            option_value = row[option]
+            if self.is_none(option_value):
+                break
+            parsed_options.append(option_value)
+        return parsed_options
+
+    def __len__(self):
+        return len(self.data)
+    
+    def get_multi_choice_info(self, options):
+        """
+        Given the list of options for multiple choice question
+        Return the index2ans and all_choices
+        """
+        
+        start_chr = 'A'
+        all_choices = []
+        index2ans = {}
+        for i, option in enumerate(options):
+            index2ans[chr(ord(start_chr) + i)] = option
+            all_choices.append(chr(ord(start_chr) + i))
+
+        return index2ans, all_choices
+    
+    def mmstar_filtering(self, pre_data):
+        data = []
+        for _, x in pre_data.iterrows():
+            data.append({'id' : x['index'],
+                        'question': x['question'],
+                        'answer': x['answer'],
+                        'category': x['category'],
+                        'l2_category': x['l2_category'],
+                        # 'bench': x['bench'],
+                        'image': x['image']})
+        return data

meteor/arch/build_module.py

@@ -0,0 +1,209 @@
+import re
+
+import torch
+import torch.nn as nn
+from transformers import CLIPVisionModel
+
+
+def build_vision_tower():
+    vision_tower = 'openai/clip-vit-large-patch14-336'
+    return CLIPVisionTower(vision_tower)
+
+
+def build_vision_projector():
+    projector_type = 'mlp2x_gelu'
+    mm_hidden_size = 1024
+    hidden_size = 4096
+
+    mlp_gelu_match = re.match(r'^mlp(\d+)x_gelu$', projector_type)
+    if mlp_gelu_match:
+        mlp_depth = int(mlp_gelu_match.group(1))
+        modules = [nn.Linear(mm_hidden_size, hidden_size)]
+        for _ in range(1, mlp_depth):
+            modules.append(nn.GELU())
+            modules.append(nn.Linear(hidden_size, hidden_size))
+        return nn.Sequential(*modules)
+
+    if projector_type == 'identity':
+        return IdentityMap()
+
+    raise ValueError(f'Unknown projector type: {projector_type}')
+
+
+class IdentityMap(nn.Module):
+
+    def __init__(self):
+        super().__init__()
+
+    def forward(self, x, *args, **kwargs):
+        return x
+
+    @property
+    def config(self):
+        return {'mm_projector_type': 'identity'}
+
+
+class CLIPVisionTower(nn.Module):
+
+    def __init__(self, vision_tower):
+        super().__init__()
+
+        self.is_loaded = False
+        self.is_resize_pos = False
+
+        self.vision_tower_name = vision_tower
+        self.select_layer = -1
+        self.select_feature = 'patch'
+        self.load_model()
+        self.resize_pos()
+
+    def load_model(self):
+        self.vision_tower = CLIPVisionModel.from_pretrained(
+            self.vision_tower_name)
+        self.vision_tower.requires_grad_(False)
+
+        self.is_loaded = True
+
+    def resize_pos(self):
+        pos_embed_checkpoint = self.vision_tower.vision_model.embeddings.position_embedding.weight
+        pos_embed_checkpoint = pos_embed_checkpoint.unsqueeze(0)
+        orig_size = 24
+        new_size = 35
+
+        if pos_embed_checkpoint.shape[1] == new_size**2 + 1:
+            self.is_resize_pos = True
+        else:
+            embedding_size = pos_embed_checkpoint.shape[-1]
+            num_extra_tokens = 1
+            new_num = new_size**2 + num_extra_tokens
+            # print('Position interpolate from %dx%d to %dx%d' %
+            #       (orig_size, orig_size, new_size, new_size))
+            extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens]
+            # only the position tokens are interpolated
+            pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:]
+            pos_tokens = pos_tokens.reshape(-1, orig_size, orig_size,
+                                            embedding_size).permute(
+                                                0, 3, 1, 2)
+            pos_tokens = torch.nn.functional.interpolate(
+                pos_tokens,
+                size=(new_size, new_size),
+                mode='bicubic',
+                align_corners=False)
+            pos_tokens = pos_tokens.permute(0, 2, 3, 1).flatten(1, 2)
+            new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1)
+
+            new_pos_embed = new_pos_embed.squeeze(0)
+
+            self.vision_tower.vision_model.embeddings.position_embedding = torch.nn.Embedding(
+                new_num, 1024)
+            self.vision_tower.vision_model.embeddings.position_embedding.weight = torch.nn.Parameter(
+                new_pos_embed.to(pos_embed_checkpoint.dtype))
+            self.vision_tower.vision_model.embeddings.position_ids = torch.arange(
+                new_num).expand((1, -1))
+
+            self.is_resize_pos = True
+
+    def feature_select(self, image_forward_outs):
+        image_features = image_forward_outs.hidden_states[self.select_layer]
+        if self.select_feature == 'patch':
+            image_features = image_features[:, 1:]
+        elif self.select_feature == 'cls_patch':
+            image_features = image_features
+        else:
+            raise ValueError(
+                f'Unexpected select feature: {self.select_feature}')
+        return image_features
+
+    def forward(self, images):
+        if not self.is_loaded:
+            self.load_model()
+        if type(images) is list:
+            image_features = []
+            for image in images:
+                image_forward_out = self.vision_tower(
+                    image.to(device=self.device,
+                             dtype=self.dtype).unsqueeze(0),
+                    output_hidden_states=True)
+                image_feature = self.feature_select(image_forward_out).to(
+                    image.dtype)
+                image_features.append(image_feature)
+        else:
+            image_forward_outs = self.vision_tower(
+                images.to(device=self.device, dtype=self.dtype),
+                output_hidden_states=True)
+            image_features = self.feature_select(image_forward_outs).to(
+                images.dtype)
+
+        return image_features
+
+    @property
+    def dummy_feature(self):
+        return torch.zeros(
+            1, self.hidden_size, device=self.device, dtype=self.dtype)
+
+    @property
+    def dtype(self):
+        return self.vision_tower.dtype
+
+    @property
+    def device(self):
+        return self.vision_tower.device
+
+    @property
+    def config(self):
+        if self.is_loaded:
+            return self.vision_tower.config
+        else:
+            return self.cfg_only
+
+    @property
+    def hidden_size(self):
+        return self.config.hidden_size
+
+    @property
+    def num_patches(self):
+        return (self.config.image_size // self.config.patch_size)**2
+
+
+class LoRA(nn.Module):
+
+    def __init__(self,
+                 in_features: int,
+                 out_features: int,
+                 bias: bool = True,
+                 device=None,
+                 dtype=None,
+                 lora_r=8,
+                 lora_alpha=16,
+                 lora_dropout=0.05,
+                 lora_len=0,
+                 **kwargs) -> None:
+        super().__init__()
+        self.lora_r = lora_r
+        self.lora_alpha = lora_alpha
+        self.lora_len = lora_len
+        if lora_dropout > 0.:
+            self.lora_dropout = nn.Dropout(p=lora_dropout)
+        else:
+            self.lora_dropout = lambda x: x
+        self.lora_scaling = self.lora_alpha / self.lora_r
+
+        self.lora_A = nn.Linear(
+            in_features, self.lora_r, bias=False, device=device, dtype=dtype)
+        self.lora_B = nn.Linear(
+            self.lora_r, out_features, bias=False, device=device, dtype=dtype)
+        self.ffn = nn.Linear(in_features, out_features, bias=bias, device=device, dtype=dtype)
+
+    def forward(self, x, im_mask=None):
+        res = self.ffn(x)
+        if im_mask is not None:
+            if torch.sum(im_mask) > 0:
+                part_x = x[im_mask]
+                res[im_mask] += self.lora_B(
+                    self.lora_A(
+                        self.lora_dropout(part_x))) * self.lora_scaling
+            else:
+                part_x = x[:, :1]
+                res[:, :1] += self.lora_B(
+                    self.lora_A(self.lora_dropout(part_x))) * 0
+        return res

meteor/arch/configuration_internlm2.py

@@ -0,0 +1,166 @@
+# Copyright (c) InternLM. All rights reserved.
+#
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
+# and OPT implementations in this library. It has been modified from its
+# original forms to accommodate minor architectural differences compared
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""InternLM model configuration."""
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+INTERNLM_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
+
+
+class InternLM2Config(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`InternLMModel`]. It is used to instantiate
+    an InternLM model according to the specified arguments, defining the model architecture. Instantiating a
+    configuration with the defaults will yield a similar configuration to that of the InternLM-7B.
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+
+    Args:
+        vocab_size (`int`, *optional*, defaults to 32000):
+            Vocabulary size of the InternLM model. Defines the number of different tokens that can be represented by the
+            `inputs_ids` passed when calling [`InternLMModel`]
+        hidden_size (`int`, *optional*, defaults to 4096):
+            Dimension of the hidden representations.
+        intermediate_size (`int`, *optional*, defaults to 11008):
+            Dimension of the MLP representations.
+        num_hidden_layers (`int`, *optional*, defaults to 32):
+            Number of hidden layers in the Transformer encoder.
+        num_attention_heads (`int`, *optional*, defaults to 32):
+            Number of attention heads for each attention layer in the Transformer encoder.
+        num_key_value_heads (`int`, *optional*):
+            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+            `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
+            by meanpooling all the original heads within that group. For more details checkout [this
+            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
+            `num_attention_heads`.
+        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+            The non-linear activation function (function or string) in the decoder.
+        max_position_embeddings (`int`, *optional*, defaults to 2048):
+            The maximum sequence length that this model might ever be used with. Typically set this to something large
+            just in case (e.g., 512 or 1024 or 2048).
+        initializer_range (`float`, *optional*, defaults to 0.02):
+            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+        rms_norm_eps (`float`, *optional*, defaults to 1e-12):
+            The epsilon used by the rms normalization layers.
+        use_cache (`bool`, *optional*, defaults to `True`):
+            Whether or not the model should return the last key/values attentions (not used by all models). Only
+            relevant if `config.is_decoder=True`.
+        tie_word_embeddings(`bool`, *optional*, defaults to `False`):
+            Whether to tie weight embeddings
+        Example:
+
+    ```python
+    >>> from transformers import InternLMModel, InternLMConfig
+
+    >>> # Initializing a InternLM internlm-7b style configuration
+    >>> configuration = InternLMConfig()
+
+    >>> # Initializing a model from the internlm-7b style configuration
+    >>> model = InternLMModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+    model_type = 'internlm'
+    _auto_class = 'AutoConfig'
+
+    def __init__(  # pylint: disable=W0102
+        self,
+        vocab_size=103168,
+        hidden_size=4096,
+        intermediate_size=11008,
+        num_hidden_layers=32,
+        num_attention_heads=32,
+        num_key_value_heads=None,
+        hidden_act='silu',
+        max_position_embeddings=2048,
+        initializer_range=0.02,
+        rms_norm_eps=1e-6,
+        use_cache=True,
+        pad_token_id=0,
+        bos_token_id=1,
+        eos_token_id=2,
+        tie_word_embeddings=False,
+        bias=True,
+        rope_theta=10000,
+        rope_scaling=None,
+        attn_implementation='eager',
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.bias = bias
+
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+        self.num_key_value_heads = num_key_value_heads
+
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.rope_scaling = rope_scaling
+        self._rope_scaling_validation()
+
+        self.attn_implementation = attn_implementation
+        if self.attn_implementation is None:
+            self.attn_implementation = 'eager'
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )
+
+    def _rope_scaling_validation(self):
+        """Validate the `rope_scaling` configuration."""
+        if self.rope_scaling is None:
+            return
+
+        if not isinstance(self.rope_scaling,
+                          dict) or len(self.rope_scaling) != 2:
+            raise ValueError(
+                '`rope_scaling` must be a dictionary with with two fields, `type` and `factor`, '
+                f'got {self.rope_scaling}')
+        rope_scaling_type = self.rope_scaling.get('type', None)
+        rope_scaling_factor = self.rope_scaling.get('factor', None)
+        if rope_scaling_type is None or rope_scaling_type not in [
+                'linear', 'dynamic'
+        ]:
+            raise ValueError(
+                f"`rope_scaling`'s type field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"
+            )
+        if rope_scaling_factor is None or not isinstance(
+                rope_scaling_factor, float) or rope_scaling_factor < 1.0:
+            raise ValueError(
+                f"`rope_scaling`'s factor field must be a float >= 1, got {rope_scaling_factor}"
+            )

meteor/arch/modeling_internlm2.py

@@ -0,0 +1,1080 @@
+# # Copyright (c) InternLM. All rights reserved.
+#
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
+# and OPT implementations in this library. It has been modified from its
+# original forms to accommodate minor architectural differences compared
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""PyTorch InternLM2 model."""
+import math
+import warnings
+from typing import List, Optional, Tuple, Union
+
+import torch
+import torch.nn.functional as F
+import torch.utils.checkpoint
+from einops import rearrange
+from torch import nn
+from transformers.activations import ACT2FN
+from transformers.modeling_outputs import BaseModelOutputWithPast
+from transformers.modeling_utils import PreTrainedModel
+from transformers.utils import (add_start_docstrings,
+                                add_start_docstrings_to_model_forward, logging)
+
+try:
+    from transformers.generation.streamers import BaseStreamer
+except:  # noqa # pylint: disable=bare-except
+    BaseStreamer = None
+
+from .build_module import LoRA
+from .configuration_internlm2 import InternLM2Config
+
+logger = logging.get_logger(__name__)
+
+_CONFIG_FOR_DOC = 'InternLM2Config'
+flash_attn_func, flash_attn_varlen_func = None, None
+pad_input, index_first_axis, unpad_input = None, None, None
+def _import_flash_attn():
+    global flash_attn_func, flash_attn_varlen_func
+    global pad_input, index_first_axis, unpad_input
+    try:
+        from flash_attn import flash_attn_func as _flash_attn_func, flash_attn_varlen_func as _flash_attn_varlen_func
+        from flash_attn.bert_padding import pad_input as _pad_input, index_first_axis as _index_first_axis, unpad_input as _unpad_input
+        flash_attn_func, flash_attn_varlen_func = _flash_attn_func, _flash_attn_varlen_func
+        pad_input, index_first_axis, unpad_input = _pad_input, _index_first_axis, _unpad_input
+    except ImportError:
+        raise ImportError("flash_attn is not installed.")
+
+# Copied from transformers.models.llama.modeling_llama._get_unpad_data
+def _get_unpad_data(attention_mask):
+    seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)
+    indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
+    max_seqlen_in_batch = seqlens_in_batch.max().item()
+    cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.torch.int32), (1, 0))
+    return (
+        indices,
+        cu_seqlens,
+        max_seqlen_in_batch,
+    )
+
+# Copied from transformers.models.bart.modeling_bart._make_causal_mask
+def _make_causal_mask(input_ids_shape: torch.Size,
+                      dtype: torch.dtype,
+                      device: torch.device,
+                      past_key_values_length: int = 0):
+    """Make causal mask used for bi-directional self-attention."""
+    bsz, tgt_len = input_ids_shape
+    mask = torch.full((tgt_len, tgt_len),
+                      torch.tensor(torch.finfo(dtype).min, device=device),
+                      device=device)
+    mask_cond = torch.arange(mask.size(-1), device=device)
+    mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0)
+    mask = mask.to(dtype)
+
+    if past_key_values_length > 0:
+        mask = torch.cat([
+            torch.zeros(
+                tgt_len, past_key_values_length, dtype=dtype, device=device),
+            mask
+        ],
+                         dim=-1)
+    return mask[None, None, :, :].expand(bsz, 1, tgt_len,
+                                         tgt_len + past_key_values_length)
+
+
+# Copied from transformers.models.bart.modeling_bart._expand_mask
+def _expand_mask(mask: torch.Tensor,
+                 dtype: torch.dtype,
+                 tgt_len: Optional[int] = None):
+    """Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len,
+    src_seq_len]`."""
+    bsz, src_len = mask.size()
+    tgt_len = tgt_len if tgt_len is not None else src_len
+
+    expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len,
+                                                  src_len).to(dtype)
+
+    inverted_mask = 1.0 - expanded_mask
+
+    return inverted_mask.masked_fill(
+        inverted_mask.to(torch.bool),
+        torch.finfo(dtype).min)
+
+
+class InternLM2RMSNorm(nn.Module):
+
+    def __init__(self, hidden_size, eps=1e-6):
+        """InternLM2RMSNorm is equivalent to T5LayerNorm."""
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.variance_epsilon = eps
+
+    def forward(self, hidden_states):
+        input_dtype = hidden_states.dtype
+        hidden_states = hidden_states.to(torch.float32)
+        variance = hidden_states.pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(variance +
+                                                    self.variance_epsilon)
+        return self.weight * hidden_states.to(input_dtype)
+
+
+class InternLM2RotaryEmbedding(nn.Module):
+
+    def __init__(self,
+                 dim,
+                 max_position_embeddings=2048,
+                 base=10000,
+                 device=None):
+        super().__init__()
+
+        self.dim = dim
+        self.max_position_embeddings = max_position_embeddings
+        self.base = base
+        inv_freq = 1.0 / (
+            self.base
+            **(torch.arange(0, self.dim, 2).float().to(device) / self.dim))
+        self.register_buffer('inv_freq', inv_freq, persistent=False)
+
+        # Build here to make `torch.jit.trace` work.
+        self._set_cos_sin_cache(
+            seq_len=max_position_embeddings,
+            device=self.inv_freq.device,
+            dtype=torch.get_default_dtype())
+
+    def _set_cos_sin_cache(self, seq_len, device, dtype):
+        self.max_seq_len_cached = seq_len
+        t = torch.arange(
+            self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype)
+
+        freqs = torch.einsum('i,j->ij', t, self.inv_freq)
+        # Different from paper, but it uses a different permutation in order to obtain the same calculation
+        emb = torch.cat((freqs, freqs), dim=-1)
+        self.register_buffer(
+            'cos_cached', emb.cos().to(dtype), persistent=False)
+        self.register_buffer(
+            'sin_cached', emb.sin().to(dtype), persistent=False)
+
+    def forward(self, x, seq_len=None):
+        # x: [bs, num_attention_heads, seq_len, head_size]
+        if seq_len > self.max_seq_len_cached:
+            self._set_cos_sin_cache(
+                seq_len=seq_len, device=x.device, dtype=x.dtype)
+
+        return (
+            self.cos_cached[:seq_len].to(dtype=x.dtype),
+            self.sin_cached[:seq_len].to(dtype=x.dtype),
+        )
+
+
+class InternLM2LinearScalingRotaryEmbedding(InternLM2RotaryEmbedding):
+    """InternLM2RotaryEmbedding extended with linear scaling.
+
+    Credits to the Reddit user /u/kaiokendev
+    """
+
+    def __init__(self,
+                 dim,
+                 max_position_embeddings=2048,
+                 base=10000,
+                 device=None,
+                 scaling_factor=1.0):
+        self.scaling_factor = scaling_factor
+        super().__init__(dim, max_position_embeddings, base, device)
+
+    def _set_cos_sin_cache(self, seq_len, device, dtype):
+        self.max_seq_len_cached = seq_len
+        t = torch.arange(
+            self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype)
+        t = t / self.scaling_factor
+
+        freqs = torch.einsum('i,j->ij', t, self.inv_freq)
+        # Different from paper, but it uses a different permutation in order to obtain the same calculation
+        emb = torch.cat((freqs, freqs), dim=-1)
+        self.register_buffer(
+            'cos_cached', emb.cos().to(dtype), persistent=False)
+        self.register_buffer(
+            'sin_cached', emb.sin().to(dtype), persistent=False)
+
+
+class InternLM2DynamicNTKScalingRotaryEmbedding(InternLM2RotaryEmbedding):
+    """InternLM2RotaryEmbedding extended with Dynamic NTK scaling.
+
+    Credits to the Reddit users /u/bloc97 and /u/emozilla.
+    """
+
+    def __init__(self,
+                 dim,
+                 max_position_embeddings=2048,
+                 base=10000,
+                 device=None,
+                 scaling_factor=1.0):
+        self.scaling_factor = scaling_factor
+        super().__init__(dim, max_position_embeddings, base, device)
+
+    def _set_cos_sin_cache(self, seq_len, device, dtype):
+        self.max_seq_len_cached = seq_len
+
+        if seq_len > self.max_position_embeddings:
+            base = self.base * ((self.scaling_factor * seq_len /
+                                 self.max_position_embeddings) -
+                                (self.scaling_factor - 1))**(
+                                    self.dim / (self.dim - 2))
+            inv_freq = 1.0 / (
+                base
+                **(torch.arange(0, self.dim, 2).float().to(device) / self.dim))
+            self.register_buffer('inv_freq', inv_freq, persistent=False)
+
+        t = torch.arange(
+            self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype)
+
+        freqs = torch.einsum('i,j->ij', t, self.inv_freq)
+        # Different from paper, but it uses a different permutation in order to obtain the same calculation
+        emb = torch.cat((freqs, freqs), dim=-1)
+        self.register_buffer(
+            'cos_cached', emb.cos().to(dtype), persistent=False)
+        self.register_buffer(
+            'sin_cached', emb.sin().to(dtype), persistent=False)
+
+
+def rotate_half(x):
+    """Rotates half the hidden dims of the input."""
+    x1 = x[..., :x.shape[-1] // 2]
+    x2 = x[..., x.shape[-1] // 2:]
+    return torch.cat((-x2, x1), dim=-1)
+
+
+# def apply_rotary_pos_emb(q, k, cos, sin, position_ids):
+#     # The first two dimensions of cos and sin are always 1, so we can `squeeze` them.
+#     cos = cos.squeeze(1).squeeze(0)  # [seq_len, dim]
+#     sin = sin.squeeze(1).squeeze(0)  # [seq_len, dim]
+#     cos = cos.unsqueeze(0).unsqueeze(0).expand(len(position_ids), -1, -1, -1)
+#     sin = sin.unsqueeze(0).unsqueeze(0).expand(len(position_ids), -1, -1, -1)
+#     if q.size(2) == 1:
+#         q_embed = (q * cos[:, :, -1:, :]) + (
+#             rotate_half(q) * sin[:, :, -1:, :])
+#     else:
+#         q_embed = (q * cos) + (rotate_half(q) * sin)
+
+#     if k.size(2) == 1:
+#         k_embed = (k * cos[:, :, -1:, :]) + (
+#             rotate_half(k) * sin[:, :, -1:, :])
+#     else:
+#         k_embed = (k * cos) + (rotate_half(k) * sin)
+
+#     return q_embed, k_embed
+
+# Copied from transformers.model.llama.modeling_llama.apply_rotary_pos_emb
+def apply_rotary_pos_emb(q, k, cos, sin, position_ids, unsqueeze_dim=1):
+    """Applies Rotary Position Embedding to the query and key tensors."""
+    cos = cos[position_ids].unsqueeze(unsqueeze_dim)
+    sin = sin[position_ids].unsqueeze(unsqueeze_dim)
+    q_embed = (q * cos) + (rotate_half(q) * sin)
+    k_embed = (k * cos) + (rotate_half(k) * sin)
+    return q_embed, k_embed
+
+
+class InternLM2MLP(nn.Module):
+
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.hidden_size = config.hidden_size
+        self.intermediate_size = config.intermediate_size
+
+        self.w1 = LoRA(
+            self.hidden_size,
+            self.intermediate_size,
+            bias=False,
+            lora_r=256,
+            lora_alpha=256,
+            lora_len=576)
+        self.w3 = LoRA(
+            self.hidden_size,
+            self.intermediate_size,
+            bias=False,
+            lora_r=256,
+            lora_alpha=256,
+            lora_len=576)
+        self.w2 = LoRA(
+            self.intermediate_size,
+            self.hidden_size,
+            bias=False,
+            lora_r=256,
+            lora_alpha=256,
+            lora_len=576)
+
+        self.act_fn = ACT2FN[config.hidden_act]
+
+    def forward(self, x, im_mask):
+        down_proj = self.w2(
+            self.act_fn(self.w1(x, im_mask)) * self.w3(x, im_mask), im_mask)
+
+        return down_proj
+
+
+def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
+    """This is the equivalent of torch.repeat_interleave(x, dim=1,
+    repeats=n_rep).
+
+    The hidden states go from (batch, num_key_value_heads, seqlen, head_dim) to
+    (batch, num_attention_heads, seqlen, head_dim)
+    """
+    batch, num_key_value_heads, slen, head_dim = hidden_states.shape
+    if n_rep == 1:
+        return hidden_states
+    hidden_states = hidden_states[:, :,
+                                  None, :, :].expand(batch,
+                                                     num_key_value_heads,
+                                                     n_rep, slen, head_dim)
+    return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen,
+                                 head_dim)
+
+
+class InternLM2Attention(nn.Module):
+    """Multi-headed attention from 'Attention Is All You Need' paper."""
+
+    def __init__(self, config: InternLM2Config):
+        super().__init__()
+        self.config = config
+        self.hidden_size = config.hidden_size
+        self.num_heads = config.num_attention_heads
+        self.head_dim = self.hidden_size // self.num_heads
+        self.num_key_value_heads = config.num_key_value_heads
+        self.num_key_value_groups = self.num_heads // self.num_key_value_heads
+        self.max_position_embeddings = config.max_position_embeddings
+        self.is_causal = True
+
+        if (self.head_dim * self.num_heads) != self.hidden_size:
+            raise ValueError(
+                f'hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}'
+                f' and `num_heads`: {self.num_heads}).')
+
+        self.wqkv = LoRA(
+            self.hidden_size,
+            (self.num_heads + 2 * self.num_key_value_heads) * self.head_dim,
+            bias=config.bias,
+            lora_r=256,
+            lora_alpha=256,
+            lora_len=576)
+
+        self.wo = LoRA(
+            self.num_heads * self.head_dim,
+            self.hidden_size,
+            bias=config.bias,
+            lora_r=256,
+            lora_alpha=256,
+            lora_len=576)
+        self._init_rope()
+
+    def _init_rope(self):
+        if self.config.rope_scaling is None:
+            self.rotary_emb = InternLM2RotaryEmbedding(
+                self.head_dim,
+                max_position_embeddings=self.max_position_embeddings,
+                base=self.config.rope_theta,
+            )
+        else:
+            scaling_type = self.config.rope_scaling['type']
+            scaling_factor = self.config.rope_scaling['factor']
+            if scaling_type == 'dynamic':
+                self.rotary_emb = InternLM2DynamicNTKScalingRotaryEmbedding(
+                    self.head_dim,
+                    max_position_embeddings=self.max_position_embeddings,
+                    base=self.config.rope_theta,
+                    scaling_factor=scaling_factor)
+            else:
+                raise ValueError(
+                    "Currently we only support rotary embedding's type being 'dynamic'."
+                )
+        return self.rotary_emb
+
+    def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
+        return tensor.view(bsz, seq_len, self.num_heads,
+                           self.head_dim).transpose(1, 2).contiguous()
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Tuple[torch.Tensor]] = None,
+        output_attentions: bool = False,
+        use_cache: bool = False,
+        im_mask: Optional[Tuple[torch.Tensor]] = None,
+        **kwargs,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor],
+               Optional[Tuple[torch.Tensor]]]:
+        if 'padding_mask' in kwargs:
+            warnings.warn(
+                'Passing `padding_mask` is deprecated and will be removed in v4.37. '
+                'Please make sure use `attention_mask` instead.`')
+
+        bsz, q_len, _ = hidden_states.size()
+
+        qkv_states = self.wqkv(hidden_states, im_mask)
+
+        qkv_states = rearrange(
+            qkv_states,
+            'b q (h gs d) -> b q h gs d',
+            gs=2 + self.num_key_value_groups,
+            d=self.head_dim,
+        )
+
+        query_states = qkv_states[..., :self.num_key_value_groups, :]
+        query_states = rearrange(query_states, 'b q h gs d -> b q (h gs) d')
+        key_states = qkv_states[..., -2, :]
+        value_states = qkv_states[..., -1, :]
+
+        query_states = query_states.transpose(1, 2)
+        key_states = key_states.transpose(1, 2)
+        value_states = value_states.transpose(1, 2)
+
+        kv_seq_len = key_states.shape[-2]
+        if past_key_value is not None:
+            kv_seq_len += past_key_value[0].shape[-2]
+        cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+        query_states, key_states = apply_rotary_pos_emb(
+            query_states, key_states, cos, sin, position_ids)
+
+        if past_key_value is not None:
+            # reuse k, v, self_attention
+            key_states = torch.cat([past_key_value[0], key_states], dim=2)
+            value_states = torch.cat([past_key_value[1], value_states], dim=2)
+
+        past_key_value = (key_states, value_states) if use_cache else None
+
+        key_states = repeat_kv(key_states, self.num_key_value_groups)
+        value_states = repeat_kv(value_states, self.num_key_value_groups)
+
+        attn_weights = torch.matmul(query_states, key_states.transpose(
+            2, 3)) / math.sqrt(self.head_dim)
+
+        if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):
+            raise ValueError(
+                f'Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is'
+                f' {attn_weights.size()}')
+
+        if attention_mask is not None:
+            if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):
+                raise ValueError(
+                    f'Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}'
+                )
+            attn_weights = attn_weights + attention_mask
+
+        # upcast attention to fp32
+        attn_weights = nn.functional.softmax(
+            attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
+        attn_output = torch.matmul(attn_weights, value_states)
+
+        if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
+            raise ValueError(
+                f'`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is'
+                f' {attn_output.size()}')
+
+        attn_output = attn_output.transpose(1, 2).contiguous()
+        attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
+
+        attn_output = self.wo(attn_output, im_mask)
+
+        if not output_attentions:
+            attn_weights = None
+
+        return attn_output, attn_weights, past_key_value
+
+
+class InternLM2FlashAttention2(InternLM2Attention):
+    """InternLM2 flash attention module.
+
+    This module inherits from `InternLM2Attention` as the weights of the module
+    stays untouched. The only required change would be on the forward pass
+    where it needs to correctly call the public API of flash attention and deal
+    with padding tokens in case the input contains any of them.
+    """
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.LongTensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Tuple[torch.Tensor]] = None,
+        output_attentions: bool = False,
+        use_cache: bool = False,
+        im_mask: Optional[Tuple[torch.Tensor]] = None,
+        **kwargs,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor],
+               Optional[Tuple[torch.Tensor]]]:
+        # InternLM2FlashAttention2 attention does not support output_attentions
+        if 'padding_mask' in kwargs:
+            warnings.warn(
+                'Passing `padding_mask` is deprecated and will be removed in v4.37. '
+                'Please make sure use `attention_mask` instead.`')
+
+            # overwrite attention_mask with padding_mask
+            attention_mask = kwargs.pop('padding_mask')
+
+        output_attentions = False
+
+        bsz, q_len, _ = hidden_states.size()
+
+        qkv_states = self.wqkv(hidden_states, im_mask)
+
+        qkv_states = rearrange(
+            qkv_states,
+            'b q (h gs d) -> b q h gs d',
+            gs=2 + self.num_key_value_groups,
+            d=self.head_dim,
+            q=q_len,
+        )
+
+        query_states = qkv_states[..., :self.num_key_value_groups, :]
+        query_states = rearrange(query_states, 'b q h gs d -> b q (h gs) d')
+        key_states = qkv_states[..., -2, :]
+        value_states = qkv_states[..., -1, :]
+        query_states = query_states.transpose(1, 2)
+        key_states = key_states.transpose(1, 2)
+        value_states = value_states.transpose(1, 2)
+
+        kv_seq_len = key_states.shape[-2]
+        if past_key_value is not None:
+            kv_seq_len += past_key_value[0].shape[-2]
+
+        cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+
+        query_states, key_states = apply_rotary_pos_emb(
+            query_states, key_states, cos, sin, position_ids)
+
+        if past_key_value is not None:
+            # reuse k, v, self_attention
+            key_states = torch.cat([past_key_value[0], key_states], dim=2)
+            value_states = torch.cat([past_key_value[1], value_states], dim=2)
+
+        past_key_value = (key_states, value_states) if use_cache else None
+
+        query_states = query_states.transpose(1, 2)
+        key_states = key_states.transpose(1, 2)
+        value_states = value_states.transpose(1, 2)
+
+        attn_output = self._flash_attention_forward(
+            query_states,
+            key_states,
+            value_states,
+            attention_mask,
+            q_len)
+
+        attn_output = attn_output.reshape(bsz, q_len,
+                                          self.hidden_size).contiguous()
+        attn_output = self.wo(attn_output, im_mask)
+
+        if not output_attentions:
+            attn_weights = None
+
+        return attn_output, attn_weights, past_key_value
+
+    def _flash_attention_forward(
+        self, query_states, key_states, value_states, attention_mask, query_length, dropout=0.0, softmax_scale=None
+    ):
+        """
+        Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token
+        first unpad the input, then computes the attention scores and pad the final attention scores.
+        Args:
+            query_states (`torch.Tensor`):
+                Input query states to be passed to Flash Attention API
+            key_states (`torch.Tensor`):
+                Input key states to be passed to Flash Attention API
+            value_states (`torch.Tensor`):
+                Input value states to be passed to Flash Attention API
+            attention_mask (`torch.Tensor`):
+                The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the
+                position of padding tokens and 1 for the position of non-padding tokens.
+            dropout (`int`, *optional*):
+                Attention dropout
+            softmax_scale (`float`, *optional*):
+                The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim)
+        """
+        # Contains at least one padding token in the sequence
+        causal = self.is_causal and query_length != 1
+        if attention_mask is not None:
+            batch_size = query_states.shape[0]
+            query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._unpad_input(
+                query_states, key_states, value_states, attention_mask, query_length
+            )
+
+            cu_seqlens_q, cu_seqlens_k = cu_seq_lens
+            max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens
+
+            attn_output_unpad = flash_attn_varlen_func(
+                query_states,
+                key_states,
+                value_states,
+                cu_seqlens_q=cu_seqlens_q,
+                cu_seqlens_k=cu_seqlens_k,
+                max_seqlen_q=max_seqlen_in_batch_q,
+                max_seqlen_k=max_seqlen_in_batch_k,
+                dropout_p=dropout,
+                softmax_scale=softmax_scale,
+                causal=causal,
+            )
+
+            attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length)
+        else:
+            attn_output = flash_attn_func(
+                query_states, key_states, value_states, dropout, softmax_scale=softmax_scale, causal=causal
+            )
+
+        return attn_output
+
+    def _unpad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length):
+        indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask)
+        batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape
+
+        key_layer = index_first_axis(
+            key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k
+        )
+        value_layer = index_first_axis(
+            value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k
+        )
+
+        if query_length == kv_seq_len:
+            query_layer = index_first_axis(
+                query_layer.reshape(batch_size * kv_seq_len, self.num_heads, head_dim), indices_k
+            )
+            cu_seqlens_q = cu_seqlens_k
+            max_seqlen_in_batch_q = max_seqlen_in_batch_k
+            indices_q = indices_k
+        elif query_length == 1:
+            max_seqlen_in_batch_q = 1
+            cu_seqlens_q = torch.arange(
+                batch_size + 1, dtype=torch.int32, device=query_layer.device
+            )  # There is a memcpy here, that is very bad.
+            indices_q = cu_seqlens_q[:-1]
+            query_layer = query_layer.squeeze(1)
+        else:
+            # The -q_len: slice assumes left padding.
+            attention_mask = attention_mask[:, -query_length:]
+            query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask)
+
+        return (
+            query_layer,
+            key_layer,
+            value_layer,
+            indices_q.to(torch.int64),
+            (cu_seqlens_q, cu_seqlens_k),
+            (max_seqlen_in_batch_q, max_seqlen_in_batch_k),
+        )
+
+class InternLM2DecoderLayer(nn.Module):
+
+    def __init__(self, config: InternLM2Config):
+        super().__init__()
+        self.hidden_size = config.hidden_size
+        self.attention = (
+            InternLM2Attention(config=config)
+            if not getattr(config, 'attn_implementation')=="flash_attention_2" else
+            InternLM2FlashAttention2(config=config))
+        self.feed_forward = InternLM2MLP(config)
+        self.attention_norm = InternLM2RMSNorm(
+            config.hidden_size, eps=config.rms_norm_eps)
+        self.ffn_norm = InternLM2RMSNorm(
+            config.hidden_size, eps=config.rms_norm_eps)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Tuple[torch.Tensor]] = None,
+        output_attentions: Optional[bool] = False,
+        use_cache: Optional[bool] = False,
+        im_mask: Optional[Tuple[torch.Tensor]] = None,
+        **kwargs,
+    ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor,
+                                                 torch.FloatTensor]]]:
+        """
+        Args:
+            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
+            attention_mask (`torch.FloatTensor`, *optional*):
+                attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,
+                query_sequence_length, key_sequence_length)` if default attention is used.
+            output_attentions (`bool`, *optional*):
+                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+                returned tensors for more detail.
+            use_cache (`bool`, *optional*):
+                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
+                (see `past_key_values`).
+            past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
+        """
+        if 'padding_mask' in kwargs:
+            warnings.warn(
+                'Passing `padding_mask` is deprecated and will be removed in v4.37. '
+                'Please make sure use `attention_mask` instead.`')
+
+        residual = hidden_states
+
+        hidden_states = self.attention_norm(hidden_states)
+
+        # Self Attention
+        hidden_states, self_attn_weights, present_key_value = self.attention(
+            hidden_states=hidden_states,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_value=past_key_value,
+            output_attentions=output_attentions,
+            use_cache=use_cache,
+            im_mask=im_mask,
+            **kwargs,
+        )
+        hidden_states = residual + hidden_states
+
+        # Fully Connected
+        residual = hidden_states
+        hidden_states = self.ffn_norm(hidden_states)
+        hidden_states = self.feed_forward(hidden_states, im_mask)
+        hidden_states = residual + hidden_states
+
+        outputs = (hidden_states, )
+
+        if output_attentions:
+            outputs += (self_attn_weights, )
+
+        if use_cache:
+            outputs += (present_key_value, )
+
+        return outputs
+
+
+InternLM2_START_DOCSTRING = r"""
+    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+    etc.)
+
+    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+    and behavior.
+
+    Parameters:
+        config ([`InternLM2Config`]):
+            Model configuration class with all the parameters of the model. Initializing with a config file does not
+            load the weights associated with the model, only the configuration. Check out the
+            [`~PreTrainedModel.from_pretrained`] method to load the model weights.
+"""
+
+
+@add_start_docstrings(
+    'The bare InternLM2 Model outputting raw hidden-states without any specific head on top.',
+    InternLM2_START_DOCSTRING,
+)
+class InternLM2PreTrainedModel(PreTrainedModel):
+    config_class = InternLM2Config
+    base_model_prefix = 'model'
+    supports_gradient_checkpointing = True
+    _no_split_modules = ['InternLM2DecoderLayer']
+    _skip_keys_device_placement = 'past_key_values'
+
+    def _init_weights(self, module):
+        std = self.config.initializer_range
+        if isinstance(module, nn.Linear):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.padding_idx is not None:
+                module.weight.data[module.padding_idx].zero_()
+
+
+InternLM2_INPUTS_DOCSTRING = r"""
+    Args:
+        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
+            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
+            it.
+
+            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+            [`PreTrainedTokenizer.__call__`] for details.
+
+            [What are input IDs?](../glossary#input-ids)
+        attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+
+            - 1 for tokens that are **not masked**,
+            - 0 for tokens that are **masked**.
+
+            [What are attention masks?](../glossary#attention-mask)
+
+            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+            [`PreTrainedTokenizer.__call__`] for details.
+
+            If `past_key_values` is used, optionally only the last `input_ids` have to be input (see
+            `past_key_values`).
+
+            If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
+            and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
+            information on the default strategy.
+
+            - 1 indicates the head is **not masked**,
+            - 0 indicates the head is **masked**.
+        position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+            config.n_positions - 1]`.
+
+            [What are position IDs?](../glossary#position-ids)
+        past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or
+            when `config.use_cache=True`):
+            Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+            `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
+            `(batch_size, num_heads, decoder_sequence_length, embed_size_per_head)`.
+
+            Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
+            blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
+
+            If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't
+            have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`
+            of shape `(batch_size, sequence_length)`.
+        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
+            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+            model's internal embedding lookup matrix.
+        use_cache (`bool`, *optional*):
+            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+            `past_key_values`).
+        output_attentions (`bool`, *optional*):
+            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+            tensors for more detail.
+        output_hidden_states (`bool`, *optional*):
+            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+            more detail.
+        return_dict (`bool`, *optional*):
+            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+
+@add_start_docstrings(
+    'The bare InternLM2 Model outputting raw hidden-states without any specific head on top.',
+    InternLM2_START_DOCSTRING,
+)
+class InternLM2Model(InternLM2PreTrainedModel):
+    """Transformer decoder consisting of *config.num_hidden_layers* layers.
+    Each layer is a [`InternLM2DecoderLayer`]
+
+    Args:
+        config: InternLM2Config
+    """
+
+    _auto_class = 'AutoModel'
+
+    def __init__(self, config: InternLM2Config):
+        super().__init__(config)
+        self.padding_idx = config.pad_token_id
+        self.vocab_size = config.vocab_size
+        self.config = config
+
+        self.tok_embeddings = nn.Embedding(config.vocab_size,
+                                           config.hidden_size,
+                                           self.padding_idx)
+        self.layers = nn.ModuleList([
+            InternLM2DecoderLayer(config)
+            for _ in range(config.num_hidden_layers)
+        ])
+        self.norm = InternLM2RMSNorm(
+            config.hidden_size, eps=config.rms_norm_eps)
+
+        self.gradient_checkpointing = False
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.tok_embeddings
+
+    def set_input_embeddings(self, value):
+        self.tok_embeddings = value
+
+    # Copied from transformers.models.bart.modeling_bart.BartDecoder._prepare_decoder_attention_mask
+    def _prepare_decoder_attention_mask(self, attention_mask, input_shape,
+                                        inputs_embeds, past_key_values_length):
+        # create causal mask
+        # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
+        combined_attention_mask = None
+        if input_shape[-1] > 1:
+            combined_attention_mask = _make_causal_mask(
+                input_shape,
+                inputs_embeds.dtype,
+                device=inputs_embeds.device,
+                past_key_values_length=past_key_values_length,
+            )
+
+        if attention_mask is not None:
+            # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
+            expanded_attn_mask = _expand_mask(
+                attention_mask, inputs_embeds.dtype,
+                tgt_len=input_shape[-1]).to(inputs_embeds.device)
+            combined_attention_mask = (
+                expanded_attn_mask if combined_attention_mask is None else
+                expanded_attn_mask + combined_attention_mask)
+
+        return combined_attention_mask
+
+    @add_start_docstrings_to_model_forward(InternLM2_INPUTS_DOCSTRING)
+    def forward(self,
+                input_ids: torch.LongTensor = None,
+                attention_mask: Optional[torch.Tensor] = None,
+                position_ids: Optional[torch.LongTensor] = None,
+                past_key_values: Optional[List[torch.FloatTensor]] = None,
+                inputs_embeds: Optional[torch.FloatTensor] = None,
+                use_cache: Optional[bool] = None,
+                output_attentions: Optional[bool] = None,
+                output_hidden_states: Optional[bool] = None,
+                return_dict: Optional[bool] = None,
+                **kwargs) -> Union[Tuple, BaseModelOutputWithPast]:
+
+        im_mask = kwargs.get('im_mask', None)
+
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else
+            self.config.output_hidden_states)
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        if self.config.attn_implementation: _import_flash_attn()
+
+        # retrieve input_ids and inputs_embeds
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError(
+                'You cannot specify both input_ids and inputs_embeds at the same time'
+            )
+        elif input_ids is not None:
+            batch_size, seq_length = input_ids.shape[:2]
+        elif inputs_embeds is not None:
+            batch_size, seq_length = inputs_embeds.shape[:2]
+        else:
+            raise ValueError(
+                'You have to specify either input_ids or inputs_embeds')
+
+        seq_length_with_past = seq_length
+        past_key_values_length = 0
+        if past_key_values is not None:
+            past_key_values_length = past_key_values[0][0].shape[2]
+            seq_length_with_past = seq_length_with_past + past_key_values_length
+
+        if position_ids is None:
+            device = input_ids.device if input_ids is not None else inputs_embeds.device
+            position_ids = torch.arange(
+                past_key_values_length,
+                seq_length + past_key_values_length,
+                dtype=torch.long,
+                device=device)
+            position_ids = position_ids.unsqueeze(0)
+
+        if inputs_embeds is None:
+            inputs_embeds = self.tok_embeddings(input_ids)
+            im_mask = torch.zeros(inputs_embeds.shape[:2]).to(
+                inputs_embeds.device).bool()
+        if self.config.attn_implementation == "flash_attention_2":
+            # 2d mask is passed through the layers
+            attention_mask = attention_mask if (attention_mask is not None and 0 in attention_mask) else None
+        else:
+            if attention_mask is None:
+                attention_mask = torch.ones(
+                    (batch_size, seq_length_with_past), dtype=torch.bool, device=inputs_embeds.device
+                )
+            attention_mask = self._prepare_decoder_attention_mask(
+                attention_mask, (batch_size, seq_length), inputs_embeds, past_key_values_length
+            )
+        # embed positions
+        # if attention_mask is None:
+        #     attention_mask = torch.ones((batch_size, seq_length_with_past),
+        #                                 dtype=torch.bool,
+        #                                 device=inputs_embeds.device)
+        # attention_mask = self._prepare_decoder_attention_mask(
+        #     attention_mask, (batch_size, seq_length), inputs_embeds,
+        #     past_key_values_length)
+
+        # embed positions
+        hidden_states = inputs_embeds
+
+        if self.gradient_checkpointing and self.training:
+            if use_cache:
+                logger.warning_once(
+                    '`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...'
+                )
+                use_cache = False
+
+        # decoder layers
+        all_hidden_states = () if output_hidden_states else None
+        all_self_attns = () if output_attentions else None
+        next_decoder_cache = () if use_cache else None
+
+        for idx, decoder_layer in enumerate(self.layers):
+            if output_hidden_states:
+                all_hidden_states += (hidden_states, )
+
+            past_key_value = past_key_values[
+                idx] if past_key_values is not None else None
+
+            if self.gradient_checkpointing and self.training:
+
+                def create_custom_forward(module):
+
+                    def custom_forward(*inputs):
+                        # None for past_key_value
+                        return module(*inputs, output_attentions, None,
+                                      im_mask)
+
+                    return custom_forward
+
+                layer_outputs = torch.utils.checkpoint.checkpoint(
+                    create_custom_forward(decoder_layer),
+                    hidden_states,
+                    attention_mask,
+                    position_ids,
+                    None,
+                )
+            else:
+                layer_outputs = decoder_layer(
+                    hidden_states,
+                    attention_mask=attention_mask,
+                    position_ids=position_ids,
+                    past_key_value=past_key_value,
+                    output_attentions=output_attentions,
+                    use_cache=use_cache,
+                    im_mask=im_mask,
+                )
+
+            hidden_states = layer_outputs[0]
+
+            if use_cache:
+                next_decoder_cache += (
+                    layer_outputs[2 if output_attentions else 1], )
+
+            if output_attentions:
+                all_self_attns += (layer_outputs[1], )
+
+        hidden_states = self.norm(hidden_states)
+
+        # add hidden states from the last decoder layer
+        if output_hidden_states:
+            all_hidden_states += (hidden_states, )
+
+        next_cache = next_decoder_cache if use_cache else None
+        if not return_dict:
+            return tuple(
+                v for v in
+                [hidden_states, next_cache, all_hidden_states, all_self_attns]
+                if v is not None)
+        return BaseModelOutputWithPast(
+            last_hidden_state=hidden_states,
+            past_key_values=next_cache,
+            hidden_states=all_hidden_states,
+            attentions=all_self_attns,
+        )

meteor/arch/modeling_meteor.py

@@ -0,0 +1,286 @@
+# System
+import torch
+from torch import nn
+from utils.utils import *
+import torch.utils.checkpoint
+from transformers.cache_utils import Cache
+from typing import List, Optional, Tuple, Union
+from .build_module import build_vision_projector, build_vision_tower
+from .modeling_internlm2 import InternLM2Model, InternLM2PreTrainedModel
+
+# Dataclass & ModelOutput
+from dataclasses import dataclass
+from transformers.modeling_outputs import ModelOutput
+@dataclass
+class MeteorCausalLMOutputWithPast(ModelOutput):
+    loss: Optional[torch.FloatTensor] = None
+    logits: torch.FloatTensor = None
+    past_key_values: Optional[List[torch.FloatTensor]] = None
+    tor_features: Optional[List[torch.FloatTensor]] = None
+    hidden_states: Optional[Tuple[torch.FloatTensor]] = None
+    attentions: Optional[Tuple[torch.FloatTensor]] = None
+    image_hidden_states: Optional[Tuple[torch.FloatTensor]] = None
+
+class MeteorForCausalLM(InternLM2PreTrainedModel):
+    _auto_class = 'AutoModelForCausalLM'
+
+    _tied_weights_keys = ['output.weight']
+
+    def __init__(self, config):
+        super().__init__(config)
+        
+        # Model
+        self.model = InternLM2Model(config)
+        self.vocab_size = config.vocab_size
+        self.output = nn.Linear(config.hidden_size, config.vocab_size-2, bias=False)
+        self.max_length = config.max_length
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+        # Vision Encoder
+        self.vit = build_vision_tower()
+
+        # Vision Projection
+        self.vision_proj = build_vision_projector()
+
+    def eval_process(
+        self,
+        inputs,
+        data,
+        tokenizer,
+        device,
+        img_token_number,
+    ):
+        
+        batched_qa_prompt=[]
+        for _input in inputs:
+
+            # Visualization
+            # imim = _input['image'].cpu().permute(1, 2, 0)
+
+            # make question, rationale, and answer
+            question = make_instruction_for_eval_meteor(_input['question'], data)
+
+            # add bundle image tokens if it has <image> token
+            question = add_bundle_tokens(question, '<image>', img_token_number) 
+
+            batched_qa_prompt.append(question)
+
+        '''For Final Outputs'''
+        qa_prompts = tokenizer(batched_qa_prompt, padding='longest', return_tensors="pt", add_special_tokens=False)
+
+        # [1] input_ids
+        input_ids = qa_prompts.input_ids.to(device)
+  
+        # [2] attention_mask
+        attention_mask = qa_prompts.attention_mask.to(device)
+
+        # [3] im_mask
+        im_mask = torch.zeros_like(input_ids).bool()
+        im_mask[torch.where(input_ids==self.config.image_token_index)] = True
+
+        return {"input_ids": input_ids, 
+                "attention_mask": attention_mask, 
+                "im_mask": im_mask,
+                }
+
+    def clip_features(self, image):
+        self.vit.eval()
+        return self.vit(image)
+
+    def _merge_input_embeds_with_tor_features(self, tor_features, inputs_embeds, input_ids):
+        
+        # batch index for image feature
+        batch_ind_tor_feature = 0
+
+        for ind, input_id in enumerate(input_ids):
+            matching = torch.where(input_id==self.config.tor_token_index)
+            num_tor_tokens_per_one_sample = len(matching[0])
+            inputs_embeds[ind][matching] = tor_features[batch_ind_tor_feature: batch_ind_tor_feature+num_tor_tokens_per_one_sample].to(inputs_embeds.dtype)
+            batch_ind_tor_feature += num_tor_tokens_per_one_sample
+
+    def _merge_input_embeds_with_image_features(self, image_features, inputs_embeds, input_ids):
+        
+        # batch index for image feature
+        batch_ind_image_feature = 0
+
+        # shape of image_features
+        _, C, D = image_features.shape
+
+        for ind, input_id in enumerate(input_ids):
+            matching = torch.where(input_id==self.config.image_token_index)
+            num_image_tokens_per_one_sample = len(matching[0]) // C
+            inputs_embeds[ind][matching] = image_features[batch_ind_image_feature: batch_ind_image_feature+num_image_tokens_per_one_sample].view(-1, D)
+            batch_ind_image_feature += num_image_tokens_per_one_sample
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        image_features: torch.FloatTensor = None,
+        tor_features: torch.FloatTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        im_mask: torch.BoolTensor = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, MeteorCausalLMOutputWithPast]:
+
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        if inputs_embeds is None:
+            # 1. Extra the input embeddings
+            inputs_embeds = self.get_input_embeddings()(input_ids)
+
+            # 2. Merge text and images
+            if image_features is not None and input_ids.shape[1] != 1:
+                image_features = self.vision_proj(image_features.to(inputs_embeds.dtype))
+                self._merge_input_embeds_with_image_features(image_features, inputs_embeds, input_ids)
+
+            # 3. Merge text and <tor>
+            if tor_features is not None and input_ids.shape[1] != 1:
+                self._merge_input_embeds_with_tor_features(tor_features, inputs_embeds, input_ids)
+
+            # In case input_ids.shape[1] == 1 & image_features==None & past_key_values != None, we are in the case of
+            # generation with cache
+            elif past_key_values is not None and image_features is not None and input_ids.shape[1] == 1:
+                # Retrieve the first layer to inspect the logits and mask out the hidden states
+                # that are set to 0
+                first_layer_past_key_value = past_key_values[0][0][:, :, :, 0]
+
+                # Sum all dimensions of head_dim (-2) to avoid random errors such as: https://github.com/huggingface/transformers/pull/28032#issuecomment-1863691941
+                batch_index, non_attended_tokens = torch.where(first_layer_past_key_value.float().sum(-2) == 0)
+
+                # Get the target length
+                target_length = input_ids.shape[1]
+                past_length = first_layer_past_key_value.shape[-1]
+
+                extended_attention_mask = torch.ones(
+                    (attention_mask.shape[0], past_length),
+                    dtype=attention_mask.dtype,
+                    device=attention_mask.device,
+                )
+
+                # Filter out only the tokens that can be un-attended, this can happen
+                # if one uses Llava + Fused modules where the cache on the
+                # first iteration is already big enough, or if one passes custom cache
+                valid_indices = non_attended_tokens < extended_attention_mask.size(-1)
+                new_batch_index = batch_index[valid_indices]
+                new_non_attended_tokens = non_attended_tokens[valid_indices]
+
+                # Zero-out the places where we don't need to attend
+                extended_attention_mask[new_batch_index, new_non_attended_tokens] = 0
+
+                attention_mask = torch.cat((extended_attention_mask, attention_mask[:, -target_length:]), dim=1)
+                position_ids = torch.sum(attention_mask, dim=1).unsqueeze(-1) - 1
+                im_mask = torch.zeros(inputs_embeds.shape[:2]).bool().to(inputs_embeds.device)
+
+        outputs = self.model(
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+            im_mask=im_mask,
+        )
+
+        hidden_states = outputs[0]
+        logits = self.output(hidden_states)
+
+        loss = None
+        if labels is not None:
+            # Shift so that tokens < n predict n
+            if attention_mask is not None:
+                shift_attention_mask = attention_mask[..., 1:]
+                shift_logits = logits[..., :-1, :][shift_attention_mask.to(logits.device) != 0].contiguous()
+                shift_labels = labels[..., 1:][shift_attention_mask.to(labels.device) != 0].contiguous()
+            else:
+                shift_logits = logits[..., :-1, :].contiguous()
+                shift_labels = labels[..., 1:].contiguous()
+            # Flatten the tokens
+            loss_fct = nn.CrossEntropyLoss()
+            loss = loss_fct(
+                shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1).to(shift_logits.device)
+            )
+
+        if not return_dict:
+            output = (logits,) + outputs[1:]
+            return (loss,) + output if loss is not None else output
+
+        return MeteorCausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            tor_features=hidden_states[torch.where(input_ids==self.config.tor_token_index)],
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+    
+    def prepare_inputs_for_generation(self,
+                                      input_ids,
+                                      past_key_values=None,
+                                      attention_mask=None,
+                                      inputs_embeds=None,
+                                      image_features=None,
+                                      tor_features=None, 
+                                      im_mask=None,
+                                      **kwargs):
+        if past_key_values is not None:
+            past_length = past_key_values[0][0].shape[2]
+
+            # Some generation methods already pass only the last input ID
+            if input_ids.shape[1] > past_length:
+                remove_prefix_length = past_length
+            else:
+                # Default to old behavior: keep only final ID
+                remove_prefix_length = input_ids.shape[1] - 1
+
+            input_ids = input_ids[:, remove_prefix_length:]
+
+        position_ids = kwargs.get('position_ids', None)
+        if attention_mask is not None and position_ids is None:
+            # create position_ids on the fly for batch generation
+            position_ids = attention_mask.long().cumsum(-1) - 1
+            position_ids.masked_fill_(attention_mask == 0, 1)
+            if past_key_values:
+                position_ids = position_ids[:, -input_ids.shape[1]:]
+
+        # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
+        if inputs_embeds is not None and past_key_values is None:
+            model_inputs = {"inputs_embeds": inputs_embeds}
+        else:
+            model_inputs = {"input_ids": input_ids}
+
+        model_inputs.update(
+            {
+                "position_ids": position_ids,
+                "past_key_values": past_key_values,
+                "use_cache": kwargs.get("use_cache"),
+                "attention_mask": attention_mask,
+                "image_features": image_features,
+                "tor_features": tor_features,
+                "im_mask": im_mask,
+            }
+        )
+        return model_inputs
+    
+    @staticmethod
+    def _reorder_cache(past_key_values, beam_idx):
+        reordered_past = ()
+        for layer_past in past_key_values:
+            reordered_past += (tuple(
+                past_state.index_select(0, beam_idx.to(past_state.device))
+                for past_state in layer_past), )
+        return reordered_past

meteor/arch/modeling_mmamba.py

@@ -0,0 +1,214 @@
+# Transformers
+import re
+import torch
+from torch import nn
+from utils.utils import *
+from typing import Optional, Tuple, Union
+from transformers import MambaForCausalLM
+from transformers import LlavaNextForConditionalGeneration, LlavaForConditionalGeneration
+
+class MambaCache:
+    def __init__(self, config, batch_size, dtype=torch.float16, device=None):
+        self.seqlen_offset = 0
+        self.dtype = dtype
+        intermediate_size = config.intermediate_size
+        ssm_state_size = config.state_size
+        conv_kernel_size = config.conv_kernel
+
+        self.conv_states = {
+            i: torch.zeros(batch_size, intermediate_size, conv_kernel_size, device=device, dtype=dtype)
+            for i in range(config.num_hidden_layers)
+        }
+        self.ssm_states = {
+            i: torch.zeros(batch_size, intermediate_size, ssm_state_size, device=device, dtype=dtype)
+            for i in range(config.num_hidden_layers)
+        }
+
+# Dataclass & ModelOutput
+from dataclasses import dataclass
+from transformers.modeling_outputs import ModelOutput
+@dataclass
+class MambaCausalLMOutput(ModelOutput):
+    loss: Optional[torch.FloatTensor] = None
+    cache_params: Optional[MambaCache] = None
+    tor_features: Optional[torch.FloatTensor] = None
+    hidden_states: Optional[Tuple[torch.FloatTensor]] = None
+
+class MeteorMambaForCausalLM(MambaForCausalLM):
+    def __init__(self, config):
+        super().__init__(config)
+
+        # initialize other projections for Vision and tor
+        self.vision_proj = self.build_vision_projector(1024, self.config.hidden_size)
+        self.tor_proj = self.build_vision_projector(self.config.hidden_size, 4096)
+        
+        # replacing embedding size of mamba with that of meteor
+        self.backbone.embeddings = nn.Embedding(num_embeddings=92546,
+                                                embedding_dim=self.config.hidden_size)
+
+        # image processing variable
+        self.mean = torch.tensor([0.48145466, 0.4578275, 0.40821073]).view(1,-1,1,1) * 255
+        self.std = torch.tensor([0.26862954, 0.26130258, 0.27577711]).view(1,-1,1,1) * 255
+
+    def image_processor(self, images):
+        norm_images = (images - self.mean.to(images.device)) / self.std.to(images.device)
+        return norm_images
+
+    @staticmethod
+    def build_vision_projector(mm_hidden_size, hidden_size):
+        projector_type = 'mlp2x_gelu'
+        mlp_gelu_match = re.match(r'^mlp(\d+)x_gelu$', projector_type)
+        if mlp_gelu_match:
+            mlp_depth = int(mlp_gelu_match.group(1))
+            modules = [nn.Linear(mm_hidden_size, hidden_size)]
+            for _ in range(1, mlp_depth):
+                modules.append(nn.GELU())
+                modules.append(nn.Linear(hidden_size, hidden_size))
+            return nn.Sequential(*modules)
+
+        raise ValueError(f'Unknown projector type: {projector_type}')
+
+    def eval_process(
+        self,
+        inputs,
+        tokenizer,
+        device,
+        img_token_number,
+    ):
+        batched_image=[]
+        batched_qa_prompt=[]
+        for _input in inputs:
+
+            # Visualization
+            # imim = _input['image'].cpu().permute(1, 2, 0)
+
+            # adding <image> to question if not included despite being an image, and adding system prompt and <tor> prompt 
+            if 'image' in _input.keys() and not '<image>' in _input['question']: _input['question'] = '<image>\n' + _input['question']
+
+            # make question, rationale, and answer
+            question = make_instruction_for_mmamba(question=_input['question'])
+
+            # add bundle image tokens if it has <image> token
+            question = add_bundle_tokens(question, '<image>', img_token_number) 
+
+            # making batched moai prompt
+            if 'image' in _input.keys() and _input['image'] != None: batched_image.append(_input['image'].to(device))
+            batched_qa_prompt.append(question)
+
+        '''For Final Outputs'''
+        qa_prompts = tokenizer(batched_qa_prompt, padding='longest', return_tensors="pt", add_special_tokens=False)
+
+        # [1] input_ids
+        input_ids = qa_prompts.input_ids.to(device)
+
+        # image or only text?
+        if len(batched_image):
+            # [2] pixel values
+            try:
+                pixel_values = self.image_processor(torch.stack(batched_image)).to(device)
+                assert pixel_values.dim() == 4
+            except:
+                new_batched_image = []
+                for batched_image_element in batched_image:
+                    if batched_image_element.dim() == 3:
+                        new_batched_image.append(batched_image_element.unsqueeze(0))
+                    else:
+                        new_batched_image.append(batched_image_element)
+                pixel_values = self.image_processor(torch.cat(new_batched_image, dim=0)).to(device)
+
+            return {"input_ids": input_ids, "image": pixel_values}
+        else:
+            return {"input_ids": input_ids}
+
+
+    def _merge_input_embeds_with_image_features(self, image_features, inputs_embeds, input_ids):
+        
+        # batch index for image feature
+        batch_ind_image_feature = 0
+
+        # shape of image_features
+        _, C, D = image_features.shape
+
+        for ind, input_id in enumerate(input_ids):
+            matching = torch.where(input_id==self.config.image_token_index)
+            num_image_tokens_per_one_sample = len(matching[0]) // C
+            inputs_embeds[ind][matching] = image_features[batch_ind_image_feature: batch_ind_image_feature+num_image_tokens_per_one_sample].view(-1, D)
+            batch_ind_image_feature += num_image_tokens_per_one_sample
+
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        image_features: Optional[torch.FloatTensor] = None,
+        cache_params: Optional[MambaCache] = None,
+        # labels: Optional[torch.LongTensor] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        use_cache: Optional[bool] = None,
+        **kwargs,  # for now we need this for generation
+    ) -> Union[Tuple, MambaCausalLMOutput]:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
+            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
+            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
+        """
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+
+        if inputs_embeds is None:
+            # 1. Extra the input embeddings
+            inputs_embeds = self.get_input_embeddings()(input_ids)
+
+            # 2. Merge text and images
+            if image_features is not None and input_ids.shape[1] != 1:
+                image_features = self.vision_proj(image_features)
+                self._merge_input_embeds_with_image_features(image_features, inputs_embeds, input_ids)
+
+        mamba_outputs = self.backbone(
+            cache_params=cache_params,
+            inputs_embeds=inputs_embeds,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+            use_cache=use_cache,
+        )
+        hidden_states = mamba_outputs[0]
+
+        # logits = self.lm_head(hidden_states.to(self.lm_head.weight.dtype)).float()
+
+        loss = None
+        # if labels is not None:
+        #     # move labels to correct device to enable model parallelism
+        #     labels = labels.to(logits.device)
+        #     # Shift so that tokens < n predict n
+        #     shift_logits = logits[..., :-1, :].contiguous()
+        #     shift_labels = labels[..., 1:].contiguous()
+        #     # Flatten the tokens
+        #     loss_fct = nn.CrossEntropyLoss()
+        #     loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
+
+        # if not return_dict:
+        #     output = (logits,) + mamba_outputs[1:]
+        #     return ((loss,) + output) if loss is not None else output
+
+        return MambaCausalLMOutput(
+            loss=loss,
+            cache_params=mamba_outputs.cache_params,
+            tor_features=self.tor_proj(hidden_states[torch.where(input_ids==self.config.tor_token_index)]),
+            hidden_states=mamba_outputs.hidden_states,
+        )
+    
+    def prepare_inputs_for_generation(
+        self, input_ids, cache_params: Optional[MambaCache] = None, inputs_embeds=None, image_features=None, **kwargs
+    ):
+        # only last token for inputs_ids if the state is passed along.
+        if cache_params is not None:
+            input_ids = input_ids[:, -1].unsqueeze(-1)
+
+        if inputs_embeds is not None and cache_params is None:
+            model_inputs = {"inputs_embeds": inputs_embeds, "image_features":image_features}
+        else:
+            model_inputs = {"input_ids": input_ids, "image_features":image_features}
+
+        model_inputs["cache_params"] = cache_params
+        return model_inputs

meteor/arch/tokenization_internlm2.py

@@ -0,0 +1,252 @@
+# Copyright (c) InternLM. All rights reserved.
+#
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
+# and OPT implementations in this library. It has been modified from its
+# original forms to accommodate minor architectural differences compared
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tokenization classes for IntermLM."""
+import os
+from shutil import copyfile
+from typing import Any, Dict, List, Optional, Tuple
+
+import sentencepiece as spm
+from transformers.tokenization_utils import PreTrainedTokenizer
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+VOCAB_FILES_NAMES = {'vocab_file': './tokenizer.model'}
+
+PRETRAINED_VOCAB_FILES_MAP = {}
+
+
+class InternLM2Tokenizer(PreTrainedTokenizer):
+    """Construct a InternLM tokenizer. Based on byte-level Byte-Pair-Encoding.
+
+    Args:
+        vocab_file (`str`):
+            Path to the vocabulary file.
+    """
+
+    vocab_files_names = VOCAB_FILES_NAMES
+    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    model_input_names = ['input_ids', 'attention_mask']
+    _auto_class = 'AutoTokenizer'
+
+    def __init__(
+        self,
+        vocab_file,
+        unk_token='<unk>',
+        bos_token='<s>',
+        eos_token='</s>',
+        pad_token='</s>',
+        sp_model_kwargs: Optional[Dict[str, Any]] = None,
+        add_bos_token=True,
+        add_eos_token=False,
+        decode_with_prefix_space=False,
+        clean_up_tokenization_spaces=False,
+        **kwargs,
+    ):
+        self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
+        self.vocab_file = vocab_file
+        self.add_bos_token = add_bos_token
+        self.add_eos_token = add_eos_token
+        self.decode_with_prefix_space = decode_with_prefix_space
+        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
+        self.sp_model.Load(vocab_file)
+        self._no_prefix_space_tokens = None
+        super().__init__(
+            bos_token=bos_token,
+            eos_token=eos_token,
+            unk_token=unk_token,
+            pad_token=pad_token,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            **kwargs,
+        )
+        """ Initialization"""
+
+    @property
+    def no_prefix_space_tokens(self):
+        if self._no_prefix_space_tokens is None:
+            vocab = self.convert_ids_to_tokens(list(range(self.vocab_size)))
+            self._no_prefix_space_tokens = {
+                i
+                for i, tok in enumerate(vocab) if not tok.startswith('▁')
+            }
+        return self._no_prefix_space_tokens
+
+    @property
+    def vocab_size(self):
+        """Returns vocab size."""
+        return self.sp_model.get_piece_size()
+
+    @property
+    def bos_token_id(self) -> Optional[int]:
+        return self.sp_model.bos_id()
+
+    @property
+    def eos_token_id(self) -> Optional[int]:
+        return self.sp_model.eos_id()
+
+    def get_vocab(self):
+        """Returns vocab as a dict."""
+        vocab = {
+            self.convert_ids_to_tokens(i): i
+            for i in range(self.vocab_size)
+        }
+        vocab.update(self.added_tokens_encoder)
+        return vocab
+
+    def _tokenize(self, text):
+        """Returns a tokenized string."""
+        return self.sp_model.encode(text, out_type=str)
+
+    def _convert_token_to_id(self, token):
+        """Converts a token (str) in an id using the vocab."""
+        return self.sp_model.piece_to_id(token)
+
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        token = self.sp_model.IdToPiece(index)
+        return token
+
+    def _maybe_add_prefix_space(self, tokens, decoded):
+        if tokens and tokens[0] not in self.no_prefix_space_tokens:
+            return ' ' + decoded
+        else:
+            return decoded
+
+    def convert_tokens_to_string(self, tokens):
+        """Converts a sequence of tokens (string) in a single string."""
+        current_sub_tokens = []
+        out_string = ''
+        prev_is_special = False
+        for token in tokens:
+            # make sure that special tokens are not decoded using sentencepiece model
+            if token in self.all_special_tokens:
+                if not prev_is_special:
+                    out_string += ' '
+                out_string += self.sp_model.decode(current_sub_tokens) + token
+                prev_is_special = True
+                current_sub_tokens = []
+            else:
+                current_sub_tokens.append(token)
+                prev_is_special = False
+        out_string += self.sp_model.decode(current_sub_tokens)
+        out_string = self.clean_up_tokenization(out_string)
+        out_string = self._maybe_add_prefix_space(
+            tokens=tokens, decoded=out_string)
+        return out_string[1:]
+
+    def save_vocabulary(self,
+                        save_directory,
+                        filename_prefix: Optional[str] = None) -> Tuple[str]:
+        """Save the vocabulary and special tokens file to a directory.
+
+        Args:
+            save_directory (`str`):
+                The directory in which to save the vocabulary.
+
+        Returns:
+            `Tuple(str)`: Paths to the files saved.
+        """
+        if not os.path.isdir(save_directory):
+            logger.error(
+                f'Vocabulary path ({save_directory}) should be a directory')
+            return
+        out_vocab_file = os.path.join(
+            save_directory,
+            (filename_prefix + '-' if filename_prefix else '') +
+            VOCAB_FILES_NAMES['vocab_file'])
+
+        if os.path.abspath(self.vocab_file) != os.path.abspath(
+                out_vocab_file) and os.path.isfile(self.vocab_file):
+            copyfile(self.vocab_file, out_vocab_file)
+        elif not os.path.isfile(self.vocab_file):
+            with open(out_vocab_file, 'wb') as fi:
+                content_spiece_model = self.sp_model.serialized_model_proto()
+                fi.write(content_spiece_model)
+
+        return (out_vocab_file, )
+
+    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+        if self.add_bos_token:
+            bos_token_ids = [self.bos_token_id]
+        else:
+            bos_token_ids = []
+
+        output = bos_token_ids + token_ids_0
+
+        if token_ids_1 is not None:
+            output = output + token_ids_1
+
+        if self.add_eos_token:
+            output = output + [self.eos_token_id]
+
+        return output
+
+    def get_special_tokens_mask(
+            self,
+            token_ids_0: List[int],
+            token_ids_1: Optional[List[int]] = None,
+            already_has_special_tokens: bool = False) -> List[int]:
+        """Retrieve sequence ids from a token list that has no special tokens
+        added. This method is called when adding special tokens using the
+        tokenizer `prepare_for_model` method.
+
+        Args:
+            token_ids_0 (`List[int]`):
+                List of IDs.
+            token_ids_1 (`List[int]`, *optional*):
+                Optional second list of IDs for sequence pairs.
+            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
+                Whether or not the token list is already formatted with special tokens for the model.
+
+        Returns:
+            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
+        """
+        if already_has_special_tokens:
+            return super().get_special_tokens_mask(
+                token_ids_0=token_ids_0,
+                token_ids_1=token_ids_1,
+                already_has_special_tokens=True)
+
+        if token_ids_1 is None:
+            return [1] + ([0] * len(token_ids_0)) + [1]
+        return [1] + ([0] * len(token_ids_0)) + [1, 1] + (
+            [0] * len(token_ids_1)) + [1]
+
+    def create_token_type_ids_from_sequences(
+            self,
+            token_ids_0: List[int],
+            token_ids_1: Optional[List[int]] = None) -> List[int]:
+        """Create a mask from the two sequences passed to be used in a
+        sequence-pair classification task. T5 does not make use of token type
+        ids, therefore a list of zeros is returned.
+
+        Args:
+            token_ids_0 (`List[int]`):
+                List of IDs.
+            token_ids_1 (`List[int]`, *optional*):
+                Optional second list of IDs for sequence pairs.
+
+        Returns:
+            `List[int]`: List of zeros.
+        """
+        eos = [self.eos_token_id]
+
+        if token_ids_1 is None:
+            return len(token_ids_0 + eos) * [0]
+        return len(token_ids_0 + eos + token_ids_1 + eos) * [0]

meteor/load_meteor.py

@@ -0,0 +1,47 @@
+import torch
+import warnings
+from config import *
+from transformers import BitsAndBytesConfig
+from .arch.modeling_meteor import MeteorForCausalLM
+from .arch.tokenization_internlm2 import InternLM2Tokenizer
+warnings.filterwarnings(action='ignore')
+
+
+def load_meteor(link, bits):
+    
+    # huggingface model configuration
+    huggingface_config = {}
+
+    # Bit quantization
+    if bits in [4, 8]:
+        huggingface_config.update(dict(
+            torch_dtype=torch.float16,
+            low_cpu_mem_usage=True,
+            attn_implementation="flash_attention_2",
+            quantization_config=BitsAndBytesConfig(
+                load_in_4bit=bits == 4,
+                load_in_8bit=bits == 8,
+                llm_int8_skip_modules=["vit", "vision_proj", "output", "ffn"],
+                llm_int8_threshold=6.0,
+                llm_int8_has_fp16_weight=False,
+                bnb_4bit_compute_dtype=torch.float16,
+                bnb_4bit_use_double_quant=True,
+                bnb_4bit_quant_type='nf4'
+            )
+        ))
+    else:
+        huggingface_config.update(dict(
+            torch_dtype=torch.float16,
+            low_cpu_mem_usage=True,
+            attn_implementation="flash_attention_2",
+        ))
+
+    # loading backbone model
+    meteor = MeteorForCausalLM.from_pretrained(link, **huggingface_config)
+
+    # loading meteor tokenizer
+    # adding <image> and <tor> special token
+    tok_meteor = InternLM2Tokenizer.from_pretrained(link, padding_side='left')
+    tok_meteor.add_tokens("<image>", special_tokens=True)
+    tok_meteor.add_tokens("<tor>", special_tokens=True)
+    return meteor, tok_meteor

meteor/load_mmamba.py

@@ -0,0 +1,17 @@
+import torch
+from .arch.modeling_mmamba import MeteorMambaForCausalLM
+
+def load_mmamba(link):
+
+    # huggingface model configuration
+    huggingface_config = {}
+    huggingface_config.update(dict(
+        ignore_mismatched_sizes=True,
+        torch_dtype=torch.float32,
+        low_cpu_mem_usage=True,
+    ))
+
+    # Meteor Mamba Model (no fp32)
+    mmamba = MeteorMambaForCausalLM.from_pretrained(link, **huggingface_config)
+
+    return mmamba

requirements.txt

@@ -0,0 +1,19 @@
+torch==2.2.2
+flash-attn --no-build-isolation
+transformers
+bitsandbytes
+accelerate
+peft
+pandas
+pyarrow
+jsonlines
+wandb
+einops
+einops_exts
+sentencepiece
+causal-conv1d>=1.2.0
+mamba-ssm
+timm
+shortuuid
+matplotlib
+gradio

utils/ddp_accel_bf16.yaml

@@ -0,0 +1,16 @@
+compute_environment: LOCAL_MACHINE
+debug: false
+distributed_type: MULTI_GPU
+downcast_bf16: 'no'
+gpu_ids: all
+machine_rank: 0
+main_training_function: main
+mixed_precision: no
+num_machines: 1
+num_processes: 1
+rdzv_backend: static
+same_network: true
+tpu_env: []
+tpu_use_cluster: false
+tpu_use_sudo: false
+use_cpu: false

utils/ddp_accel_fp16.yaml

@@ -0,0 +1,16 @@
+compute_environment: LOCAL_MACHINE
+debug: false
+distributed_type: MULTI_GPU
+downcast_bf16: 'no'
+gpu_ids: all
+machine_rank: 0
+main_training_function: main
+mixed_precision: no
+num_machines: 1
+num_processes: 1
+rdzv_backend: static
+same_network: true
+tpu_env: []
+tpu_use_cluster: false
+tpu_use_sudo: false
+use_cpu: false

utils/ds_accel_fp16.yaml

@@ -0,0 +1,23 @@
+compute_environment: LOCAL_MACHINE
+debug: false
+deepspeed_config:
+  gradient_accumulation_steps: 1
+  offload_optimizer_device: none
+  offload_param_device: none
+  zero3_init_flag: false
+  zero3_save_16bit_model: false
+  zero_stage: 3
+distributed_type: DEEPSPEED
+downcast_bf16: 'no'
+enable_cpu_affinity: false
+machine_rank: 0
+main_training_function: main
+mixed_precision: fp16
+num_machines: 1
+num_processes: 1
+rdzv_backend: static
+same_network: true
+tpu_env: []
+tpu_use_cluster: false
+tpu_use_sudo: false
+use_cpu: false

utils/utils.py

@@ -0,0 +1,194 @@
+import os
+import gc
+import math
+import torch
+import base64
+import numpy as np
+from config import *
+import torch.nn.functional as F
+
+def memory_optimization():
+    # memory deallocation
+    gc.collect()
+
+    # removing cache
+    torch.cuda.empty_cache()
+
+def freeze_model(model):
+    for param in model.parameters():
+        param.requires_grad=False
+
+def find_special_token(string, special_token):
+    start = 0
+    while True:
+        start = string.find(special_token, start)
+        if start == -1: return
+        yield start
+        start += len(special_token) # use start += 1 to find overlapping matches
+
+def insert_tor(sentence, tor_count):
+    words = sentence.split()
+    gap =  len(words) // (tor_count-1)
+
+    # filtering
+    if 0<=gap<=2:
+        return False
+
+    count = 0
+    result = ""
+    for i, word in enumerate(words):
+        if 0<i<len(words)-1:
+            result+=' '
+        if i % gap == 0 and count != tor_count-1:
+            result += '<tor>'
+            count += 1
+        result += word
+    result = result + "<tor>"
+    assert len(list(find_special_token(result, '<tor>'))) == tor_count
+    return result
+
+def add_bundle_tokens(input_string, special_token, num):
+
+    # number of special tokens in input_string
+    num_special_tokens = len(list(find_special_token(input_string, special_token)))
+
+    # No special token -> return the raw
+    if not num_special_tokens:
+        return input_string
+    
+    result = ""
+    index = 0
+    while index < len(input_string):
+        if input_string[index:index + len(special_token)] == special_token:
+            result += special_token * num
+            index += len(special_token)
+        else:
+            result += input_string[index]
+            index += 1
+
+    assert len(list(find_special_token(result, special_token))) == num_special_tokens * num
+    return result
+
+def make_instruction_for_mmamba(question, tor=None):
+    
+    if tor:
+        qa_prompt = make_human_string(f"<s>[UNUSED_TOKEN_146]user\n{question}[UNUSED_TOKEN_145]",
+                                    f"[UNUSED_TOKEN_146]rationale\n{tor}[UNUSED_TOKEN_145]\n</s>",
+                                    split='\n')
+    else:
+        qa_prompt = make_human_string(f"<s>[UNUSED_TOKEN_146]user\n{question}[UNUSED_TOKEN_145]",
+                                    f"[UNUSED_TOKEN_146]rationale\n"+"<tor>"*10+"[UNUSED_TOKEN_145]\n</s>",
+                                    split='\n')
+    return qa_prompt
+
+def make_instruction_for_eval_meteor(question, dataset):
+    system_prompt = "You should give helpful answer to user based on the rationale."
+    
+    if dataset != "mmmu" and dataset != "mathverse" and dataset != "hallusionbench" and dataset != "demo":
+        question = "<image>" + question
+
+    if dataset in ["sqa", "mmbench", "mmbench_cn", "mmbench_dev", "mmbench_cn_dev", "seed", "qbench", "ai2d", "mmstar"]:
+        question = question + "\nAnswer with the option's letter from the given choices directly."
+
+    elif dataset in ["vqav2", "gqa", "pope", "chartqa"]:
+        question = question + "\nAnswer the question using a single word or phrase."
+
+    elif dataset in ["vizwiz"]:
+        question = question + "\nWhen the provided information is insufficient, respond with 'Unanswerable'. Answer the question using a single word or phrase."
+
+    elif dataset in ["mmmu"]:
+        if "A." in question:
+            question = question + "\nAnswer with the option's letter from the given choices directly."
+        else:
+            question = question + "\nAnswer the question using a single word or phrase."
+        
+    elif dataset in ["hallusionbench"]:
+        if "Please answer yes or no." not in question:
+            question = question + "Please answer yes or no."
+
+    qa_prompt = make_human_string("<s>"+"<tor>"*10+f"[UNUSED_TOKEN_146]system\n{system_prompt}[UNUSED_TOKEN_145]",
+                                f"[UNUSED_TOKEN_146]user\n{question}[UNUSED_TOKEN_145]",
+                                "[UNUSED_TOKEN_146]assistant\n",
+                                split='\n')
+
+    return qa_prompt
+
+
+def make_human_string(*args, split):
+    out = ''
+    for i, arg in enumerate(args):
+        out += arg
+        if i != len(args)-1:
+            out += split
+    return out
+
+def get_max_new_tokens(data_name):
+    if data_name.lower() in ["mme", "pope", "sqa", "mmbench", "mmbench_cn", "mmbench_dev","mmbench_cn_dev", "seed", "qbench", "ai2d", "mmstar", "vqav2", "gqa", "chartqa", "hallusionbench", "textvqa", "mmmu"]:
+        return 5
+    if data_name.lower() in ["llava", "mm-vet"]:
+        return 1024
+    else:
+        return 128
+
+"""
+Print Data Statistics
+"""
+def print_data_statistics(data):
+    # name set
+    name_set = {'caption',
+                'instruction',
+                'minigemini',
+                'docdownstream',
+                'docreason',
+                'gllava',
+                'mathvision',
+                'mathinstruct',
+                'mathplus'}
+    caption = []
+    instruction = []
+    minigemini = []
+    docdownstream = []
+    docreason = []
+    gllava = []
+    mathvision = []
+    mathinstruct = []
+    mathplus = []
+    for d in data:
+        for name in name_set:
+            if name in d['id']:
+                eval(f'{name}.append(1)')
+                break
+    num_caption = sum(caption)
+    num_instruction = sum(instruction)
+    num_minigemini = sum(minigemini)
+    num_docdownstream = sum(docdownstream)
+    num_docreason = sum(docreason)
+    num_gllava = sum(gllava)
+    num_mathvision = sum(mathvision)
+    num_mathinstruct = sum(mathinstruct)
+    num_mathplus = sum(mathplus)
+
+    total_len = num_caption + num_instruction + num_minigemini + \
+    num_docdownstream + num_docreason + num_gllava + \
+    num_mathvision +  num_mathinstruct + num_mathplus
+
+    print('Meteor Dataset Structure Statistics')
+    print(f'Total Length: {total_len}')
+    print('--------------------------------------------')
+    print(f'ShareGPT4V-Caption: {num_caption}')
+    print(f'ShareGPT4V-Instruction: {num_instruction}')
+    print(f'MiniGemini: {num_minigemini}')
+    print(f'DocDownstream: {num_docdownstream}')
+    print(f'DocReason: {num_docreason}')
+    print(f'GLLaVA: {num_gllava}')
+    print(f'MathVision: {num_mathvision}')
+    print(f'MathInstruct: {num_mathinstruct}')
+    print(f'MathPlus: {num_mathplus}')
+    print('--------------------------------------------')
+    print(f'Real-World Image: {num_caption + num_instruction}')
+    print(f'Document & Chart & Diagram & Sign & Symbol: {num_minigemini + num_docdownstream + num_docreason}')
+    print(f'Math: {num_gllava + num_mathvision + num_mathinstruct + num_mathplus}')
+    print(f'     Math with Vision: {num_gllava + num_mathvision}')
+    print(f'     Math with Text only: {num_mathinstruct + num_mathplus}')
+    print('--------------------------------------------')
+    print('')