Upload 3 files

config_kwargs.json

@@ -0,0 +1,83 @@
+{
+  "monitor_metric": "eval_exact_match",
+  "trainer_kwargs": {
+    "class_name": "RA_Reader"
+  },
+  "trainee_kwargs": {
+    "class_name": "Encoder_Generator",
+    "loss": {
+      "class_name": "NLLLoss"
+    },
+    "freeze_prefixes": [],
+    "use_image": true,
+    "quantize": false,
+    "codebook_kwargs": {
+      "class_name": "GroupedResidualVQ",
+      "decay": 0.5,
+      "commitment_weight": 1,
+      "dim": 1280,
+      "codebook_size": 1024,
+      "num_quantizers": 128,
+      "groups": 8,
+      "sample_codebook_temp": 0.0,
+      "learnable_codebook": false,
+      "use_cosine_sim": false,
+      "shared_codebook": false,
+      "kmeans_init": false
+    }
+  },
+  "data_module_kwargs": {
+    "class_name": "multi_modal_DataModule",
+    "data_processor": {
+      "class_name": "viquae_data_processor",
+      "dataset_path": "../../all_data/",
+      "kb_path": "../../all_data/passages/",
+      "entity_kb_path": "../../all_data/kb/"
+    },
+    "relevant_indices_key": "BM25_provenance_indices",
+    "irrelevant_indices_key": "BM25_irrelevant_indices",
+    "input_key": "input",
+    "passage_key": "passage",
+    "dataloader_kwargs": {
+      "num_workers": 6,
+      "prefetch_factor": 2
+    },
+    "image_processor_kwargs": {
+      "class_name": "ImageFormatter",
+      "feature_extractor_kwargs": {
+        "class_name": "CLIPFeatureExtractor",
+        "pretrained_model_name_or_path": "clip-vit-base-patch32_FE"
+      }
+    }
+  },
+  "answer_generator_kwargs": {
+    "class_name": "Blip2ForConditionalGeneration",
+    "checkpoint_name": "answer_generator",
+    "checkpoint": "saved_models/blip2_triviaqa/answer_generator/pytorch_model.bin",
+    "title_key": "wikipedia_title",
+    "column": "my_DPR_few_shot",
+    "pretrained_model_name_or_path": "/home/data/meerqat/my_transformers_cache/blip2",
+    "k_train": 5,
+    "k_test": 5,
+    "k_entities": 3,
+    "rag_training": false,
+    "rag": false,
+    "generator_only": true,
+    "entity_prompt": false,
+    "beam_search": true,
+    "best_answer": false,
+    "best_entity": false,
+    "tokenizer_kwargs": {
+      "class_name": "Blip2Processor",
+      "pretrained_model_name_or_path": "blip2_processor"
+    },
+    "image_processor_kwargs": {
+      "class_name": "ImageFormatter",
+      "feature_extractor_kwargs": {
+        "class_name": "Blip2Processor",
+        "pretrained_model_name_or_path": "blip2_processor"
+      }
+    },
+    "inference_path": "saved_models/blip2_triviaqa_viquae/answer_generator"
+  }
+}

experiment_params.json

@@ -0,0 +1,156 @@
+{
+  "subparser": null,
+  "max_seq_length": 256,
+  "question_max_seq_length": 256,
+  "RC": false,
+  "coco": false,
+  "flickr": false,
+  "aokvqa": false,
+  "grad_check": false,
+  "MJL": false,
+  "debug_run_time": false,
+  "only_neighbors": false,
+  "full_kb": false,
+  "image_eval": false,
+  "embedding_column": "my_DPR_few_shot",
+  "output_ds": null,
+  "output_kb": null,
+  "embed": false,
+  "data_path": null,
+  "train_batch_size": 4,
+  "eval_batch_size": 4,
+  "viquae_dev_batch_size": 1000,
+  "transformer_model_name": "bert-base-uncased",
+  "gradient_accumulation_steps": 1,
+  "warmup_proportion": 0.1,
+  "weight_decay": 0.01,
+  "lr_decay": false,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.999,
+  "warmup_steps": 4,
+  "adam_epsilon": 1e-08,
+  "num_train_epochs": 10,
+  "learning_rate": 0.0001,
+  "dropout": 0.5,
+  "bert_hidden_size": 768,
+  "MM": null,
+  "use_graph_P": null,
+  "use_graph_Q": null,
+  "syntactic_P": null,
+  "syntactic_Q": null,
+  "NS": null,
+  "IC_Q": false,
+  "IC_P": false,
+  "gcn_lr": 2e-05,
+  "head_lr": 2e-06,
+  "func": "cat",
+  "graph_pooling": "mean",
+  "max_nbr_nodes": 2,
+  "num_neib": 1,
+  "neib_depth": 1,
+  "undirect": false,
+  "node_hidden_size": 128,
+  "max_num_relations": null,
+  "max_num_syntactic_relations": null,
+  "map_wikidataRelCode_to_idx": null,
+  "graph_layout": "fr",
+  "residual": false,
+  "num_gcn": null,
+  "node_alias_max_len": 32,
+  "attention": false,
+  "num_paths": 1,
+  "kg_embed": false,
+  "graph_only": false,
+  "freeze": false,
+  "sanity_run": false,
+  "enhanced_question": false,
+  "use_entity_type": false,
+  "filter_edges": false,
+  "draw": false,
+  "layer_norm": false,
+  "filters_3": 768,
+  "tensor_neurons": 16,
+  "output_dir": "saved_models/blip2_triviaqa_viquae",
+  "main_dir": "/home/data/meerqat/ViQuAE/meerqat/GP-VQA",
+  "experiment_dir": "None",
+  "experiment_name": "blip2_triviaqa_viquae",
+  "search": null,
+  "xlnet": false,
+  "tune_loss": false,
+  "tune_dev": true,
+  "tune_valid_loss": false,
+  "sanity_val_steps": 0,
+  "cpu": false,
+  "keep_in_memory": false,
+  "N": 0,
+  "nbr_workers": 1,
+  "num_proc": 1,
+  "local_cache": "/home/data/meerqat/my_transformers_cache",
+  "IMAGE_PATH": null,
+  "transformer_path": null,
+  "resume_from": null,
+  "checkpoint": null,
+  "linear_MEP_checkpoint": null,
+  "linear_MLM_checkpoint": null,
+  "config": "experiments/ir/viquae/rag_multimodal/config_generator_only_blip2.json",
+  "context_checkpoint": null,
+  "grad_accum": 1,
+  "cls_token": "[CLS]",
+  "sep_token": "[SEP]",
+  "pad_token": "[PAD]",
+  "cls_token_at_end": false,
+  "mask_padding_with_zero": true,
+  "pad_on_left": false,
+  "sequence_a_segment_id": 0,
+  "pad_token_segment_id": 0,
+  "cls_token_segment_id": 0,
+  "pad_token_label_id": 0,
+  "pad_token_id": 0,
+  "past_index": -1,
+  "world_size": 1,
+  "use_lstm": false,
+  "use_entity": false,
+  "use_question_graph": false,
+  "use_question_objects": false,
+  "EFeat": null,
+  "NFeat": null,
+  "ltn": false,
+  "use_kelm": false,
+  "unshared": false,
+  "mlm": false,
+  "test": false,
+  "validate": false,
+  "split_dpr": false,
+  "rename_model": false,
+  "mask_rate": 0.15,
+  "pretrained_triviaq": false,
+  "error_analysis": false,
+  "debug": false,
+  "stop_debug": false,
+  "entity_linking": false,
+  "entity_linking_split": false,
+  "build_entity_paths": null,
+  "object_detection": null,
+  "update_wikidata_ids": null,
+  "update_None_ids": null,
+  "dependency_parsing": null,
+  "image_captionning": null,
+  "build_graph": false,
+  "from_bert": false,
+  "fast": false,
+  "get_examples": false,
+  "get_statistics": null,
+  "word_vocab_size": 0,
+  "char_vocab_size": 0,
+  "max_word_len": 30,
+  "device": null,
+  "embedding_matrix": null,
+  "w2v_file": "word_vector_200d.vec",
+  "word_emb_dim": 200,
+  "char_lstm": false,
+  "char_cnn": false,
+  "seed": 32,
+  "tune_batch_size": false,
+  "val_check_interval": null,
+  "model_names": null
+}

model-epoch=02-eval_exact_match=0.7088_sd=32.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc3ff22e8bf10b17abc125c18f4596bb5786ebd787d1702f9ee596a7e7dccf86
+size 15848832819