working version with base model + mcqbert

config.json

@@ -5,7 +5,7 @@
   ],
   "auto_map": {
     "AutoConfig": "configuration_mcqbert.MCQBertConfig",
-    "AutoModelForCausalLM": "modeling_mcqbert.MCQBert"
+    "AutoModel": "modeling_mcqbert.MCQBert"
   },
   "attention_probs_dropout_prob": 0.1,
   "classifier_dropout": null,
@@ -18,7 +18,7 @@
   "intermediate_size": 3072,
   "layer_norm_eps": 1e-12,
   "max_position_embeddings": 512,
-  "model_type": "bert",
+  "model_type": "mcqbert",
   "num_attention_heads": 12,
   "num_hidden_layers": 12,
   "pad_token_id": 0,

modeling_mcqbert.py

@@ -1,12 +1,14 @@
-from transformers import BertPreTrainedModel
+from transformers import BertModel
 import torch
 
 from .configuration_mcqbert import MCQBertConfig
 
-class MCQBert(BertPreTrainedModel):
+class MCQBert(BertModel):
     def __init__(self, config: MCQBertConfig):
         super().__init__(config)
-        self.student_embedding_layer = torch.nn.Linear(config.student_embedding_size, config.hidden_size)
+
+        if config.integration_strategy is not None:
+            self.student_embedding_layer = torch.nn.Linear(config.student_embedding_size, config.hidden_size)
         
         cls_input_dim_multiplier = 2 if config.integration_strategy == "cat" else 1
         cls_input_dim = self.config.hidden_size * cls_input_dim_multiplier