NorGLM/NorGPT-3B-NO-BoolQ-peft

Model Card

NorGPT-3B-NO-BoolQ-peft is trained on top of NorGPT-3B model on NO-BoolQ dataset.

Data format:

input: {passage}[SEP]{question}
label: {True, False} -> {1,0}

Run the Model

from peft import PeftModel, PeftConfig
from transformers import AutoModelForCausalLM, AutoTokenizer
import torch

torch_device = "cuda" if torch.cuda.is_available() else "cpu"

source_model_id = "NorGLM/NorGPT-3B"
peft_model_id = "NorGLM/NorGPT-3B-NO-BoolQ-peft"

config = PeftConfig.from_pretrained(peft_model_id)
model = AutoModelForCausalLM.from_pretrained(source_model_id, device_map='balanced')

tokenizer_max_len = 2048
tokenizer_config = {'pretrained_model_name_or_path': source_model_id,
                            'max_len': tokenizer_max_len}
tokenizer = tokenizer = AutoTokenizer.from_pretrained(**tokenizer_config)
tokenizer.pad_token = tokenizer.eos_token

model = PeftModel.from_pretrained(model, peft_model_id)

Inference Example

Load the model to evaluate on the validation set:

def getDataSetFromFiles(df):
    # convert dataset    
    df["text"] = df[["passage", "question"]].apply(lambda x: " [SEP] ".join(x.astype(str)), axis =1)
    df = df.drop(["idx", "passage", "question"], axis=1)
    #df['label'] = df['label'].replace({1:'contradiction', -1:'entailment', 0:'neutral'})
    df["label"] = df.label.map({True: 1, False: 0})
    return Dataset.from_pandas(df)

print("--LOADING EVAL DATAS---")
eval_data = load_dataset("NorGLM/NO-BoolQ", data_files="val.jsonl")
eval_data = getDataSetFromFiles(eval_data["train"].to_pandas())

print("--MAKING PREDICTIONS---")
model.eval()

y_true = []
y_pred = []
count = 0

for data in eval_data:
    count = count + 1
    if count % 100 == 0:
        print(count)
    inputs = tokenizer(data['text'], return_tensors="pt").to(torch_device)
    
    with torch.no_grad():
        logits = model(**inputs).logits
        #print(logits)

    predicted_class_id = logits.argmax().item()

    y_true.append(data['label'])
    y_pred.append(predicted_class_id)

print(y_pred)

print(f"Lenght of true_values: {len(y_true)}")   
print(f"Lenght of predicted_values: {len(y_pred)}")    

y_true = np.array(y_true)
y_pred = np.array(y_pred)

F_score = f1_score(y_true, y_pred, average="macro")
print(f"F1 score: {F_score}")

accuracy = accuracy_score(y_true, y_pred)
print(f"Accuracy: {accuracy}")

Note

More training details will be released soon!