Evaluate model

.idea/discord.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="DiscordProjectSettings">
-    <option name="show" value="ASK" />
+    <option name="show" value="PROJECT_FILES" />
     <option name="description" value="" />
   </component>
 </project>

evaluate.py

@@ -0,0 +1,62 @@
+import pandas as pd
+import numpy as np
+from datasets import Dataset
+from sklearn.metrics import accuracy_score, precision_recall_fscore_support
+from transformers import BertTokenizer, BertForSequenceClassification, Trainer, TrainingArguments
+
+# Load the saved model and tokenizer
+def load_model_and_tokenizer(model_path):
+    model = BertForSequenceClassification.from_pretrained(model_path, num_labels=2)
+    tokenizer = BertTokenizer.from_pretrained(model_path)
+    return model, tokenizer
+
+# Function to tokenize the evaluation dataset
+def tokenize_function(examples, tokenizer):
+    return tokenizer(examples["text"], padding="max_length", truncation=True, max_length=512)
+
+# Load and prepare the evaluation dataset
+def load_evaluation_data(file_path, tokenizer):
+    df = pd.read_csv(file_path)
+    eval_dataset = Dataset.from_pandas(df)
+    eval_dataset = eval_dataset.map(lambda examples: tokenize_function(examples, tokenizer), batched=True)
+    eval_dataset.set_format(type='torch', columns=['input_ids', 'attention_mask', 'labels'])
+    return eval_dataset
+
+# Define the compute_metrics function to be used by the Trainer
+def compute_metrics(pred):
+    labels = pred.label_ids
+    preds = np.argmax(pred.predictions, axis=-1)
+    precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average='binary')
+    acc = accuracy_score(labels, preds)
+    return {
+        'accuracy': acc,
+        'f1': f1,
+        'precision': precision,
+        'recall': recall
+    }
+
+# Evaluation function using Hugging Face's Trainer
+def evaluate_model(model, tokenizer, eval_dataset):
+    training_args = TrainingArguments(
+        output_dir="./results",
+        per_device_eval_batch_size=8
+    )
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        eval_dataset=eval_dataset,
+        compute_metrics=compute_metrics
+    )
+    results = trainer.evaluate()
+    return results
+
+# Main function to run the evaluation
+if __name__ == "__main__":
+    model_path = "./trained_model"  # Path where the model and tokenizer are saved
+    eval_data_path = "path_to_evaluation_data.csv"  # Path to your evaluation dataset CSV file
+
+    model, tokenizer = load_model_and_tokenizer(model_path)
+    eval_dataset = load_evaluation_data(eval_data_path, tokenizer)
+    evaluation_results = evaluate_model(model, tokenizer, eval_dataset)
+
+    print("Evaluation Results:", evaluation_results)

train.py

@@ -13,6 +13,7 @@ train_df, eval_df = train_test_split(df, test_size=0.2)
 # Tokenizer
 tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
 
+
 def tokenize_function(examples):
     return tokenizer(examples["text"], padding="max_length", truncation=True, max_length=512)