dev

app.py

@@ -8,6 +8,7 @@ from sklearn.metrics import accuracy_score
 import numpy as np
 import torch
 import os
+import evaluate
 from huggingface_hub import login
 from peft import get_peft_model, LoraConfig
 
@@ -28,14 +29,23 @@ model.save_pretrained(model_save_path)
 
 def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size, lr, grad):
     try:
-        def compute_metrics(eval_pred):
-            logits, labels = eval_pred
-            predictions = np.argmax(logits, axis=1)
-            accuracy = accuracy_score(labels, predictions)
-            return {
-                'eval_accuracy': accuracy,
-                'eval_loss': eval_pred.loss,  # If you want to include loss as well
-            }        
+        metric = evaluate.load("rouge", cache_dir='/cache')
+        def compute_metrics(eval_preds):
+            preds, labels = eval_preds
+            if isinstance(preds, tuple):
+                preds = preds[0]
+            # Replace -100s used for padding as we can't decode them
+            preds = np.where(preds != -100, preds, tokenizer.pad_token_id)
+            decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)
+            labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
+            decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
+      
+            result = metric.compute(predictions=decoded_preds, references=decoded_labels, use_stemmer=True)
+            result = {k: round(v * 100, 4) for k, v in result.items()}
+            prediction_lens = [np.count_nonzero(pred != tokenizer.pad_token_id) for pred in preds]
+            result["gen_len"] = np.mean(prediction_lens)
+            return result
+        
         login(api_key.strip())
    
     

requirements.txt

@@ -6,3 +6,4 @@ huggingface_hub
 scikit-learn
 numpy
 torch
+evaluate