Spaces:
Running
on
L4
Running
on
L4
| from src.deepeval.base_task import BaseTask | |
| from collections import defaultdict | |
| from src.deepeval.utils import accuracy, accuracy_standard_error | |
| from typing import Any | |
| class CommonsenseReasoningTask(BaseTask): | |
| def __init__(self, model_name): | |
| super().__init__("metunlp/commonsense", model_name=model_name) | |
| def load_dataset_from_hf(self): | |
| dataset = super().load_dataset_from_hf() | |
| return dataset | |
| def evaluate(self) -> dict[str, Any]: | |
| responses = [] | |
| difficulty_results = defaultdict(lambda: {'correct': 0, 'total': 0}) | |
| total_count = 0 | |
| true = 0 | |
| for row in self.dataset: | |
| total_count += 1 | |
| # Get values from row | |
| label = row["label"] | |
| choices=[row["choice1"], row["choice2"]] | |
| formatted_choices = "\n".join([f"{chr(65+i)}: {choice}" for i, choice in enumerate(choices)]) | |
| category = row["difficulty"] | |
| answer = row["answer"] | |
| text = row["text"] | |
| context = row["context"] | |
| # Prints for debugging | |
| # print(f"Choices: {choices}") | |
| # print("Type of choices:", type(choices)) | |
| # print("Type of answer:", type(answer)) | |
| # Get answer index (starting from 0) | |
| if type(answer) == int: | |
| answer_index = answer - 1 # 1 or 2 | |
| else: | |
| answer_index = int(answer) - 1 | |
| correct_answer_letter = chr(65 + answer_index) | |
| # Get question based on label | |
| if label == "effect": | |
| question = "Seçeneklerden hangisi verilen önermenin bir sonucu veya etkisi olabilir?" | |
| elif label == "cause": | |
| question = "Seçeneklerden hangisi verilen önermenin bir neden veya sebebi olabilir?" | |
| else: | |
| question = "Seçeneklerden hangisi uygun?" # Alternatif | |
| # Construct the prompt/message | |
| instruction = "" | |
| prompt = f"Bağlam:\n{text}\nÖnerme:\n{context}\nSoru:{question}\nSeçenekler:\n{formatted_choices}\n{instruction}\n" | |
| message = prompt | |
| # Get/format answer of the model | |
| model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=2) | |
| responses.append(model_answer) | |
| model_answer_cleaned = model_answer.strip().replace('\n', '').replace(' ', '').upper() | |
| # Print answers | |
| # print(f"Correct Answer: {correct_answer_letter}") | |
| # print(f"Model Answer: {model_answer}") | |
| # print(f"Model Answer Cleaned: {model_answer_cleaned}") | |
| # Check if correct based on metric | |
| if correct_answer_letter == model_answer_cleaned: | |
| true += 1 | |
| difficulty_results[category]['correct'] += 1 | |
| difficulty_results[category]['total'] += 1 | |
| # Print results categorized by difficulty | |
| for category, stats in difficulty_results.items(): | |
| calculatedAccuracy = stats['correct'] / stats['total'] if stats['total'] > 0 else 0 | |
| print(f"{category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({stats['correct']}/{stats['total']})") | |
| print("Results:", responses) | |
| print("Overall Accuracy:", true / total_count) | |
| acc = accuracy(true, total_count) | |
| acc_stderr = accuracy_standard_error(acc, total_count) | |
| return {"acc": acc, "acc_stderr": acc_stderr} | |