Spaces:
Paused
Paused
| from src.deepeval.base_task import BaseTask | |
| from collections import defaultdict | |
| from src.deepeval.utils import accuracy, accuracy_standard_error | |
| from typing import Any | |
| import os | |
| import ast | |
| import re | |
| from datasets import load_dataset,get_dataset_split_names | |
| HF_TOKEN=os.getenv("HF_TOKEN") | |
| class TurkishVocabularyTask(BaseTask): | |
| def __init__(self, model_name): | |
| self.subsets = ["rare", "loan"] | |
| super().__init__("metunlp/turkish_vocabulary", model_name=model_name) | |
| def load_dataset_from_hf(self): | |
| evaluate_count = 50 | |
| dataset_dict = {} | |
| for subset in self.subsets: | |
| subset_data = load_dataset(self.dataset_repo, subset, token=HF_TOKEN, split="train") | |
| dataset_dict[subset] = subset_data.select(range(min(evaluate_count, len(subset_data)))) | |
| return dataset_dict | |
| def evaluate(self) -> dict[str, Any]: | |
| responses = [] | |
| difficulty_results = defaultdict(lambda: defaultdict(lambda: {'correct': 0, 'total': 0})) | |
| total_count = 0 | |
| true = 0 | |
| for subset in self.subsets: | |
| curr_dataset = self.dataset[subset] | |
| print(curr_dataset[0]) | |
| # Determine the question based on the subset | |
| if subset == "rare": | |
| question = "Verilen kelimenin eş anlamlısı aşağıdakilerden hangisidir?" | |
| elif subset == "loan": | |
| question = "Verilen kelimenin Türkçe kökenli eş anlamlısı aşağıdakilerden hangisidir?" | |
| else: | |
| question = "Verilen kelimenin eş anlamlısı aşağıdakilerden hangisidir?" | |
| for row in curr_dataset: | |
| total_count += 1 | |
| # Get values from row | |
| category = "hard" if row["level"]== 1 else "easy" if row["level"] == 0 else None | |
| answer_index = row["answer"] | |
| correct_answer_letter = chr(65 + answer_index) | |
| word = row["word"] | |
| choices = ast.literal_eval(row["choices"]) # Convert string to list | |
| formatted_choices = "\n".join([f"{chr(65 + i)}: {choice}" for i, choice in enumerate(choices)]) | |
| # Construct the prompt/message | |
| instruction = "" | |
| prompt = f"Soru: {question}\nKelime: {word}\nSeçenekler:\n{formatted_choices}\n{instruction}\n" | |
| message = prompt | |
| # Get/format answer of the model | |
| model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=2) | |
| responses.append(model_answer) | |
| model_answer_cleaned = model_answer.strip().replace('\n', '').replace(' ', '').upper().replace(':','') | |
| # Check if correct based on metric | |
| if correct_answer_letter == model_answer_cleaned: | |
| true += 1 | |
| difficulty_results[subset][category]['correct'] += 1 | |
| difficulty_results[subset][category]['total'] += 1 | |
| # Print results categorized by difficulty | |
| for subset in self.subsets: | |
| subset_results = difficulty_results[subset] | |
| for category, stats in subset_results.items(): | |
| correct = stats['correct'] | |
| total = stats['total'] | |
| calculatedAccuracy = correct / total if total > 0 else 0 | |
| print(f"{subset.capitalize()} {category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({correct}/{total})") | |
| print("Results:", responses) | |
| print("Overall Accuracy:", true / total_count) | |
| acc = accuracy(true, total_count) | |
| acc_stderr = accuracy_standard_error(acc, total_count) | |
| return {"acc": acc, "acc_stderr": acc_stderr} | |