Spaces:
Paused
Paused
File size: 2,775 Bytes
f6890a5 6807ea3 f74f2a9 ec9c39a dbf76bc 6807ea3 e8c3b4b 6807ea3 e8c3b4b 6807ea3 e8c3b4b 6807ea3 e8c3b4b 6807ea3 e8c3b4b 6807ea3 9dd8848 e8c3b4b 9dd8848 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 |
from datetime import datetime
from src.deepeval.base_task import BaseTask
from deepeval.test_case import LLMTestCase
from typing import Any
from deepeval.metrics import GEval
from deepeval.test_case import LLMTestCaseParams
class TruthfulnessTask(BaseTask):
def __init__(self, model_name: str):
super().__init__("metunlp/sosyoloji_truthfulness", model_name=model_name)
self.correctness_metric = GEval(
name="Truthfulness",
criteria="Determine whether the actual output is factually correct based on the expected output.",
evaluation_steps=[
"Check whether the facts in 'actual output' contradict any facts in 'expected output'",
"Heavily penalize omission of detail",
"Vague language, or contradicting OPINIONS, are OK"
],
model="gpt-4o-mini",
evaluation_params=[
LLMTestCaseParams.INPUT,
LLMTestCaseParams.ACTUAL_OUTPUT,
LLMTestCaseParams.EXPECTED_OUTPUT
],
)
def load_dataset_from_hf(self):
dataset = super().load_dataset_lmjudge_from_hf()
return dataset
def evaluate(self) -> dict[str, Any]:
results = []
total_model_time = 0
total_judge_time = 0
for i, row in enumerate(self.dataset):
start_model = datetime.now()
question = row["question"]
expected_output = row["answer"]
prompt = f"Soru: {question}\nCevap:"
actual_output = self.generate_response(prompt, max_new_tokens=100)
end_model = datetime.now()
total_model_time += (end_model - start_model).total_seconds()
start_judge = datetime.now()
test_case = LLMTestCase(
input=question,
actual_output=actual_output,
expected_output=expected_output
)
self.correctness_metric.measure(test_case)
end_judge = datetime.now()
total_judge_time += (end_judge - start_judge).total_seconds()
results.append({
"index": i,
"score": self.correctness_metric.score,
"reason": self.correctness_metric.reason,
"input": question,
"expected_output": expected_output,
"actual_output": actual_output
})
#Sum all scores in results and divide to nubmer of results
overallScore = (sum([result["score"] for result in results]) / len(results)) * 100
print(f"Total model time: {total_model_time} seconds")
print(f"Total judge time: {total_judge_time} seconds")
return {"results": overallScore}
|