Spaces:
Sleeping
Sleeping
Ahmet Kaan Sever
commited on
Commit
·
2ac340e
1
Parent(s):
b9fba6d
added another deleted function bc of conflict
Browse files
src/deepeval/base_task.py
CHANGED
@@ -150,6 +150,14 @@ class BaseTask(ABC):
|
|
150 |
result = self.tokenizer.decode(output[0], skip_special_tokens=True)
|
151 |
return result
|
152 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
153 |
@abstractmethod
|
154 |
def load_dataset_from_hf(self):
|
155 |
"""
|
|
|
150 |
result = self.tokenizer.decode(output[0], skip_special_tokens=True)
|
151 |
return result
|
152 |
|
153 |
+
def get_chat_template_tokens(self):
|
154 |
+
allowed_token_chat = [
|
155 |
+
{"role": "user", "content": ""},
|
156 |
+
{"role": "assistant", "content": ""}
|
157 |
+
]
|
158 |
+
allowed_special_tokens = self.tokenizer.apply_chat_template(allowed_token_chat, tokenize=True)
|
159 |
+
return allowed_special_tokens
|
160 |
+
|
161 |
@abstractmethod
|
162 |
def load_dataset_from_hf(self):
|
163 |
"""
|