Ahmet Kaan Sever commited on
Commit
2ac340e
·
1 Parent(s): b9fba6d

added another deleted function bc of conflict

Browse files
Files changed (1) hide show
  1. src/deepeval/base_task.py +8 -0
src/deepeval/base_task.py CHANGED
@@ -150,6 +150,14 @@ class BaseTask(ABC):
150
  result = self.tokenizer.decode(output[0], skip_special_tokens=True)
151
  return result
152
 
 
 
 
 
 
 
 
 
153
  @abstractmethod
154
  def load_dataset_from_hf(self):
155
  """
 
150
  result = self.tokenizer.decode(output[0], skip_special_tokens=True)
151
  return result
152
 
153
+ def get_chat_template_tokens(self):
154
+ allowed_token_chat = [
155
+ {"role": "user", "content": ""},
156
+ {"role": "assistant", "content": ""}
157
+ ]
158
+ allowed_special_tokens = self.tokenizer.apply_chat_template(allowed_token_chat, tokenize=True)
159
+ return allowed_special_tokens
160
+
161
  @abstractmethod
162
  def load_dataset_from_hf(self):
163
  """