nioushasadjadi commited on
Commit
4a303bd
·
1 Parent(s): 156a2ea

Change _encode function to encode.

Browse files
Files changed (1) hide show
  1. tokenizer.py +1 -1
tokenizer.py CHANGED
@@ -36,7 +36,7 @@ class KmerTokenizer(PreTrainedTokenizer):
36
  splits = [text[i:i + self.k] for i in range(0, len(text) - self.k + 1, self.stride)]
37
  return splits
38
 
39
- def _encode(self, text, **kwargs):
40
  tokens = self.tokenize(text, **kwargs)
41
  token_ids = self.convert_tokens_to_ids(tokens)
42
  if kwargs.get('return_tensors') == 'pt':
 
36
  splits = [text[i:i + self.k] for i in range(0, len(text) - self.k + 1, self.stride)]
37
  return splits
38
 
39
+ def encode(self, text, **kwargs):
40
  tokens = self.tokenize(text, **kwargs)
41
  token_ids = self.convert_tokens_to_ids(tokens)
42
  if kwargs.get('return_tensors') == 'pt':