Salesforce
/

xgen-7b-8k-base

Text Generation

text-generation-inference

Inference Endpoints

Model card Files Files and versions Community

tianxie-sf commited on Jul 7, 2023

Commit

c2f5b28

•

1 Parent(s): 3987e09

fix issue in get vocab

Files changed (1) hide show

tokenization_xgen.py +1 -1

tokenization_xgen.py CHANGED Viewed

@@ -139,7 +139,7 @@ class XgenTokenizer(PreTrainedTokenizer):
     def get_vocab(self):
         """Returns vocab as a dict"""
-        vocab = {self._convert_id_to_token(i): i for i in range(self.vocab_size)}
         return vocab
     def _tokenize(self, text, **kwargs):

     def get_vocab(self):
         """Returns vocab as a dict"""
+        vocab = {self.encoder.decode_single_token_bytes(i): i for i in range(self.vocab_size)}
         return vocab
     def _tokenize(self, text, **kwargs):