Qwen
/

Qwen-VL-Chat

Text Generation

Model card Files Files and versions Community

simonJJJ commited on Aug 22, 2023

Commit

80ce4e4

•

1 Parent(s): c5924e3

update

Files changed (1) hide show

tokenization_qwen.py +1 -0

tokenization_qwen.py CHANGED Viewed

@@ -168,6 +168,7 @@ class QWenTokenizer(PreTrainedTokenizer):
         self.eod_id = self.tokenizer.eot_token
         self.im_start_id = self.special_tokens[IMSTART]
         self.im_end_id = self.special_tokens[IMEND]
     def __len__(self) -> int:
         return self.tokenizer.n_vocab

         self.eod_id = self.tokenizer.eot_token
         self.im_start_id = self.special_tokens[IMSTART]
         self.im_end_id = self.special_tokens[IMEND]
+        self.font_path = os.path.dirname(__file__)
     def __len__(self) -> int:
         return self.tokenizer.n_vocab