fix with convert_tokens_to_string

tokenization_chatglm.py

@@ -90,13 +90,6 @@ class ChatGLM4Tokenizer(PreTrainedTokenizer):
             tokens.append(self.decoder[t])
         return tokens
 
-    def _tokenize(self, text, **kwargs):
-        tokens = []
-        ids = self.tokenizer.encode(text)
-        for t in ids:
-            tokens.append(self.decoder[t])
-        return tokens
-
     def _convert_token_to_id(self, token):
         """ Converts a token (str) in an id using the vocab. """
         return self.mergeable_ranks[token]