Update hf_rwkv_tokenizer.py

Bug fix of `convert_tokens_to_ids` and `decode` for byte-fallback characters.

hf_rwkv_tokenizer.py

@@ -182,8 +182,15 @@ class Rwkv6Tokenizer(PreTrainedTokenizer):
         # return self.wordpiece_tokenizer.tokenize(text.encode("utf-8"))
         return self.trie_tokenizer.encode(text)[0]
 
+    def _decode(self, token_ids, **kwargs):
+        return self.trie_tokenizer.decodeBytes(token_ids).decode("utf-8")
+
     def _convert_token_to_id(self, token):
-        return token
+        if isinstance(token, (int)):
+            return token
+        elif isinstance(token, (bytes)):
+            return self.encoder.get(token)
+        return self.encoder.get(token.encode("utf-8"))
 
     def _convert_id_to_token(self, index):
         """Converts an index (integer) in a token (byte) using the vocab."""