8d33560
1
2
3
4
5
6
7
from transformers import PreTrainedTokenizerFast class CryptGPTTokenizer(PreTrainedTokenizerFast): @staticmethod def clean_up_tokenization(out_string): return out_string.replace(' ', "")
