TOkenizador

added_tokens.json

@@ -0,0 +1 @@
+{}

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<|startoftext|>", "eos_token": "<|endoftext|>", "unk_token": "<|endoftext|>"}

tokenizer_config.json

@@ -0,0 +1 @@
+{"bos_token": "<|startoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "max_len": 1024, "init_inputs": []}