Upload Tokenizer v3 (#8)

- Upload Tokenizer v3 (a921d0f6efaac24620b92222c130d1888180be01)

README.md

@@ -1,16 +1,16 @@
 ---
-library_name: transformers
-base_model: codellama/CodeLlama-7b-Instruct-hf
-license: llama2
-datasets:
-- semantixai/Test-Dataset-Lloro
 language:
 - pt
+license: llama2
+library_name: transformers
 tags:
 - code
 - analytics
 - analise-dados
 - portugues-BR
+base_model: codellama/CodeLlama-7b-Instruct-hf
+datasets:
+- semantixai/Test-Dataset-Lloro
 ---
 
 **Lloro 7B**

special_tokens_map.json

@@ -7,13 +7,42 @@
     "▁<SUF>",
     "▁<MID>",
     "▁<EOT>",
+    "[PAD]",
+    "▁<PRE>",
+    "▁<MID>",
+    "▁<SUF>",
+    "▁<EOT>",
     "▁<PRE>",
     "▁<MID>",
     "▁<SUF>",
     "▁<EOT>"
   ],
-  "bos_token": "<s>",
-  "eos_token": "</s>",
-  "pad_token": "<unk>",
-  "unk_token": "<unk>"
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
 }

tokenizer.json

@@ -65,6 +65,15 @@
       "rstrip": true,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 32016,
+      "content": "[PAD]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": {
@@ -170,6 +179,7 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
+    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

tokenizer_config.json

@@ -1,4 +1,6 @@
 {
+  "add_bos_token": true,
+  "add_eos_token": false,
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",
@@ -55,6 +57,14 @@
       "rstrip": true,
       "single_word": false,
       "special": true
+    },
+    "32016": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
     }
   },
   "additional_special_tokens": [
@@ -65,12 +75,18 @@
     "▁<SUF>",
     "▁<MID>",
     "▁<EOT>",
+    "[PAD]",
+    "▁<PRE>",
+    "▁<MID>",
+    "▁<SUF>",
+    "▁<EOT>",
     "▁<PRE>",
     "▁<MID>",
     "▁<SUF>",
     "▁<EOT>"
   ],
   "bos_token": "<s>",
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content | trim + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' '  + content | trim + ' ' + eos_token }}{% endif %}{% endfor %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
   "eot_token": "▁<EOT>",
@@ -78,7 +94,7 @@
   "legacy": null,
   "middle_token": "▁<MID>",
   "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "<unk>",
+  "pad_token": "[PAD]",
   "prefix_token": "▁<PRE>",
   "sp_model_kwargs": {},
   "suffix_token": "▁<SUF>",