tokenizer_v2 / special_tokens_map.json
mahrnoud's picture
Enhance Vocab and Special Tokens
203167f
raw
history blame contribute delete
498 Bytes
{
"additional_special_tokens": [
"[LANG_EN]",
"[LANG_AR]",
"[LANG_AR_EG]",
"[TRANSLATION]",
"[SUMMARIZATION]",
"[PARAPHRASING]",
"[INSTRUCTIONS]",
"[CALCULATE]",
"[REORDER]",
"[QUESTION]",
"[ANSWER]",
"[CHOICES]",
"[START_OPTION]",
"[END_OPTION]",
"[REASONING]",
"[START_THINKING]",
"[END_THINKING]"
],
"cls_token": "[CLS]",
"mask_token": "[MASK]",
"pad_token": "[PAD]",
"sep_token": "[SEP]",
"unk_token": "[UNK]"
}