miscovery
/

tokenizer

Model card Files Files and versions Community

mahrnoud commited on Apr 4

Commit

017af27

·

1 Parent(s): 6825cf2

Update Tokenizer

Files changed (2) hide show

tokenizer.json +2 -2
tokenizer_config.json +2 -1

tokenizer.json CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ae919b28537a24ab14728cdb016a2c3b0bdda997cb9e53779b168ab5500caf89
-size 14975662

 version https://git-lfs.github.com/spec/v1
+oid sha256:046482a5ffbb33555a23deffdafbc1c85384660eb2ca4d4bb83ed91c06d62992
+size 5000773

tokenizer_config.json CHANGED Viewed

@@ -67,7 +67,8 @@
   },
   "additional_special_tokens": [
     "[LANG_EN]",
-    "[LANG_AR]"
   ],
   "clean_up_tokenization_spaces": true,
   "cls_token": "[CLS]",

   },
   "additional_special_tokens": [
     "[LANG_EN]",
+    "[LANG_AR]",
+    "[LANG_AR_EG]"
   ],
   "clean_up_tokenization_spaces": true,
   "cls_token": "[CLS]",