dewdev
/

language_detection

Text Classification

Model card Files Files and versions Community

dewdev commited on Feb 26

Commit

7262f79

·

verified ·

1 Parent(s): b6b2ea0

Upload tokenizer.json

Files changed (1) hide show

tokenizer.json +1 -4

tokenizer.json CHANGED Viewed

@@ -52,9 +52,6 @@
   "normalizer": {
     "type": "Sequence",
     "normalizers": [
-      {
-        "type": "NFD"
-      },
       {
         "type": "BertNormalizer",
         "clean_text": true,
@@ -50348,4 +50345,4 @@
       "erg": 50256
     }
   }
-}

   "normalizer": {
     "type": "Sequence",
     "normalizers": [
       {
         "type": "BertNormalizer",
         "clean_text": true,
       "erg": 50256
     }
   }
+}