Upload processor

added_tokens.json

@@ -1,82 +1,80 @@
 {
-  "''": 57526,
-  "1": 57525,
   "<s_iitcdip>": 57523,
   "<s_synthdog>": 57524,
   "<sep/>": 57522,
-  "a''1 ": 57582,
-  "a''16 ": 57586,
-  "a''2 ": 57583,
-  "a''4 ": 57584,
-  "a''8 ": 57585,
-  "a'1 ": 57577,
-  "a'16 ": 57581,
-  "a'2 ": 57578,
-  "a'4 ": 57579,
-  "a'8 ": 57580,
-  "b''1 ": 57592,
-  "b''16 ": 57596,
-  "b''2 ": 57593,
-  "b''4 ": 57594,
-  "b''8 ": 57595,
-  "b'1 ": 57587,
-  "b'16 ": 57591,
-  "b'2 ": 57588,
-  "b'4 ": 57589,
-  "b'8 ": 57590,
-  "c''1 ": 57532,
-  "c''16 ": 57536,
-  "c''2 ": 57533,
-  "c''4 ": 57534,
-  "c''8 ": 57535,
-  "c'1 ": 57527,
-  "c'16 ": 57531,
-  "c'2 ": 57528,
-  "c'4 ": 57529,
-  "c'8 ": 57530,
-  "d''1 ": 57542,
-  "d''16 ": 57546,
-  "d''2 ": 57543,
-  "d''4 ": 57544,
-  "d''8 ": 57545,
-  "d'1 ": 57537,
-  "d'16 ": 57541,
-  "d'2 ": 57538,
-  "d'4 ": 57539,
-  "d'8 ": 57540,
-  "e''1 ": 57552,
-  "e''16 ": 57556,
-  "e''2 ": 57553,
-  "e''4 ": 57554,
-  "e''8 ": 57555,
-  "e'1 ": 57547,
-  "e'16 ": 57551,
-  "e'2 ": 57548,
-  "e'4 ": 57549,
-  "e'8 ": 57550,
-  "f''1 ": 57562,
-  "f''16 ": 57566,
-  "f''2 ": 57563,
-  "f''4 ": 57564,
-  "f''8 ": 57565,
-  "f'1 ": 57557,
-  "f'16 ": 57561,
-  "f'2 ": 57558,
-  "f'4 ": 57559,
-  "f'8 ": 57560,
-  "g''1 ": 57572,
-  "g''16 ": 57576,
-  "g''2 ": 57573,
-  "g''4 ": 57574,
-  "g''8 ": 57575,
-  "g'1 ": 57567,
-  "g'16 ": 57571,
-  "g'2 ": 57568,
-  "g'4 ": 57569,
-  "g'8 ": 57570,
-  "r1 ": 57597,
-  "r16 ": 57601,
-  "r2 ": 57598,
-  "r4 ": 57599,
-  "r8 ": 57600
+  "a''1 ": 57580,
+  "a''16 ": 57584,
+  "a''2 ": 57581,
+  "a''4 ": 57582,
+  "a''8 ": 57583,
+  "a'1 ": 57575,
+  "a'16 ": 57579,
+  "a'2 ": 57576,
+  "a'4 ": 57577,
+  "a'8 ": 57578,
+  "b''1 ": 57590,
+  "b''16 ": 57594,
+  "b''2 ": 57591,
+  "b''4 ": 57592,
+  "b''8 ": 57593,
+  "b'1 ": 57585,
+  "b'16 ": 57589,
+  "b'2 ": 57586,
+  "b'4 ": 57587,
+  "b'8 ": 57588,
+  "c''1 ": 57530,
+  "c''16 ": 57534,
+  "c''2 ": 57531,
+  "c''4 ": 57532,
+  "c''8 ": 57533,
+  "c'1 ": 57525,
+  "c'16 ": 57529,
+  "c'2 ": 57526,
+  "c'4 ": 57527,
+  "c'8 ": 57528,
+  "d''1 ": 57540,
+  "d''16 ": 57544,
+  "d''2 ": 57541,
+  "d''4 ": 57542,
+  "d''8 ": 57543,
+  "d'1 ": 57535,
+  "d'16 ": 57539,
+  "d'2 ": 57536,
+  "d'4 ": 57537,
+  "d'8 ": 57538,
+  "e''1 ": 57550,
+  "e''16 ": 57554,
+  "e''2 ": 57551,
+  "e''4 ": 57552,
+  "e''8 ": 57553,
+  "e'1 ": 57545,
+  "e'16 ": 57549,
+  "e'2 ": 57546,
+  "e'4 ": 57547,
+  "e'8 ": 57548,
+  "f''1 ": 57560,
+  "f''16 ": 57564,
+  "f''2 ": 57561,
+  "f''4 ": 57562,
+  "f''8 ": 57563,
+  "f'1 ": 57555,
+  "f'16 ": 57559,
+  "f'2 ": 57556,
+  "f'4 ": 57557,
+  "f'8 ": 57558,
+  "g''1 ": 57570,
+  "g''16 ": 57574,
+  "g''2 ": 57571,
+  "g''4 ": 57572,
+  "g''8 ": 57573,
+  "g'1 ": 57565,
+  "g'16 ": 57569,
+  "g'2 ": 57566,
+  "g'4 ": 57567,
+  "g'8 ": 57568,
+  "r1 ": 57595,
+  "r16 ": 57599,
+  "r2 ": 57596,
+  "r4 ": 57597,
+  "r8 ": 57598
 }

tokenizer.json

@@ -39,15 +39,6 @@
       "normalized": false,
       "special": true
     },
-    {
-      "id": 28431,
-      "content": "'",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
     {
       "id": 57521,
       "content": "<mask>",
@@ -86,24 +77,6 @@
     },
     {
       "id": 57525,
-      "content": "1",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 57526,
-      "content": "''",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 57527,
       "content": "c'1 ",
       "single_word": false,
       "lstrip": false,
@@ -112,7 +85,7 @@
       "special": false
     },
     {
-      "id": 57528,
+      "id": 57526,
       "content": "c'2 ",
       "single_word": false,
       "lstrip": false,
@@ -121,7 +94,7 @@
       "special": false
     },
     {
-      "id": 57529,
+      "id": 57527,
       "content": "c'4 ",
       "single_word": false,
       "lstrip": false,
@@ -130,7 +103,7 @@
       "special": false
     },
     {
-      "id": 57530,
+      "id": 57528,
       "content": "c'8 ",
       "single_word": false,
       "lstrip": false,
@@ -139,7 +112,7 @@
       "special": false
     },
     {
-      "id": 57531,
+      "id": 57529,
       "content": "c'16 ",
       "single_word": false,
       "lstrip": false,
@@ -148,7 +121,7 @@
       "special": false
     },
     {
-      "id": 57532,
+      "id": 57530,
       "content": "c''1 ",
       "single_word": false,
       "lstrip": false,
@@ -157,7 +130,7 @@
       "special": false
     },
     {
-      "id": 57533,
+      "id": 57531,
       "content": "c''2 ",
       "single_word": false,
       "lstrip": false,
@@ -166,7 +139,7 @@
       "special": false
     },
     {
-      "id": 57534,
+      "id": 57532,
       "content": "c''4 ",
       "single_word": false,
       "lstrip": false,
@@ -175,7 +148,7 @@
       "special": false
     },
     {
-      "id": 57535,
+      "id": 57533,
       "content": "c''8 ",
       "single_word": false,
       "lstrip": false,
@@ -184,7 +157,7 @@
       "special": false
     },
     {
-      "id": 57536,
+      "id": 57534,
       "content": "c''16 ",
       "single_word": false,
       "lstrip": false,
@@ -193,7 +166,7 @@
       "special": false
     },
     {
-      "id": 57537,
+      "id": 57535,
       "content": "d'1 ",
       "single_word": false,
       "lstrip": false,
@@ -202,7 +175,7 @@
       "special": false
     },
     {
-      "id": 57538,
+      "id": 57536,
       "content": "d'2 ",
       "single_word": false,
       "lstrip": false,
@@ -211,7 +184,7 @@
       "special": false
     },
     {
-      "id": 57539,
+      "id": 57537,
       "content": "d'4 ",
       "single_word": false,
       "lstrip": false,
@@ -220,7 +193,7 @@
       "special": false
     },
     {
-      "id": 57540,
+      "id": 57538,
       "content": "d'8 ",
       "single_word": false,
       "lstrip": false,
@@ -229,7 +202,7 @@
       "special": false
     },
     {
-      "id": 57541,
+      "id": 57539,
       "content": "d'16 ",
       "single_word": false,
       "lstrip": false,
@@ -238,7 +211,7 @@
       "special": false
     },
     {
-      "id": 57542,
+      "id": 57540,
       "content": "d''1 ",
       "single_word": false,
       "lstrip": false,
@@ -247,7 +220,7 @@
       "special": false
     },
     {
-      "id": 57543,
+      "id": 57541,
       "content": "d''2 ",
       "single_word": false,
       "lstrip": false,
@@ -256,7 +229,7 @@
       "special": false
     },
     {
-      "id": 57544,
+      "id": 57542,
       "content": "d''4 ",
       "single_word": false,
       "lstrip": false,
@@ -265,7 +238,7 @@
       "special": false
     },
     {
-      "id": 57545,
+      "id": 57543,
       "content": "d''8 ",
       "single_word": false,
       "lstrip": false,
@@ -274,7 +247,7 @@
       "special": false
     },
     {
-      "id": 57546,
+      "id": 57544,
       "content": "d''16 ",
       "single_word": false,
       "lstrip": false,
@@ -283,7 +256,7 @@
       "special": false
     },
     {
-      "id": 57547,
+      "id": 57545,
       "content": "e'1 ",
       "single_word": false,
       "lstrip": false,
@@ -292,7 +265,7 @@
       "special": false
     },
     {
-      "id": 57548,
+      "id": 57546,
       "content": "e'2 ",
       "single_word": false,
       "lstrip": false,
@@ -301,7 +274,7 @@
       "special": false
     },
     {
-      "id": 57549,
+      "id": 57547,
       "content": "e'4 ",
       "single_word": false,
       "lstrip": false,
@@ -310,7 +283,7 @@
       "special": false
     },
     {
-      "id": 57550,
+      "id": 57548,
       "content": "e'8 ",
       "single_word": false,
       "lstrip": false,
@@ -319,7 +292,7 @@
       "special": false
     },
     {
-      "id": 57551,
+      "id": 57549,
       "content": "e'16 ",
       "single_word": false,
       "lstrip": false,
@@ -328,7 +301,7 @@
       "special": false
     },
     {
-      "id": 57552,
+      "id": 57550,
       "content": "e''1 ",
       "single_word": false,
       "lstrip": false,
@@ -337,7 +310,7 @@
       "special": false
     },
     {
-      "id": 57553,
+      "id": 57551,
       "content": "e''2 ",
       "single_word": false,
       "lstrip": false,
@@ -346,7 +319,7 @@
       "special": false
     },
     {
-      "id": 57554,
+      "id": 57552,
       "content": "e''4 ",
       "single_word": false,
       "lstrip": false,
@@ -355,7 +328,7 @@
       "special": false
     },
     {
-      "id": 57555,
+      "id": 57553,
       "content": "e''8 ",
       "single_word": false,
       "lstrip": false,
@@ -364,7 +337,7 @@
       "special": false
     },
     {
-      "id": 57556,
+      "id": 57554,
       "content": "e''16 ",
       "single_word": false,
       "lstrip": false,
@@ -373,7 +346,7 @@
       "special": false
     },
     {
-      "id": 57557,
+      "id": 57555,
       "content": "f'1 ",
       "single_word": false,
       "lstrip": false,
@@ -382,7 +355,7 @@
       "special": false
     },
     {
-      "id": 57558,
+      "id": 57556,
       "content": "f'2 ",
       "single_word": false,
       "lstrip": false,
@@ -391,7 +364,7 @@
       "special": false
     },
     {
-      "id": 57559,
+      "id": 57557,
       "content": "f'4 ",
       "single_word": false,
       "lstrip": false,
@@ -400,7 +373,7 @@
       "special": false
     },
     {
-      "id": 57560,
+      "id": 57558,
       "content": "f'8 ",
       "single_word": false,
       "lstrip": false,
@@ -409,7 +382,7 @@
       "special": false
     },
     {
-      "id": 57561,
+      "id": 57559,
       "content": "f'16 ",
       "single_word": false,
       "lstrip": false,
@@ -418,7 +391,7 @@
       "special": false
     },
     {
-      "id": 57562,
+      "id": 57560,
       "content": "f''1 ",
       "single_word": false,
       "lstrip": false,
@@ -427,7 +400,7 @@
       "special": false
     },
     {
-      "id": 57563,
+      "id": 57561,
       "content": "f''2 ",
       "single_word": false,
       "lstrip": false,
@@ -436,7 +409,7 @@
       "special": false
     },
     {
-      "id": 57564,
+      "id": 57562,
       "content": "f''4 ",
       "single_word": false,
       "lstrip": false,
@@ -445,7 +418,7 @@
       "special": false
     },
     {
-      "id": 57565,
+      "id": 57563,
       "content": "f''8 ",
       "single_word": false,
       "lstrip": false,
@@ -454,7 +427,7 @@
       "special": false
     },
     {
-      "id": 57566,
+      "id": 57564,
       "content": "f''16 ",
       "single_word": false,
       "lstrip": false,
@@ -463,7 +436,7 @@
       "special": false
     },
     {
-      "id": 57567,
+      "id": 57565,
       "content": "g'1 ",
       "single_word": false,
       "lstrip": false,
@@ -472,7 +445,7 @@
       "special": false
     },
     {
-      "id": 57568,
+      "id": 57566,
       "content": "g'2 ",
       "single_word": false,
       "lstrip": false,
@@ -481,7 +454,7 @@
       "special": false
     },
     {
-      "id": 57569,
+      "id": 57567,
       "content": "g'4 ",
       "single_word": false,
       "lstrip": false,
@@ -490,7 +463,7 @@
       "special": false
     },
     {
-      "id": 57570,
+      "id": 57568,
       "content": "g'8 ",
       "single_word": false,
       "lstrip": false,
@@ -499,7 +472,7 @@
       "special": false
     },
     {
-      "id": 57571,
+      "id": 57569,
       "content": "g'16 ",
       "single_word": false,
       "lstrip": false,
@@ -508,7 +481,7 @@
       "special": false
     },
     {
-      "id": 57572,
+      "id": 57570,
       "content": "g''1 ",
       "single_word": false,
       "lstrip": false,
@@ -517,7 +490,7 @@
       "special": false
     },
     {
-      "id": 57573,
+      "id": 57571,
       "content": "g''2 ",
       "single_word": false,
       "lstrip": false,
@@ -526,7 +499,7 @@
       "special": false
     },
     {
-      "id": 57574,
+      "id": 57572,
       "content": "g''4 ",
       "single_word": false,
       "lstrip": false,
@@ -535,7 +508,7 @@
       "special": false
     },
     {
-      "id": 57575,
+      "id": 57573,
       "content": "g''8 ",
       "single_word": false,
       "lstrip": false,
@@ -544,7 +517,7 @@
       "special": false
     },
     {
-      "id": 57576,
+      "id": 57574,
       "content": "g''16 ",
       "single_word": false,
       "lstrip": false,
@@ -553,7 +526,7 @@
       "special": false
     },
     {
-      "id": 57577,
+      "id": 57575,
       "content": "a'1 ",
       "single_word": false,
       "lstrip": false,
@@ -562,7 +535,7 @@
       "special": false
     },
     {
-      "id": 57578,
+      "id": 57576,
       "content": "a'2 ",
       "single_word": false,
       "lstrip": false,
@@ -571,7 +544,7 @@
       "special": false
     },
     {
-      "id": 57579,
+      "id": 57577,
       "content": "a'4 ",
       "single_word": false,
       "lstrip": false,
@@ -580,7 +553,7 @@
       "special": false
     },
     {
-      "id": 57580,
+      "id": 57578,
       "content": "a'8 ",
       "single_word": false,
       "lstrip": false,
@@ -589,7 +562,7 @@
       "special": false
     },
     {
-      "id": 57581,
+      "id": 57579,
       "content": "a'16 ",
       "single_word": false,
       "lstrip": false,
@@ -598,7 +571,7 @@
       "special": false
     },
     {
-      "id": 57582,
+      "id": 57580,
       "content": "a''1 ",
       "single_word": false,
       "lstrip": false,
@@ -607,7 +580,7 @@
       "special": false
     },
     {
-      "id": 57583,
+      "id": 57581,
       "content": "a''2 ",
       "single_word": false,
       "lstrip": false,
@@ -616,7 +589,7 @@
       "special": false
     },
     {
-      "id": 57584,
+      "id": 57582,
       "content": "a''4 ",
       "single_word": false,
       "lstrip": false,
@@ -625,7 +598,7 @@
       "special": false
     },
     {
-      "id": 57585,
+      "id": 57583,
       "content": "a''8 ",
       "single_word": false,
       "lstrip": false,
@@ -634,7 +607,7 @@
       "special": false
     },
     {
-      "id": 57586,
+      "id": 57584,
       "content": "a''16 ",
       "single_word": false,
       "lstrip": false,
@@ -643,7 +616,7 @@
       "special": false
     },
     {
-      "id": 57587,
+      "id": 57585,
       "content": "b'1 ",
       "single_word": false,
       "lstrip": false,
@@ -652,7 +625,7 @@
       "special": false
     },
     {
-      "id": 57588,
+      "id": 57586,
       "content": "b'2 ",
       "single_word": false,
       "lstrip": false,
@@ -661,7 +634,7 @@
       "special": false
     },
     {
-      "id": 57589,
+      "id": 57587,
       "content": "b'4 ",
       "single_word": false,
       "lstrip": false,
@@ -670,7 +643,7 @@
       "special": false
     },
     {
-      "id": 57590,
+      "id": 57588,
       "content": "b'8 ",
       "single_word": false,
       "lstrip": false,
@@ -679,7 +652,7 @@
       "special": false
     },
     {
-      "id": 57591,
+      "id": 57589,
       "content": "b'16 ",
       "single_word": false,
       "lstrip": false,
@@ -688,7 +661,7 @@
       "special": false
     },
     {
-      "id": 57592,
+      "id": 57590,
       "content": "b''1 ",
       "single_word": false,
       "lstrip": false,
@@ -697,7 +670,7 @@
       "special": false
     },
     {
-      "id": 57593,
+      "id": 57591,
       "content": "b''2 ",
       "single_word": false,
       "lstrip": false,
@@ -706,7 +679,7 @@
       "special": false
     },
     {
-      "id": 57594,
+      "id": 57592,
       "content": "b''4 ",
       "single_word": false,
       "lstrip": false,
@@ -715,7 +688,7 @@
       "special": false
     },
     {
-      "id": 57595,
+      "id": 57593,
       "content": "b''8 ",
       "single_word": false,
       "lstrip": false,
@@ -724,7 +697,7 @@
       "special": false
     },
     {
-      "id": 57596,
+      "id": 57594,
       "content": "b''16 ",
       "single_word": false,
       "lstrip": false,
@@ -733,7 +706,7 @@
       "special": false
     },
     {
-      "id": 57597,
+      "id": 57595,
       "content": "r1 ",
       "single_word": false,
       "lstrip": false,
@@ -742,7 +715,7 @@
       "special": false
     },
     {
-      "id": 57598,
+      "id": 57596,
       "content": "r2 ",
       "single_word": false,
       "lstrip": false,
@@ -751,7 +724,7 @@
       "special": false
     },
     {
-      "id": 57599,
+      "id": 57597,
       "content": "r4 ",
       "single_word": false,
       "lstrip": false,
@@ -760,7 +733,7 @@
       "special": false
     },
     {
-      "id": 57600,
+      "id": 57598,
       "content": "r8 ",
       "single_word": false,
       "lstrip": false,
@@ -769,7 +742,7 @@
       "special": false
     },
     {
-      "id": 57601,
+      "id": 57599,
       "content": "r16 ",
       "single_word": false,
       "lstrip": false,

tokenizer_config.json

@@ -11,7 +11,7 @@
     "single_word": false
   },
   "model_max_length": 1000000000000000019884624838656,
-  "name_or_path": "./model_5",
+  "name_or_path": "./model_7",
   "pad_token": "<pad>",
   "processor_class": "DonutProcessor",
   "sep_token": "</s>",