Spaces:
Sleeping
Sleeping
Commit
·
c563cac
1
Parent(s):
ead38a1
Updated load method
Browse files- language_bpe/base.py +1 -1
language_bpe/base.py
CHANGED
@@ -119,7 +119,7 @@ class Tokenizer:
|
|
119 |
# read the model file
|
120 |
merges = {}
|
121 |
special_tokens = {}
|
122 |
-
idx =
|
123 |
with open(model_file, 'r', encoding="utf-8") as f:
|
124 |
# read the version
|
125 |
version = f.readline().strip()
|
|
|
119 |
# read the model file
|
120 |
merges = {}
|
121 |
special_tokens = {}
|
122 |
+
idx = 384
|
123 |
with open(model_file, 'r', encoding="utf-8") as f:
|
124 |
# read the version
|
125 |
version = f.readline().strip()
|