AkashDataScience commited on
Commit
c563cac
·
1 Parent(s): ead38a1

Updated load method

Browse files
Files changed (1) hide show
  1. language_bpe/base.py +1 -1
language_bpe/base.py CHANGED
@@ -119,7 +119,7 @@ class Tokenizer:
119
  # read the model file
120
  merges = {}
121
  special_tokens = {}
122
- idx = 256
123
  with open(model_file, 'r', encoding="utf-8") as f:
124
  # read the version
125
  version = f.readline().strip()
 
119
  # read the model file
120
  merges = {}
121
  special_tokens = {}
122
+ idx = 384
123
  with open(model_file, 'r', encoding="utf-8") as f:
124
  # read the version
125
  version = f.readline().strip()