Update model_hf.py
Browse files- model_hf.py +1 -1
model_hf.py
CHANGED
@@ -1518,7 +1518,7 @@ def create_model(param: Dimensions) -> Echo:
|
|
1518 |
|
1519 |
return model
|
1520 |
|
1521 |
-
def setup_tokenizer(token: str, local_tokenizer_path: str = "
|
1522 |
from tokenizers import Tokenizer
|
1523 |
tokenizer = Tokenizer.from_file(f"{local_tokenizer_path}/tokenizer.json")
|
1524 |
orig_encode = tokenizer.encode
|
|
|
1518 |
|
1519 |
return model
|
1520 |
|
1521 |
+
def setup_tokenizer(token: str, local_tokenizer_path: str = "./"):
|
1522 |
from tokenizers import Tokenizer
|
1523 |
tokenizer = Tokenizer.from_file(f"{local_tokenizer_path}/tokenizer.json")
|
1524 |
orig_encode = tokenizer.encode
|