Sin2pi commited on
Commit
023253e
·
verified ·
1 Parent(s): 66453f2

Update model_hf.py

Browse files
Files changed (1) hide show
  1. model_hf.py +1 -1
model_hf.py CHANGED
@@ -1518,7 +1518,7 @@ def create_model(param: Dimensions) -> Echo:
1518
 
1519
  return model
1520
 
1521
- def setup_tokenizer(token: str, local_tokenizer_path: str = "D:/newmodel/model/tokenn/"):
1522
  from tokenizers import Tokenizer
1523
  tokenizer = Tokenizer.from_file(f"{local_tokenizer_path}/tokenizer.json")
1524
  orig_encode = tokenizer.encode
 
1518
 
1519
  return model
1520
 
1521
+ def setup_tokenizer(token: str, local_tokenizer_path: str = "./"):
1522
  from tokenizers import Tokenizer
1523
  tokenizer = Tokenizer.from_file(f"{local_tokenizer_path}/tokenizer.json")
1524
  orig_encode = tokenizer.encode