Update gptx_tokenizer.py
Browse files- gptx_tokenizer.py +1 -1
gptx_tokenizer.py
CHANGED
@@ -62,7 +62,7 @@ class HFGPTXTokenizer(PreTrainedTokenizer):
|
|
62 |
f"<placeholder_tok_{i}>" for i in range(256)
|
63 |
]
|
64 |
|
65 |
-
def find_tokenizer_config(self, config_path: Path, repo_id: str = None) -> Optional[Path]:
|
66 |
if not os.path.isfile(config_path):
|
67 |
config_path = try_to_load_from_cache(repo_id=repo_id, filename=Path(config_path).name)
|
68 |
if not config_path:
|
|
|
62 |
f"<placeholder_tok_{i}>" for i in range(256)
|
63 |
]
|
64 |
|
65 |
+
def find_tokenizer_config(self, config_path: Path, repo_id: str = None) -> Optional[Path]:
|
66 |
if not os.path.isfile(config_path):
|
67 |
config_path = try_to_load_from_cache(repo_id=repo_id, filename=Path(config_path).name)
|
68 |
if not config_path:
|