Text Generation
Transformers
Safetensors
llama
text-generation-inference
Inference Endpoints
mfromm commited on
Commit
4915b03
·
verified ·
1 Parent(s): e3c7073

Update gptx_tokenizer.py

Browse files
Files changed (1) hide show
  1. gptx_tokenizer.py +1 -1
gptx_tokenizer.py CHANGED
@@ -62,7 +62,7 @@ class HFGPTXTokenizer(PreTrainedTokenizer):
62
  f"<placeholder_tok_{i}>" for i in range(256)
63
  ]
64
 
65
- def find_tokenizer_config(self, config_path: Path, repo_id: str = None) -> Optional[Path]:
66
  if not os.path.isfile(config_path):
67
  config_path = try_to_load_from_cache(repo_id=repo_id, filename=Path(config_path).name)
68
  if not config_path:
 
62
  f"<placeholder_tok_{i}>" for i in range(256)
63
  ]
64
 
65
+ def find_tokenizer_config(self, config_path: Path, repo_id: str = None) -> Optional[Path]:
66
  if not os.path.isfile(config_path):
67
  config_path = try_to_load_from_cache(repo_id=repo_id, filename=Path(config_path).name)
68
  if not config_path: