Update tokenizer.py

tokenizer.py

@@ -16,6 +16,7 @@ class Tokenizer:
         self.eos_id = None
 
         # some checkpoints have both files, `.model` takes precedence
+        print(f'tokenizer.py checkpoint_dir is : {checkpoint_dir}')
         if (vocabulary_path := checkpoint_dir / "tokenizer.model").is_file():
             from sentencepiece import SentencePieceProcessor