Update human_text_detect.py
Browse files- human_text_detect.py +3 -3
human_text_detect.py
CHANGED
@@ -98,10 +98,10 @@ def detect_human_text(model_name, topic, text):
|
|
98 |
# Init model
|
99 |
print('Init tokenizer')
|
100 |
lm_name = 'gpt2-xl' if model_name == 'GPT2XL' else 'microsoft/phi-2'
|
101 |
-
tokenizer = AutoTokenizer.from_pretrained(
|
102 |
|
103 |
-
|
104 |
-
|
105 |
|
106 |
print('Init model')
|
107 |
cache_dir_model = f"/data/cacheHuggingface/{model_name}/model"
|
|
|
98 |
# Init model
|
99 |
print('Init tokenizer')
|
100 |
lm_name = 'gpt2-xl' if model_name == 'GPT2XL' else 'microsoft/phi-2'
|
101 |
+
tokenizer = AutoTokenizer.from_pretrained(lm_name, cache_dir=cache_dir_tokenizer) #
|
102 |
|
103 |
+
print("Save tokenizer")
|
104 |
+
tokenizer.save_pretrained(cache_dir_tokenizer)
|
105 |
|
106 |
print('Init model')
|
107 |
cache_dir_model = f"/data/cacheHuggingface/{model_name}/model"
|