Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
@@ -8,6 +8,10 @@ tokenizer_name = "HuggingFaceTB/SmolLM-135M"
|
|
8 |
def load_model_and_tokenizer():
|
9 |
tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
|
10 |
model = AutoModelForCausalLM.from_pretrained(model_name)
|
|
|
|
|
|
|
|
|
11 |
return tokenizer, model
|
12 |
|
13 |
tokenizer, model = load_model_and_tokenizer()
|
|
|
8 |
def load_model_and_tokenizer():
|
9 |
tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
|
10 |
model = AutoModelForCausalLM.from_pretrained(model_name)
|
11 |
+
# Ensure tokenizer.pad_token is set to tokenizer.eos_token
|
12 |
+
if tokenizer.pad_token is None:
|
13 |
+
tokenizer.pad_token = tokenizer.eos_token
|
14 |
+
tokenizer.pad_token_id = tokenizer.eos_token_id
|
15 |
return tokenizer, model
|
16 |
|
17 |
tokenizer, model = load_model_and_tokenizer()
|