shenzye46 commited on
Commit
5299779
·
verified ·
1 Parent(s): 5d72c4c

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +4 -0
app.py CHANGED
@@ -8,6 +8,10 @@ tokenizer_name = "HuggingFaceTB/SmolLM-135M"
8
  def load_model_and_tokenizer():
9
  tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
10
  model = AutoModelForCausalLM.from_pretrained(model_name)
 
 
 
 
11
  return tokenizer, model
12
 
13
  tokenizer, model = load_model_and_tokenizer()
 
8
  def load_model_and_tokenizer():
9
  tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
10
  model = AutoModelForCausalLM.from_pretrained(model_name)
11
+ # Ensure tokenizer.pad_token is set to tokenizer.eos_token
12
+ if tokenizer.pad_token is None:
13
+ tokenizer.pad_token = tokenizer.eos_token
14
+ tokenizer.pad_token_id = tokenizer.eos_token_id
15
  return tokenizer, model
16
 
17
  tokenizer, model = load_model_and_tokenizer()