Update app.py

app.py

@@ -37,6 +37,11 @@ tokenizer = AutoTokenizer.from_pretrained(model_name)
 data = {"text": [clean_text]}
 dataset = Dataset.from_dict(data)
 
+# Set a padding token manually
+tokenizer.pad_token = tokenizer.eos_token  # Use EOS as PAD token
+# Alternatively, add a new custom pad token
+# tokenizer.add_special_tokens({'pad_token': '[PAD]'})
+
 # Tokenization function
 def tokenize_function(examples):
     tokens = tokenizer(examples["text"], truncation=True, padding="max_length", max_length=512)