Update app.py

app.py

@@ -6,10 +6,11 @@ from transformers import AutoTokenizer, AutoModelForCausalLM
 # Load the model and tokenizer
 tokenizer = AutoTokenizer.from_pretrained("TheBloke/Chronoboros-33B-GPTQ")
 model = AutoModelForCausalLM.from_pretrained("TheBloke/Chronoboros-33B-GPTQ", device_map="auto")
-model.eval()  # set model to evaluation mode
 
-# Optional: Use torch.compile() if you're on PyTorch 2.0+ for further speed-up
-# model = torch.compile(model)
+# Set a valid pad_token_id to avoid generation errors
+model.generation_config.pad_token_id = tokenizer.eos_token_id
+
+model.eval()  # Ensure the model is in evaluation mode
 
 @spaces.GPU
 def respond(message, history: list[tuple[str, str]], system_message, max_tokens, temperature, top_p):
@@ -33,12 +34,13 @@ def respond(message, history: list[tuple[str, str]], system_message, max_tokens,
             temperature=temperature,
             top_p=top_p,
             do_sample=True,
+            pad_token_id=tokenizer.eos_token_id,  # also pass it here to be safe
         )
     
-    # Extract the new tokens
+    # Extract the new tokens (tokens generated after the prompt)
     new_tokens = output_ids[0][input_ids.shape[1]:]
     
-    # Stream output in chunks (e.g., 5 tokens per chunk)
+    # Stream output in chunks (here yielding every 5 tokens)
     chunk_size = 5
     for i in range(0, new_tokens.shape[0], chunk_size):
         current_response = tokenizer.decode(new_tokens[: i + chunk_size], skip_special_tokens=True)