Update app.py

app.py

@@ -77,8 +77,8 @@ def respond(
     messages = [{"role": "user", "content": "Hello, how are you?"}]
     input_ids = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_tensors="pt").to('cuda')
 ## <BOS_TOKEN><|START_OF_TURN_TOKEN|><|USER_TOKEN|>Hello, how are you?<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>
-
-    gen_tokens = model.generate(
+    with autocast():
+        gen_tokens = model.generate(
     input_ids, 
     max_new_tokens=100, 
     # do_sample=True,