Update handler.py

handler.py

@@ -22,8 +22,11 @@ class EndpointHandler:
 
         #for chat in prompts:
             #print(chat)
-        inputs = self.tokenizer(prompt, return_tensors="pt")
-        outputs = self.model.generate(**inputs, max_new_tokens=20)
-        return self.tokenizer.decode(outputs[0], skip_special_tokens=True)
+        encodeds = self.tokenizer.encode(prompt, return_tensors="pt")
+        model_inputs = encodeds.to("cuda")
+        self.model.to("cuda")
+        generated_ids = self.model.generate(model_inputs, max_new_tokens=1000, do_sample=True)
+        decoded = self.tokenizer.decode(generated_ids[0])
+        return decoded