Update app.py

app.py

@@ -93,11 +93,22 @@ def generate_response(system_instruction, user_input):
         )
     ], return_tensors = "pt").to("cuda")
 
+    meta_config = {
+        "do_sample": True,
+        "temperature": 0.1,
+        "top_p": 0.9,
+        "max_new_tokens": 256,
+        "repetition_penalty": 1.2,
+        "use_cache": True 
+    }
+
+    generation_config = GenerationConfig(**meta_config)
+
     with torch.no_grad():
-        outputs = model.generate(**inputs, max_new_tokens = 1000, use_cache = True)
+        outputs = model.generate(**inputs, generation_config=generation_config)
     
     decoded_output = tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]
-    assistant_response = decoded_output
+    assistant_response = decoded_output.split("### Response:")[-1].strip()
     
     # tokenizer.batch_decode(outputs)
     # # Generate model response