Update app.py

app.py

@@ -4,13 +4,13 @@ from huggingface_hub import InferenceClient
 client = InferenceClient(model="https://zgg3nzdpswxy4a-80.proxy.runpod.net")
 
 def inference(message, history):
-    return client.text_generation(prompt=message, details=True, max_new_tokens=256, stream=False, 
-                                        temperature=0.5, do_sample=True, top_p=0.9)
-    # partial_message = ""
-    # for token in client.text_generation(message, max_new_tokens=256, stream=False, 
-    #                                     temperature=0.5, do_sample=True, top_p=0.9):
-    #     partial_message += token
-    #     yield partial_message
+    partial_message = ""
+    for token in client.text_generation(prompt=message, max_new_tokens=512, stream=True, best_of=1, temperature=0.1, watermark=True, 
+                                        top_p=0.95, do_sample=True, repetition_penalty=1.03):
+        if token.startswith("<s>"):
+            return partial_message
+        partial_message += token
+        yield partial_message
 
 gr.ChatInterface(
     inference,