Update app.py

app.py

@@ -1,3 +1,4 @@
+import logging
 import os
 import re
 from time import sleep
@@ -9,6 +10,8 @@ import yaml
 with open("./config.yml", "r") as f:
     config = yaml.load(f, Loader=yaml.Loader)
 
+logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"))
+
 
 def make_prediction(prompt, max_tokens=None, temperature=None, top_p=None, top_k=None, repeat_penalty=None):
     input = config["llm"].copy()
@@ -40,13 +43,18 @@ def make_prediction(prompt, max_tokens=None, temperature=None, top_p=None, top_k
 
 def poll_for_status(task_id):
     url = f"https://api.runpod.ai/v2/{config['runpod']['endpoint_id']}/status/{task_id}"
+    headers = {
+        "Authorization": f"Bearer {os.environ['RUNPOD_AI_API_KEY']}"
+    }
 
     while True:
-        response = requests.get(url)
+        response = requests.get(url, headers=headers)
         if response.status_code == 200:
             data = response.json()
             if data.get('status') == 'COMPLETED':
                 return data["output"]
+        elif response.status_code >= 400:
+            logging.error(response.json())
         # Sleep for 3 seconds between each request
         sleep(3)
 
@@ -58,11 +66,11 @@ def delay_typer(words, delay=0.8):
         sleep(delay)
 
 
-def user(message, history):
+def user(message, nudge_msg, history):
     history = history or []
     # Append the user's message to the conversation history
-    history.append([message, ""])
-    return "", history
+    history.append([message, nudge_msg])
+    return "", nudge_msg, history
 
 
 def chat(history, system_message, max_tokens, temperature, top_p, top_k, repeat_penalty):
@@ -73,9 +81,8 @@ def chat(history, system_message, max_tokens, temperature, top_p, top_k, repeat_
                           for item in history])
 
     # remove last space from assistant, some models output a ZWSP if you leave a space
-    messages = messages[:-1]
+    messages = messages.rstrip()
 
-    history[-1][1] = ""
     prediction = make_prediction(
         messages,
         max_tokens=max_tokens,
@@ -87,9 +94,11 @@ def chat(history, system_message, max_tokens, temperature, top_p, top_k, repeat_
     tokens = re.findall(r'\s*\S+\s*', prediction)
     for s in tokens:
         answer = s
+        print(history)
+        print(history[-1])
         history[-1][1] += answer
         # stream the response
-        yield history, history
+        yield history, history, ""
         sleep(config['typer']['delay'])
 
 
@@ -102,9 +111,8 @@ def rp_chat(history, system_message, max_tokens, temperature, top_p, top_k, repe
                           for item in history])
 
     # remove last space from assistant, some models output a ZWSP if you leave a space
-    messages = messages[:-1]
+    messages = messages.rstrip()
 
-    history[-1][1] = ""
     prediction = make_prediction(
         messages,
         max_tokens=max_tokens,
@@ -118,14 +126,15 @@ def rp_chat(history, system_message, max_tokens, temperature, top_p, top_k, repe
         answer = s
         history[-1][1] += answer
         # stream the response
-        yield history, history
+        yield history, history, ""
         sleep(config['typer']['delay'])
 
 
-def clear_chat(chat_history_state, chat_message):
+def clear_chat(chat_history_state, chat_message, nudge_msg):
     chat_history_state = []
     chat_message = ''
-    return chat_history_state, chat_message
+    nudge_msg = ''
+    return chat_history_state, chat_message, nudge_msg
 
 
 start_message = """
@@ -146,7 +155,6 @@ with gr.Blocks() as demo:
                     - [Duplicate the Space](https://huggingface.co/spaces/openaccess-ai-collective/ggml-runpod-ui?duplicate=true) to skip the queue and run in a private space or to use your own GGML models. You will need to configure you own runpod serverless endpoint.
                     - When using your own models, simply update the [config.yml](https://huggingface.co/spaces/openaccess-ai-collective/ggml-runpod-ui/blob/main/config.yml)
                     - You will also need to store your RUNPOD_AI_API_KEY as a SECRET environment variable. DO NOT STORE THIS IN THE config.yml.
-                    - Contribute at [https://github.com/OpenAccess-AI-Collective/ggml-webui](https://github.com/OpenAccess-AI-Collective/ggml-webui)
                     - Many thanks to [TheBloke](https://huggingface.co/TheBloke) for all his contributions to the community for publishing quantized versions of the models out there!  
                     """)
     with gr.Tab("Chatbot"):
@@ -174,19 +182,22 @@ with gr.Blocks() as demo:
         system_msg = gr.Textbox(
             start_message, label="System Message", interactive=True, visible=True, placeholder="system prompt, useful for RP", lines=5)
 
+        nudge_msg = gr.Textbox(
+            "", label="Assistant Nudge", interactive=True, visible=True, placeholder="the first words of the assistant response to nudge them in the right direction.", lines=1)
+
         chat_history_state = gr.State()
-        clear.click(clear_chat, inputs=[chat_history_state, message], outputs=[chat_history_state, message], queue=False)
+        clear.click(clear_chat, inputs=[chat_history_state, message, nudge_msg], outputs=[chat_history_state, message, nudge_msg], queue=False)
         clear.click(lambda: None, None, chatbot, queue=False)
 
         submit_click_event = submit.click(
-            fn=user, inputs=[message, chat_history_state], outputs=[message, chat_history_state], queue=True
+            fn=user, inputs=[message, nudge_msg, chat_history_state], outputs=[message, nudge_msg, chat_history_state], queue=True
         ).then(
-            fn=chat, inputs=[chat_history_state, system_msg, max_tokens, temperature, top_p, top_k, repeat_penalty], outputs=[chatbot, chat_history_state], queue=True
+            fn=chat, inputs=[chat_history_state, system_msg, max_tokens, temperature, top_p, top_k, repeat_penalty], outputs=[chatbot, chat_history_state, message], queue=True
         )
         roleplay_click_event = roleplay.click(
-            fn=user, inputs=[message, chat_history_state], outputs=[message, chat_history_state], queue=True
+            fn=user, inputs=[message, nudge_msg, chat_history_state], outputs=[message, nudge_msg, chat_history_state], queue=True
         ).then(
-            fn=rp_chat, inputs=[chat_history_state, system_msg, max_tokens, temperature, top_p, top_k, repeat_penalty], outputs=[chatbot, chat_history_state], queue=True
+            fn=rp_chat, inputs=[chat_history_state, system_msg, max_tokens, temperature, top_p, top_k, repeat_penalty], outputs=[chatbot, chat_history_state, message], queue=True
         )
         stop.click(fn=None, inputs=None, outputs=None, cancels=[submit_click_event, roleplay_click_event], queue=False)
 

config.yml

@@ -3,7 +3,7 @@ model_url: https://huggingface.co/openaccess-ai-collective/manticore-13b-chat-py
 typer:
   delay: 0.1
 runpod:
-  endpoint_id: 4k4aom7l1xa64k
+  endpoint_id: jifr1oczbrmr3n
   prefer_async: true
 llm:
   top_k: