unqueue the queue + random shit fixes idk

app.py

@@ -4,9 +4,10 @@ import json
 import threading
 import os
 import datetime
+import queue
+import time
 from requests.exceptions import RequestException
 
-stop_generation = threading.Event()
 API_URL = os.environ.get('API_URL')
 API_KEY = os.environ.get('API_KEY')
 
@@ -15,8 +16,6 @@ headers = {
     "Content-Type": "application/json"
 }
 
-session = requests.Session()
-
 DEFAULT_PARAMS = {
     "temperature": 0.8,
     "top_p": 0.95,
@@ -27,12 +26,20 @@ DEFAULT_PARAMS = {
     "max_tokens": 512
 }
 
+class ThreadLocalStorage:
+    def __init__(self):
+        self.stop_generation = False
+        self.active_requests = set()
+        self.lock = threading.Lock()
+
+thread_local = ThreadLocalStorage()
+
 def get_timestamp():
     return datetime.datetime.now().strftime("%H:%M:%S")
 
 def predict(message, history, system_prompt, temperature, top_p, top_k, frequency_penalty, presence_penalty, repetition_penalty, max_tokens):
-    global stop_generation, session
-    stop_generation.clear()  
+    with thread_local.lock:
+        thread_local.stop_generation = False
 
     history_format = [{"role": "system", "content": system_prompt}]
     for human, assistant in history:
@@ -56,7 +63,7 @@ def predict(message, history, system_prompt, temperature, top_p, top_k, frequenc
     }
 
     non_default_params = {k: v for k, v in current_params.items() if v != DEFAULT_PARAMS[k]}
-    
+
     if non_default_params and not message.startswith(('*', '"')):
         for param, value in non_default_params.items():
             print(f"{param}={value}")
@@ -75,12 +82,14 @@ def predict(message, history, system_prompt, temperature, top_p, top_k, frequenc
     }
 
     try:
-        with session.post(API_URL, headers=headers, data=json.dumps(data), stream=True) as response:
+        with requests.post(API_URL, headers=headers, data=json.dumps(data), stream=True) as response:
+            with thread_local.lock:
+                thread_local.active_requests.add(response)
             partial_message = ""
             for line in response.iter_lines():
-                if stop_generation.is_set():
-                    response.close()
-                    break
+                with thread_local.lock:
+                    if thread_local.stop_generation:
+                        return partial_message  
                 if line:
                     line = line.decode('utf-8')
                     if line.startswith("data: "):
@@ -100,8 +109,16 @@ def predict(message, history, system_prompt, temperature, top_p, top_k, frequenc
             yield partial_message
 
     except RequestException as e:
-        print(f"Request error: {e}")
-        yield f"An error occurred: {str(e)}"
+        error_message = f"Request error: {str(e)}"
+        print(error_message)
+        yield error_message
+    except Exception as e:
+        error_message = f"Unexpected error: {str(e)}"
+        print(error_message)
+        yield error_message
+    finally:
+        with thread_local.lock:
+            thread_local.active_requests.discard(response)
 
 def import_chat(custom_format_string):
     try:
@@ -138,12 +155,18 @@ def export_chat(history, system_prompt):
     return export_data
 
 def stop_generation_func():
-    global stop_generation, session
-    stop_generation.set()
-    session.close()  
-    session = requests.Session()  
-
-with gr.Blocks(theme='gradio/monochrome') as demo:
+    with thread_local.lock:
+        thread_local.stop_generation = True
+        for request in thread_local.active_requests:
+            try:
+                request.close()
+            except Exception as e:
+                print(f"Error closing request: {str(e)}")
+        thread_local.active_requests.clear()
+    time.sleep(0.1)  
+    return gr.update(), gr.update()  
+
+with gr.Blocks(theme=gr.themes.Monochrome()) as demo:
     with gr.Row():
         with gr.Column(scale=2):
             chatbot = gr.Chatbot(value=[])
@@ -174,20 +197,22 @@ with gr.Blocks(theme='gradio/monochrome') as demo:
         return "", history + [[user_message, None]]
 
     def bot(history, system_prompt, temperature, top_p, top_k, frequency_penalty, presence_penalty, repetition_penalty, max_tokens):
-        global stop_generation
         history = history or []
         if not history:
             return history
         user_message = history[-1][0]
         bot_message = predict(user_message, history[:-1], system_prompt, temperature, top_p, top_k, frequency_penalty, presence_penalty, repetition_penalty, max_tokens)
         history[-1][1] = ""
-        for chunk in bot_message:
-            if stop_generation.is_set():
-                history[-1][1] += " [Generation stopped]"
-                break
-            history[-1][1] = chunk
+        try:
+            for chunk in bot_message:
+                if thread_local.stop_generation:
+                    break
+                history[-1][1] = chunk
+                yield history
+        except Exception as e:
+            print(f"Error in bot function: {str(e)}")
+            history[-1][1] = "An error occurred while generating the response."
             yield history
-        stop_generation.clear()
 
     def regenerate_response(history, system_prompt, temperature, top_p, top_k, frequency_penalty, presence_penalty, repetition_penalty, max_tokens):
         if history and len(history) > 0:
@@ -202,18 +227,24 @@ with gr.Blocks(theme='gradio/monochrome') as demo:
         imported_history, imported_system_prompt = import_chat(custom_format_string)
         return imported_history, imported_system_prompt
 
-    msg.submit(user, [msg, chatbot], [msg, chatbot], queue=False).then(
+    submit_event = msg.submit(user, [msg, chatbot], [msg, chatbot]).then(
         bot, [chatbot, system_prompt, temperature, top_p, top_k, frequency_penalty, presence_penalty, repetition_penalty, max_tokens], chatbot
     )
 
-    clear.click(lambda: None, None, chatbot, queue=False)
+    clear.click(lambda: None, None, chatbot)
 
-    regenerate.click(
+    regenerate_event = regenerate.click(
         regenerate_response,
         [chatbot, system_prompt, temperature, top_p, top_k, frequency_penalty, presence_penalty, repetition_penalty, max_tokens],
         chatbot
     )
 
+    stop_btn.click(
+        stop_generation_func,
+        inputs=[],
+        outputs=[chatbot, msg]
+    )
+
     import_button.click(import_chat_wrapper, inputs=[import_textbox], outputs=[chatbot, system_prompt])
 
     export_button.click(
@@ -222,7 +253,5 @@ with gr.Blocks(theme='gradio/monochrome') as demo:
         outputs=[import_textbox]
     )
 
-    stop_btn.click(stop_generation_func, inputs=[], outputs=[])
-
 if __name__ == "__main__":
-    demo.launch(debug=True)
+    demo.launch(debug=True, server_name="0.0.0.0", server_port=7860, share=True)