Add application file

app.py

@@ -0,0 +1,170 @@
+import gradio as gr
+import time
+import requests
+import json
+import os
+
+MODEL = "gpt-4-0125-preview"
+API_URL = os.getenv("API_URL")
+API_KEY = os.getenv("API_KEY")
+
+url = f"{API_URL}/v1/chat/completions"
+
+# The headers for the HTTP request
+headers = {
+    "accept": "application/json",
+    "Content-Type": "application/json",
+    "Authorization": f"Bearer {API_KEY}",
+}
+
+
+def is_valid_json(data):
+    try:
+        # Attempt to parse the JSON data
+        parsed_data = json.loads(data)
+        return True, parsed_data
+    except ValueError as e:
+        # If an error occurs, the JSON is not valid
+        return False, str(e)
+
+
+with gr.Blocks() as demo:
+
+    chatbot = gr.Chatbot()
+    msg = gr.Textbox()
+    clear = gr.Button("Clear")
+    with gr.Row():
+
+        with gr.Column(scale=4):
+            # Define inputs for additional parameters
+            system_prompt_input = gr.Textbox(
+                label="System Prompt",
+                placeholder="Type system prompt here...",
+                value="You are a helpful assistant.",
+            )
+            temperature_input = gr.Slider(
+                label="Temperature", minimum=0.0, maximum=1.0, value=0.9, step=0.01
+            )
+            max_new_tokens_input = gr.Slider(
+                label="Max New Tokens", minimum=0, maximum=1024, value=256, step=1
+            )
+            top_p_input = gr.Slider(
+                label="Top P", minimum=0.0, maximum=1.0, value=0.9, step=0.01
+            )
+            repetition_penalty_input = gr.Slider(
+                label="Repetition Penalty",
+                minimum=1.0,
+                maximum=2.0,
+                value=1.2,
+                step=0.01,
+            )
+        with gr.Column(scale=1):
+            markup = gr.Markdown("## Mistral 7B Instruct v0.2 GGUF")
+
+    def update_globals(
+        system_prompt, temperature, max_new_tokens, top_p, repetition_penalty
+    ):
+        global global_system_prompt, global_temperature, global_max_new_tokens, global_top_p, global_repetition_penalty
+        global_system_prompt = system_prompt
+        global_temperature = temperature
+        global_max_new_tokens = max_new_tokens
+        global_top_p = top_p
+        global_repetition_penalty = repetition_penalty
+
+    def user(user_message, history):
+        # print(f"User: {user_message}")
+        # print(f"History: {history}")
+        return "", history + [[user_message, None]]
+
+    def bot(
+        history, system_prompt, temperature, max_new_tokens, top_p, repetition_penalty
+    ):
+        print(f"History in bot: {history}")
+        print(f"System Prompt: {system_prompt}")
+        print(f"Temperature: {temperature}")
+        print(f"Max New Tokens: {max_new_tokens}")
+        print(f"Top P: {top_p}")
+        print(f"Repetition Penalty: {repetition_penalty}")
+
+        # print(f"History in bot: {history}")
+        # [['Capital of France', 'The capital city of France is Paris.'], ['Thansk', 'You are welcome.'], ['What is the capital of France?', '']]
+        # convert this to [['Capital of France', 'The capital city of France is Paris.'], ['Thansk', 'You are welcome.'], ['What is the capital of France?', '']] to list of dict of role user and assiatant
+        history_messages = [{"content": h[0], "role": "user"} for h in history if h[0]]
+        # let's extract the user's question which should be the last touple first element
+        # user_question = history[-1][0]
+        history[-1][1] = ""
+
+        history_messages = system_prompt + history_messages
+        print(history_messages)
+
+        data = {
+            "messages": history_messages,
+            "stream": True,
+            "temprature": temperature,
+            "top_k": 50,
+            "top_p": 0.95,
+            "seed": 42,
+            "repeat_penalty": repetition_penalty,
+            "chat_format": "mistral-instruct",
+            "max_tokens": max_new_tokens,
+            "response_format": {
+                "type": "json_object",
+            },
+        }
+
+        # # Making the POST request and streaming the response
+        response = requests.post(
+            url, headers=headers, data=json.dumps(data), stream=True
+        )
+        for line in response.iter_lines():
+            # Filter out keep-alive new lines
+            if line:
+                data = line.decode("utf-8").lstrip("data: ")
+                # Check if the examples are valid
+                valid_check = is_valid_json(data)
+                if valid_check[0]:
+                    try:
+                        # Attempt to parse the JSON dataa
+                        # json_data = json.loads(data)
+                        json_data = valid_check[1]
+
+                        delta_content = (
+                            json_data.get("choices", [{}])[0]
+                            .get("delta", {})
+                            .get("content", "")
+                        )
+
+                        if delta_content:  # Ensure there's content to print
+                            # print(f"Bot: {delta_content}")
+                            history[-1][1] += delta_content
+                            # print(history)
+                            time.sleep(0.05)
+                            yield history
+                    except json.JSONDecodeError as e:
+                        print(
+                            f"Error decoding JSON: {e} date: {data}"
+                        )  # print(delta_content, flush=True, end="")
+
+                        # print(json_data['choices'][0])
+
+    msg.submit(
+        user, [msg, chatbot], [msg, chatbot], queue=False, concurrency_limit=5
+    ).then(
+        bot,
+        inputs=[
+            chatbot,
+            system_prompt_input,
+            temperature_input,
+            max_new_tokens_input,
+            top_p_input,
+            repetition_penalty_input,
+        ],
+        outputs=chatbot,
+    )
+
+    clear.click(lambda: None, None, chatbot, queue=False)
+
+
+demo.queue()
+if __name__ == "__main__":
+    demo.launch(show_api=False, share=False)