add passsage templates

app.py

@@ -1,173 +1,154 @@
 import gradio as gr
 import os
+import sys
+import json
+import requests
+import random
 
-import aiohttp
-import gradio as gr
-import modelscope_studio as mgr
-from http import HTTPStatus
-from dashscope.api_entities.dashscope_response import Role
-from typing import List, Optional, Tuple, Dict, Any, AsyncGenerator
-from urllib.error import HTTPError
-
-
-default_system = 'Bạn là Trợ lý gia sư AI dạy ngôn ngữ Tiếng Anh, tên là Teacher Bee AI. Bạn được xây dựng bởi Prep Education để hướng dẫn học viên làm bài tập trên nền tảng Prepedu.com.'
-
-History = List[Tuple[str, str]]
-Messages = List[Dict[str, str]]
-
-latex_delimiters = [{
-    "left": "\\(",
-    "right": "\\)",
-    "display": True
-}, {
-    "left": "\\begin\{equation\}",
-    "right": "\\end\{equation\}",
-    "display": True
-}, {
-    "left": "\\begin\{align\}",
-    "right": "\\end\{align\}",
-    "display": True
-}, {
-    "left": "\\begin\{alignat\}",
-    "right": "\\end\{alignat\}",
-    "display": True
-}, {
-    "left": "\\begin\{gather\}",
-    "right": "\\end\{gather\}",
-    "display": True
-}, {
-    "left": "\\begin\{CD\}",
-    "right": "\\end\{CD\}",
-    "display": True
-}, {
-    "left": "\\[",
-    "right": "\\]",
-    "display": True
-}]
-
-
-def clear_session() -> tuple[str, list[Any]]:
-    return '', []
-
-
-def modify_system_session(system: str) -> tuple[str, str, list[Any]]:
-    if system is None or len(system) == 0:
-        system = default_system
-    return system, system, []
-
-
-def history_to_messages(history: History, system: str) -> Messages:
-    messages = [{'role': Role.SYSTEM, 'content': system}]
-    for h in history:
-        messages.append({'role': Role.USER, 'content': h[0].text})
-        messages.append({'role': Role.ASSISTANT, 'content': h[1].text})
-    return messages
-
-
-def messages_to_history(messages: Messages) -> tuple[str, list[list[str]]]:
-    assert messages[0]['role'] == Role.SYSTEM
-    system = messages[0]['content']
-    history = []
-    for q, r in zip(messages[1::2], messages[2::2]):
-        history.append([q['content'], r['content']])
-    return system, history
-
-
-async def model_chat(query: Optional[str], history: Optional[History], system: str, radio: str) -> AsyncGenerator[Tuple[str, str, History, str], None]:
-    if query is None:
-        query = ''
-    if history is None:
-        history = []
-    messages = history_to_messages(history, system)
-    messages.append({'role': Role.USER, 'content': query})
-
-    label_model = radio
-
-    async with aiohttp.ClientSession() as session:
-        async with session.post(
-            url="http://bore.testsprep.online:8082/v1/chat/completions",
-            json={
-                "model": label_model,
-                "messages": messages,
-                "result_format": "message",
-                "stream": True
-            }
-        ) as response:
-            if response.status == HTTPStatus.OK:
-                async for line in response.content:
-                    decoded_line = line.decode('utf-8')
-                    yield '', decoded_line, history, system
-            else:
-                raise ValueError(f"Request failed with status {response.status}")
+API_URL = "http://bore.testsprep.online:8082/v1/chat/completions"
+DISABLED = os.getenv("DISABLED") == 'True'
+BEARER_TOKEN = "ABC@123"
+NUM_THREADS = 16
 
+SYSTEM_PROMPT = """Bạn là Trợ lý gia sư AI dạy ngôn ngữ Tiếng Anh, tên là Teacher Bee AI. Bạn được xây dựng bởi Prep Education để hướng dẫn học viên làm bài tập trên nền tảng Prepedu.com.
+Bạn là một trợ lý thân thiện, tính cách tốt bụng và supportive. Giả sử bạn đang hướng dẫn, giải thích và trả lời câu hỏi cho một đứa trẻ 12 tuổi hoặc ở trình độ ngôn ngữ không cao hơn trình độ của người học."""
 
-def choose_radio(radio, system):
-    chatbot = mgr.Chatbot(label=f'{radio.lower()}')
 
-    if system is None or len(system) == 0:
-        system = default_system
+def exception_handler(exception_type, exception, traceback):
+    print("%s: %s" % (exception_type.__name__, exception))
 
-    return chatbot, system, system, ""
 
+sys.excepthook = exception_handler
+sys.tracebacklimit = 0
 
-def update_other_radios(value, other_radio1, other_radio2):
-    if value == "":
-        if other_radio1 != "":
-            selected = other_radio1
-        else:
-            selected = other_radio2
-        return selected, other_radio1, other_radio2
-    return value, "", ""
 
+def predict(system_prompt, inputs, top_p, temperature, max_tokens, chat_counter, chatbot, history, request: gr.Request):
+    messages = [{"role": "system", "content": system_prompt}] if system_prompt else []
 
-def main():
-    with gr.Blocks() as demo:
-        gr.Markdown("""<center><font size=8>LA: A Party of Foundation Models!</center>""")
-        with gr.Row():
-            model_options = [
-                "PrepAI/LA-llama3.1-8b-16k-instruct_4bit_r32_alpha_16_lr1e-4_3ep-sft-awq",
-                "PrepAI/LA-llama-3.1-Storm-8B-16k-instruct_4bit_r32_alpha_16_lr1e-4_3ep-sft-awq"
-            ]
-            with gr.Row():
-                radio = gr.Radio(choices=model_options, label="LA-In-house Models：", value=model_options[0])
+    headers = {
+        "accept": "application/json",
+        "Authorization": "Bearer Prep@123",
+        "Content-Type": "application/json"
+    }
+
+    print("\n\n")
+    print("="*100)
+    print(f"chat_counter: {chat_counter}")
+    print(f"history: {history}")
+
+    if chat_counter != 0:
+        for i, data in enumerate(history):
+            if i % 2 == 0:
+                role = 'user'
+            else:
+                role = 'assistant'
+
+            messages.append({"role": role, "content": data})
+
+        messages.append({"role": "user", "content": inputs})
+
+        print(f"messages: {messages}")
+        
+        payload = {
+            "model": "LA-SFT",
+            "messages": messages,
+            "do_sample": True,
+            "temperature": temperature,
+            "top_p": top_p,
+            "max_tokens": max_tokens,
+            "n": 1,
+            "stream": True,
+            "presence_penalty": 0,
+            "frequency_penalty": 0,
+        }
+    else:
+        messages.append({"role": "user", "content": inputs})
+        payload = {
+            "model": "LA-SFT",
+            "messages": messages,
+            "do_sample": True,
+            "temperature": temperature,
+            "top_p": top_p,
+            "max_tokens": max_tokens,
+            "n": 1,
+            "stream": True,
+            "presence_penalty": 0,
+            "frequency_penalty": 0,
+        }
+
+    chat_counter += 1
+
+    history.append(inputs)
+    token_counter = 0
+    partial_words = ""
+    counter = 0
+
+    try:
+        if payload:
+            print(f"\n>>> Payload: {payload}")
+            # Gọi API với stream=True
+            response = requests.post(API_URL, headers=headers, json=payload, stream=True)
+
+            for chunk in response.iter_lines():
+                if counter == 0:
+                    counter += 1
+                    continue
+
+                if chunk.decode():
+                    chunk = chunk.decode()
+                    if len(chunk) > 12 and "content" in json.loads(chunk[6:])['choices'][0]['delta']:
+                        partial_words += json.loads(chunk[6:])['choices'][0]["delta"]["content"]
+                        if token_counter == 0:
+                            history.append(" " + partial_words)
+                        else:
+                            history[-1] = partial_words
+
+                        token_counter += 1
+                        yield [(history[i], history[i + 1]) for i in range(0, len(history) - 1, 2)], history, chat_counter, response, gr.update(interactive=False), gr.update(interactive=False)
+                            
+    except Exception as e:
+        print(f'error found: {e}')
+
+    yield [(history[i], history[i + 1]) for i in range(0, len(history) - 1, 2)], history, chat_counter, response, gr.update(interactive=True), gr.update(interactive=True)
+
+
+def reset_textbox():
+    return gr.update(value='', interactive=False), gr.update(interactive=False)
+
+
+title = """<h1 align="center">Learning Assistant In-house Model</h1>"""
+theme = gr.themes.Default(primary_hue="green")
+
+with gr.Blocks(
+        css="""#col_container { margin-left: auto; margin-right: auto;} #chatbot {height: 520px; overflow: auto;}""",
+        theme=theme) as demo:
+    gr.HTML(title)
+
+    with gr.Column(elem_id="col_container", visible=True) as main_block:
+        chatbot = gr.Chatbot(elem_id='chatbot')
+        inputs = gr.Textbox(placeholder="Hi there!", label="Type an input and press Enter")
+        state = gr.State([])
 
         with gr.Row():
-            with gr.Accordion():
-                with gr.Row():
-                    with gr.Column(scale=3):
-                        system_input = gr.Textbox(value=default_system, lines=1, label='System')
-                    with gr.Column(scale=1):
-                        modify_system = gr.Button("🛠️ Set system prompt and clear history", scale=2)
-                    system_state = gr.Textbox(value=default_system, visible=False)
-                chatbot = mgr.Chatbot(label=model_options[0].lower(), latex_delimiters=latex_delimiters)
-                textbox = gr.Textbox(lines=1, label='Input')
-
-                with gr.Row():
-                    clear_history = gr.Button("🧹 Clear history")
-                    sumbit = gr.Button("🚀 Send")
-
-                textbox.submit(model_chat,
-                               inputs=[textbox, chatbot, system_state, radio],
-                               outputs=[textbox, chatbot, system_input])
-
-                sumbit.click(model_chat,
-                             inputs=[textbox, chatbot, system_state, radio],
-                             outputs=[textbox, chatbot, system_input],
-                             concurrency_limit=5)
-                clear_history.click(fn=clear_session,
-                                    inputs=[],
-                                    outputs=[textbox, chatbot])
-                modify_system.click(fn=modify_system_session,
-                                    inputs=[system_input],
-                                    outputs=[system_state, system_input, chatbot])
-
-        radio.change(choose_radio,
-                     inputs=[radio, system_input],
-                     outputs=[chatbot, system_state, system_input, textbox])
-
-    demo.queue(api_open=False, default_concurrency_limit=40)
-    demo.launch(max_threads=40, share=True)
-
-
-if __name__ == "__main__":
-    main()
+            with gr.Column(scale=7):
+                b1 = gr.Button(visible=True)
+            with gr.Column(scale=3):
+                server_status_code = gr.Textbox(label="Status code from OpenAI server")
+
+        system_prompt = gr.Textbox(placeholder="Enter system prompt here", label="System Prompt", value=SYSTEM_PROMPT)
+
+        with gr.Accordion("Parameters", open=False):
+            top_p = gr.Slider(minimum=0, maximum=1.0, value=0.9, step=0.05, interactive=True,
+                              label="Top-p (nucleus sampling)")
+            temperature = gr.Slider(minimum=0, maximum=5.0, value=0.1, step=0.1, interactive=True, label="Temperature")
+            max_tokens = gr.Slider(minimum=0, maximum=16_000, value=4096, step=0.1, interactive=True, label="Max tokens")
+            chat_counter = gr.Number(value=0, visible=False, precision=0)
+
+    inputs.submit(reset_textbox, [], [inputs, b1], queue=False)
+    inputs.submit(predict, [system_prompt, inputs, top_p, temperature, max_tokens, chat_counter, chatbot, state],
+                  [chatbot, state, chat_counter, server_status_code, inputs, b1])
+    b1.click(reset_textbox, [], [inputs, b1], queue=False)
+    b1.click(predict, [system_prompt, inputs, top_p, temperature, max_tokens, chat_counter, chatbot, state],
+             [chatbot, state, chat_counter, server_status_code, inputs, b1])
+
+    demo.queue(max_size=10, default_concurrency_limit=NUM_THREADS, api_open=False).launch(share=False)

requirements.txt

@@ -1,3 +0,0 @@
-modelscope_studio
-gradio
-dashscope