Update app.py

app.py

@@ -7,9 +7,10 @@ import random
 import time
 import uuid
 import websocket
-
+from websocket import WebSocketConnectionClosedException
 import gradio as gr
 import requests
+import logging
 
 from fastchat.conversation import SeparatorStyle
 from fastchat.constants import (
@@ -42,41 +43,40 @@ from fastchat.utils import (
 
 logger = build_logger("gradio_web_server", "gradio_web_server.log")
 
-PRESET_ANSWERS = "刚到美国的时候，觉得美国人像傻子一样，到处都是漏洞。任何地方的厕所都有免费纸，有些人定期去扯很多回家，纸都不用买。快餐店的饮料，有的可以无限续杯，有些几个人买一份饮料，接回来灌到各自的杯子里；等等。尽管美国有许多“漏洞”，但作为超级大国，显然能带给人以无尽的故事与思考。我来分享一下哪些是去了美国才知道的事，主题主要围绕着生活、衣食住行、文化冲击、教育医疗等展开叙说，本文有5千字左右，你也可以跳到感兴趣的部分阅读。美国的城市风貌与基础设施1、去到了美国才知道，纽约的城市样貌跟我想象的发达不一样，真实的纽约街景是很嘈杂和市井。例如，在曼哈顿区路旁，随处可见的小摊位，卖鲜花的、卖各种小食、卖自制首饰的，卖艺术品等等。我留意一下，发现每个路边摊都有合法的营业执照。"
-
-
+no_change_dropdown = gr.Dropdown.update()
+no_change_slider = gr.Slider.update()
+no_change_textbox = gr.Textbox.update()
 no_change_btn = gr.Button.update()
 enable_btn = gr.Button.update(interactive=True)
 disable_btn = gr.Button.update(interactive=False)
 
-# enable_moderation = False
-# concurrency_count = 10
-# model_list_mode = 'reload'
+enable_moderation = False
+concurrency_count = 10
+model_list_mode = 'reload'
+allow_running = 1
 
-# midware_url = "http://159.138.58.253:8080/api/v1/chat/models"
-# chat_token = 'abc'
-# worker_addr = 'http://159.138.58.253:8080/api/v1/chat'
+midware_url = "http://119.8.43.169:8080/api/v1/chat/models"
+worker_addr = 'http://119.8.43.169:8080/api/v1/chat'
+chat_token = 'abc'
+ft_list_job_url = "http://119.8.43.169:8080/api/v1/job"
+ft_submit_job_url = "http://119.8.43.169:8080/api/v1/job"
+ft_remove_job_url = "http://119.8.43.169:8080/api/v1/job/"
+ft_console_log_url = "ws://119.8.43.169:8080/api/v1/log/"
 
-# allow_running = 5
-# ft_list_job_url = "http://49.0.247.41:30139/api/v1/job"
-# ft_submit_job_url = "http://49.0.247.41:30139/api/v1/job"
-# ft_remove_job_url = "http://49.0.247.41:30139/api/v1/job/"
-# ft_console_log_url = "ws://49.0.247.41:30139/api/v1/log/"
 
+# enable_moderation = True if os.environ.get('enable_moderation', default='False')=="True" else False
+# concurrency_count = int(os.environ.get('concurrency_count', default='10'))
+# model_list_mode = os.environ.get('model_list_mode', default='reload')
 
-enable_moderation = True if os.environ.get('enable_moderation', default='False')=="True" else False
-concurrency_count = int(os.environ.get('concurrency_count', default='10'))
-model_list_mode = os.environ.get('model_list_mode', default='reload')
+# midware_url = os.environ.get('midware_url', default='')
+# chat_token = os.environ.get('chat_token', default='')
+# worker_addr = os.environ.get('worker_addr', default='')
 
-midware_url = os.environ['midware_url']
-chat_token = os.environ.get('chat_token', default='')
-worker_addr = os.environ.get('worker_addr', default='')
-
-allow_running = int(os.environ.get('allow_running', default='1'))
-ft_list_job_url = os.environ.get('ft_console_log_url', default='')
-ft_submit_job_url = os.environ.get('ft_console_log_url', default='')
-ft_remove_job_url = os.environ.get('ft_console_log_url', default='')
-ft_console_log_url = os.environ.get('ft_console_log_url', default='')
+# allow_running = int(os.environ.get('allow_running', default='1'))
+# ft_list_job_url = os.environ.get('ft_list_job_url', default='')
+# ft_submit_job_url = os.environ.get('ft_submit_job_url', default='')
+# ft_remove_job_url = os.environ.get('ft_remove_job_url', default='')
+# ft_console_log_url = os.environ.get('ft_console_log_url', default='')
 
 
 headers = {"User-Agent": "FastChat Client", "PRIVATE-TOKEN": chat_token}
@@ -89,6 +89,39 @@ The service is a research preview intended for non-commercial use only, subject
 ip_expiration_dict = defaultdict(lambda: 0)
 
 
+def is_legal_char(c):
+    if c.isalnum():
+        return True
+    if c in "！？｡＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏.":
+        return True
+    if c in '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~':
+        return True
+    return False
+
+
+def str_filter(s):
+    for _ in range(2):
+        if len(s) > 0 and (not is_legal_char(s[-1])):
+            s = s[:-1]
+    return s
+
+
+def str_not_int(s):
+    try:
+        int(s)
+        return False
+    except ValueError:
+        return True
+    
+
+def str_not_float(s):
+    try:
+        float(s)
+        return False
+    except ValueError:
+        return True
+
+
 class State:
     def __init__(self, model_name):
         self.conv = get_conversation_template(model_name)
@@ -122,26 +155,17 @@ def get_conv_log_filename():
 
 
 def get_model_list(midware_url):
-    ret = requests.get(midware_url, headers={"PRIVATE-TOKEN": chat_token})
-    models = ret.json()["data"]
+    try:
+        ret = requests.get(midware_url, headers={"PRIVATE-TOKEN": chat_token}, timeout=5)
+        models = ret.json()["data"]
+    except requests.exceptions.RequestException:
+        models = ["CANNOT GET MODEL"]
 
     priority = {k: f"___{i:02d}" for i, k in enumerate(model_info)}
     models.sort(key=lambda x: priority.get(x, x))
     logger.info(f"Models: {models}")
     return models
 
-df_headers = [
-    "Job Name",
-    "Create By",
-    "Create At",
-    "Model",
-    "Dataset",
-    "Status",
-    "HPs"
-]
-values= [["task111", "Tom", "20230829 14:30", "Vicuna", "cat", "Done", "{\"epochs\": \"1\", \"train_batch_size\": \"2\",\"eval_batch_size\": \"3\" ,\"train_batch_size\": \"2\",\"train_batch_size\": \"2\"}"], 
-["task222", "Jerry", "20230829 15:30", "Vicuna", "dog", "Doing", "{\"train_batch_size\": \"2\", \"train_batch_size\": \"2\",\"train_batch_size\": \"2\" ,\"train_batch_size\": \"2\",\"train_batch_size\": \"2\"}"], 
-["task333", "Somebody", "20230830 15:30", "Vicuna", "cat", "Error", "{\"train_batch_size\": \"2\", \"train_batch_size\": \"2\",\"train_batch_size\": \"2\" ,\"train_batch_size\": \"2\",\"train_batch_size\": \"2\"}"]]
 
 def load_demo_single(models, url_params):
     selected_model = models[0] if len(models) > 0 else ""
@@ -283,9 +307,9 @@ def model_worker_stream_iter(
 ):
     # Make requests
     gen_params = {
-        "model": model_name,
-        "prompt": prompt,
-        "temperature": temperature,
+        "model_name": model_name,
+        "question": prompt,
+        "temperature": 1e-6,
         "repetition_penalty": repetition_penalty,
         "top_p": top_p,
         "max_new_tokens": max_new_tokens,
@@ -384,6 +408,7 @@ def bot_response(state, temperature, top_p, max_new_tokens, request: gr.Request)
                 output = data["text"].strip()
                 if "vicuna" in model_name:
                     output = post_process_code(output)
+                output = str_filter(output)
                 conv.update_last_message(output + "▌")
                 yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5
             else:
@@ -504,42 +529,9 @@ def get_model_description_md(models):
 
 
 def build_single_model_ui(models, add_promotion_links=False):
-    global_notice_markdown = f"""
-# Vicuna runs on Ascend
-## What does this space do
-This is a space that providing a demo for users to try vicuna big model on Ascend 910B hardware. Using this space you can chat/finetune with vicuna.
-## What is changed
-We modified some opensource libraries to make thems run well on Ascend. It includes: fastchat, torch_npu, deepspeed-npu.
-## What is not changed
-1. The Vicuna model is not changed. All the model runs here are from lmsys.
-2. All the libraries are not changed, excepet the ones mentioned above.
-## What hardware are used
-1. This web page is hosted on huggingface wih the free resource(2U16G)
-2. The chat/fietune function is hosted on a Kunpeng920(CPU) + Asend 910B(NPU) machine.
-## Useful link
-- [Ascend home page](https://www.hiascend.com/)
-- [Ascend related library](https://github.com/ascend)
-"""
-
-    dialog_notice_markdown = f"""
-# Chat with Vicuna (Ascend Backended)
-
-### Notice
-This space is originally from [FastChat](https://github.com/lm-sys/FastChat), but the backend computational hardware is Ascend.
-
-### Choose a model to chat with
-"""
-    finetune_notice_markdown = f"""
-# Finetune with Ascend
-### Finetuning with Ascend
-### Access to Finetuning
-Because of the limited computational resources, you will need a token to finetune models. Send an E-mail to [email protected] to apply for a token.
-"""
-    gr.Markdown(global_notice_markdown)
     with gr.Column():
         with gr.Tab("🧠 模型对话 Dialog"):
             state = gr.State()
-            gr.Markdown(dialog_notice_markdown, elem_id="dialog_notice_markdown")
 
             with gr.Row(elem_id="model_selector_row"):
                 model_selector = gr.Dropdown(
@@ -642,134 +634,120 @@ Because of the limited computational resources, you will need a token to finetun
                 [state, temperature, top_p, max_output_tokens],
                 [state, chatbot] + btn_list,
             )
-        with gr.Tab("🎚️ 模型微调 Finetune"):
-            gr.Markdown(finetune_notice_markdown)
-            ft_selected_row_data = gr.State()
-            ft_latest_running_cnt = gr.State()
-            df_headers = [
-                "Job Name",
-                "Create By",
-                "Create At",
-                "Model",
-                "Dataset",
-                "Status",
-                "HPs"
-            ]
-            values= [["task111", "Tom", "20230829 14:30", "Vicuna", "cat", "Done", "{\"epochs\": \"1\", \"train_batch_size\": \"2\",\"eval_batch_size\": \"3\" ,\"train_batch_size\": \"2\",\"train_batch_size\": \"2\"}"], 
-            ["task222", "Jerry", "20230829 15:30", "Vicuna", "dog", "Doing", "{\"train_batch_size\": \"2\", \"train_batch_size\": \"2\",\"train_batch_size\": \"2\" ,\"train_batch_size\": \"2\",\"train_batch_size\": \"2\"}"], 
-            ["task333", "Somebody", "20230830 15:30", "Vicuna", "cat", "Error", "{\"train_batch_size\": \"2\", \"train_batch_size\": \"2\",\"train_batch_size\": \"2\" ,\"train_batch_size\": \"2\",\"train_batch_size\": \"2\"}"]]
-            ft_jobs_info = gr.Dataframe(
-                headers=df_headers,
-                type='array',
-                datatype=["str", "str", "str", "str", "str", "str", "str"],
-                value=values,
-                interactive=False,
-            )
-            with gr.Row():
-                ft_show_btn = gr.Button(value="Show Logs")
-                ft_refresh_btn = gr.Button(value="Refresh")
-                ft_remove_btn = gr.Button(value="Remove Running")
-            with gr.Row():
-                with gr.Column(scale=1):
-                    ft_user_name = gr.Textbox(value="", label="User Name")
-                    ft_model = gr.Dropdown(["vicuna-7b-v1.5-16k"], value="vicuna-7b-v1.5-16k", label="Model", interactive=True)
-                    ft_dataset_name = gr.Dropdown(["cat", "dog", "bird"], value="cat", label="Dataset", interactive=True)
-                    ft_token = gr.Textbox(value="", label="Finetune token")
-                    ft_submit_btn = gr.Button(value="Submit")
-                    ft_cease_btn = gr.Button(value="Cease Streaming")
-                with gr.Column(scale=1):
-                    ft_epochs = gr.Slider(
-                        minimum=1,
-                        maximum=3,
-                        value=3,
-                        step=1,
-                        interactive=True,
-                        label="epochs",
-                    )
-                    ft_train_batch_size = gr.Textbox(value="2", label="train batch size", interactive=True)
-                    ft_eval_batch_size = gr.Textbox(value="2", label="eval batch size", interactive=True)
-                    ft_gradient_accumulation_steps = gr.Textbox(value="16", label="gradient accumulation steps", interactive=True)
-                    ft_learning_rate = gr.Textbox(value="2e-5", label="learning rate", interactive=True)
-                    ft_weight_decay = gr.Textbox(value="0.", label="weight decay", interactive=True)
-                    ft_model_max_length = gr.Textbox(value="1024", label="model max length", interactive=True)
-                with gr.Column(scale=8):
-                    ft_console = gr.Textbox(value="", lines=28, label="Console", interactive=False)
-            ft_jobs_info.select(ft_jobs_info_select, [ft_jobs_info, ft_model, ft_dataset_name, ft_epochs, ft_train_batch_size, ft_eval_batch_size, ft_gradient_accumulation_steps, ft_learning_rate, ft_weight_decay, ft_model_max_length], [ft_selected_row_data, ft_model, ft_dataset_name, ft_epochs, ft_train_batch_size, ft_eval_batch_size, ft_gradient_accumulation_steps, ft_learning_rate, ft_weight_decay, ft_model_max_length])
+        
             
-            ft_show_evt = ft_show_btn.click(ft_show_click, ft_selected_row_data, ft_console)
-            ft_remove_btn.click(ft_remove_click, [ft_selected_row_data, ft_token], ft_console)
-            ft_refresh_btn.click(ft_refresh_click, None, [ft_jobs_info, ft_latest_running_cnt])
-
-            ft_submit_evt = ft_submit_btn.click(ft_submit_click, [ft_latest_running_cnt, ft_user_name, ft_model, ft_dataset_name, ft_token, ft_epochs, ft_train_batch_size, ft_eval_batch_size, ft_gradient_accumulation_steps, ft_learning_rate, ft_weight_decay, ft_model_max_length], [ft_jobs_info, ft_latest_running_cnt, ft_console])
-            ft_cease_btn.click(ft_cease_click, ft_console, ft_console, cancels=[ft_submit_evt, ft_show_evt])
-            
-    return state, model_selector, chatbot, textbox, send_btn, button_row, parameter_row, ft_jobs_info, ft_latest_running_cnt
+    return state, model_selector, chatbot, textbox, send_btn, button_row, parameter_row
 
 
 def ft_get_job_data():
-    response = requests.get(ft_list_job_url)
-    res_lst = []
     running = 0
-    for d in response.json():
-        if isinstance(d['status'], str) and d['status'].lower() == "running":
-            running += 1 
-        res_lst.append([d['jobName'], d['username'], d['created_at'], d['model'], d['dataset'], d['status'], d['parameter']])
+    res_lst = []
+    try:
+        r = requests.get(ft_list_job_url, timeout=8)
+        for d in r.json():
+            if isinstance(d['status'], str) and d['status'].lower() == "running":
+                running += 1 
+            res_lst.append([d['jobName'], d['username'], d['created_at'], d['model'], d['dataset'], d['status'], json.dumps(d['parameter'])])
+        res_lst = sorted(res_lst,key=(lambda x:x[2]), reverse=True)
+        res_lst = sorted(res_lst,key=(lambda x:x[5]), reverse=True)
+    except requests.exceptions.RequestException:
+        logger.info(f"Get job list fail")
+        res_lst.append([])
     return res_lst, running
 
 
 def ft_refresh_click():
     return ft_get_job_data()
 
+
 def ft_cease_click(ft_console):
     output = ft_console + "\n" + "** Streaming output ceased by user **"
     return output
 
+
+def console_generator(addr, sleep_time):
+    total_str = ""
+    ws = websocket.WebSocket()
+    ws.connect(addr)
+    while True:
+        try:
+            new_str = ws.recv()
+            total_str = total_str + new_str
+            time.sleep(sleep_time)
+            yield total_str
+        except WebSocketConnectionClosedException:
+            ws.close()
+            break
+    ws.close()
+
+
 def ft_submit_click(ft_latest_running_cnt, ft_user_name, ft_model, ft_dataset_name, ft_token, ft_epochs, ft_train_batch_size, ft_eval_batch_size, ft_gradient_accumulation_steps, ft_learning_rate, ft_weight_decay, ft_model_max_length):
-    if ft_latest_running_cnt < allow_running:
+    if ft_user_name == "":
+        gr.Warning(f"Submit fail, empty username.")
+        res_lst, running = ft_get_job_data()
+        return res_lst, running, no_change_textbox
+    if str_not_int(ft_train_batch_size) or str_not_int(ft_eval_batch_size) or str_not_int(ft_gradient_accumulation_steps) or str_not_float(ft_learning_rate) or str_not_float(ft_weight_decay) or str_not_int(ft_model_max_length):
+        gr.Warning(f"Submit fail, check the types. [learning rate] and [weight decay] should be float, others HPs should be int.")
+        res_lst, running = ft_get_job_data()
+        return res_lst, running, no_change_textbox
+    if ft_latest_running_cnt < int(allow_running):
         midware_header = {'Content-Type': 'application/json'}
         hps_json = {
             "epochs": str(ft_epochs),
-            "train_batch_size": ft_train_batch_size,
-            "eval_batch_size": ft_eval_batch_size,
-            "gradient_accumulation_steps": ft_gradient_accumulation_steps,
-            "learning_rate": ft_learning_rate,
-            "weight_decay": ft_weight_decay,
-            "model_max_length": ft_model_max_length
+            "train_batch_size": str(ft_train_batch_size),
+            "eval_batch_size": str(ft_eval_batch_size),
+            "gradient_accumulation_steps": str(ft_gradient_accumulation_steps),
+            "learning_rate": str(ft_learning_rate),
+            "weight_decay": str(ft_weight_decay),
+            "model_max_length": str(ft_model_max_length)
         }
         json_data = {
             "dataset": ft_dataset_name,
             "model": ft_model,
-            "parameter": json.dumps(hps_json),
+            "parameter": hps_json,
             "secret": ft_token,
             "username": ft_user_name
         }
-        r = requests.post(ft_submit_job_url, json=json_data, headers=midware_header)
-        gr.Info(f"Job submit success!")
-        res_lst, running = ft_get_job_data()
-        return res_lst, running, json.dumps(json_data) + "\n" + str(r.status_code) + json.dumps(r.json())
+        try:
+            r = requests.post(ft_submit_job_url, json=json_data, headers=midware_header, timeout=120)
+            job_name = r.json()["jobName"]
+            gr.Info(f"Job {job_name} submit success.")
+            res_lst, running = ft_get_job_data()
+            total_str = ""
+            for s in console_generator(ft_console_log_url + job_name, 1):
+                total_str = s
+                yield res_lst, running, s
+            res_lst, running = ft_get_job_data()
+            yield res_lst, running, total_str
+        except requests.exceptions.RequestException:
+            gr.Warning(f"Connection Failure.")
+            res_lst, running = ft_get_job_data()
+            return res_lst, running, ""
     else:
-        gr.Info(f"Only allow {str(allow_running)} job(s) running simultaneously, please wait.")
-        return None
+        gr.Warning(f"Only allow {str(allow_running)} job(s) running simultaneously, please wait.")
+        res_lst, running = ft_get_job_data()
+        return res_lst, running, no_change_textbox
+
 
 def ft_show_click(ft_selected_row_data):
-    s = PRESET_ANSWERS * 10
-    for i in range(1000):
-        yield s[:i*40]
-        time.sleep(0.01)
-    yield s
+    for s in console_generator(ft_console_log_url + ft_selected_row_data[0], 0.2):
+        yield s
+
 
 def ft_remove_click(ft_selected_row_data, ft_token):
     status = ft_selected_row_data[5]
     if isinstance(status, str) and status.lower() == "running":
-        if not ft_token.strip():
-            gr.Info("Remove fail, token needed.")
+        r = requests.delete(ft_remove_job_url + ft_selected_row_data[0], json={"secret": ft_token})
+        if r.status_code == 200:
+            gr.Info("Remove success.")
         else:
-            pass
+            gr.Warning(f"Remove fail. {r.status_code} {r.reason}.")
     else:
-        gr.Info("Remove fail, can only remove a running job.")
-    return ft_selected_row_data[0]
+        gr.Warning("Remove fail. Can only remove a running job.")
+    return ft_get_job_data()
 
-def ft_jobs_info_select(ft_jobs_info, ft_model, ft_dataset_name, ft_epochs, ft_train_batch_size, ft_eval_batch_size, ft_gradient_accumulation_steps, ft_learning_rate, ft_weight_decay, ft_model_max_length, evt: gr.SelectData):
+
+def ft_jobs_info_select(ft_jobs_info, evt: gr.SelectData):
     selected_row = ft_jobs_info[evt.index[0]]
     if evt.index[1] in (3, 4, 6):
         try:
@@ -779,11 +757,12 @@ def ft_jobs_info_select(ft_jobs_info, ft_model, ft_dataset_name, ft_epochs, ft_t
         return [selected_row, selected_row[3], selected_row[4], Hps.get('epochs', ''), Hps.get('train_batch_size', ''), Hps.get('eval_batch_size', ''), 
                 Hps.get('gradient_accumulation_steps', ''), Hps.get('learning_rate', ''), Hps.get('weight_decay', ''), Hps.get('model_max_length', '')]
     else:
-        return [selected_row, ft_model, ft_dataset_name, ft_epochs, ft_train_batch_size, ft_eval_batch_size, ft_gradient_accumulation_steps, ft_learning_rate, ft_weight_decay, ft_model_max_length]
+        return [selected_row, no_change_dropdown, no_change_dropdown, no_change_slider, no_change_textbox, no_change_textbox, no_change_textbox, no_change_textbox, no_change_textbox, no_change_textbox]
+
 
 def build_demo(models):
     with gr.Blocks(
-        title="Chat with Vicuna (Ascend Backended)",
+        title="Vicuna (Ascend Backended)",
         theme=gr.themes.Base(),
         css = block_css
     ) as demo:
@@ -796,8 +775,6 @@ def build_demo(models):
             send_btn,
             button_row,
             parameter_row,
-            ft_jobs_info,
-            ft_latest_running_cnt,
         ) = build_single_model_ui(models)
 
         if model_list_mode not in ["once", "reload"]:
@@ -816,14 +793,6 @@ def build_demo(models):
             ],
             _js=get_window_url_params_js,
         )
-        demo.load(
-            ft_get_job_data,
-            None,
-            [
-                ft_jobs_info,
-                ft_latest_running_cnt,
-            ]
-        )
 
     return demo