Duplicate from ascend-ai/vicuna-on-ascend

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,14 @@
+---
+title: Vicuna On Ascend
+emoji: 🏃
+colorFrom: red
+colorTo: blue
+sdk: gradio
+sdk_version: 3.41.2
+app_file: app.py
+pinned: false
+license: apache-2.0
+duplicated_from: ascend-ai/vicuna-on-ascend
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,839 @@
+import argparse
+from collections import defaultdict
+import datetime
+import json
+import os
+import random
+import time
+import uuid
+import websocket
+
+import gradio as gr
+import requests
+
+from fastchat.conversation import SeparatorStyle
+from fastchat.constants import (
+    LOGDIR,
+    WORKER_API_TIMEOUT,
+    ErrorCode,
+    MODERATION_MSG,
+    CONVERSATION_LIMIT_MSG,
+    SERVER_ERROR_MSG,
+    INACTIVE_MSG,
+    INPUT_CHAR_LEN_LIMIT,
+    CONVERSATION_TURN_LIMIT,
+    SESSION_EXPIRATION_TIME,
+)
+from fastchat.model.model_adapter import get_conversation_template
+from fastchat.model.model_registry import model_info
+from fastchat.serve.api_provider import (
+    anthropic_api_stream_iter,
+    openai_api_stream_iter,
+    palm_api_stream_iter,
+    init_palm_chat,
+)
+from fastchat.utils import (
+    build_logger,
+    violates_moderation,
+    get_window_url_params_js,
+    parse_gradio_auth_creds,
+)
+
+
+logger = build_logger("gradio_web_server", "gradio_web_server.log")
+
+PRESET_ANSWERS = "刚到美国的时候，觉得美国人像傻子一样，到处都是漏洞。任何地方的厕所都有免费纸，有些人定期去扯很多回家，纸都不用买。快餐店的饮料，有的可以无限续杯，有些几个人买一份饮料，接回来灌到各自的杯子里；等等。尽管美国有许多“漏洞”，但作为超级大国，显然能带给人以无尽的故事与思考。我来分享一下哪些是去了美国才知道的事，主题主要围绕着生活、衣食住行、文化冲击、教育医疗等展开叙说，本文有5千字左右，你也可以跳到感兴趣的部分阅读。美国的城市风貌与基础设施1、去到了美国才知道，纽约的城市样貌跟我想象的发达不一样，真实的纽约街景是很嘈杂和市井。例如，在曼哈顿区路旁，随处可见的小摊位，卖鲜花的、卖各种小食、卖自制首饰的，卖艺术品等等。我留意一下，发现每个路边摊都有合法的营业执照。"
+
+
+no_change_btn = gr.Button.update()
+enable_btn = gr.Button.update(interactive=True)
+disable_btn = gr.Button.update(interactive=False)
+
+# enable_moderation = False
+# concurrency_count = 10
+# model_list_mode = 'reload'
+
+# midware_url = "http://159.138.58.253:8080/api/v1/chat/models"
+# chat_token = 'abc'
+# worker_addr = 'http://159.138.58.253:8080/api/v1/chat'
+
+# allow_running = 5
+# ft_list_job_url = "http://49.0.247.41:30139/api/v1/job"
+# ft_submit_job_url = "http://49.0.247.41:30139/api/v1/job"
+# ft_remove_job_url = "http://49.0.247.41:30139/api/v1/job/"
+# ft_console_log_url = "ws://49.0.247.41:30139/api/v1/log/"
+
+
+enable_moderation = True if os.environ.get('enable_moderation', default='False')=="True" else False
+concurrency_count = int(os.environ.get('concurrency_count', default='10'))
+model_list_mode = os.environ.get('model_list_mode', default='reload')
+
+midware_url = os.environ['midware_url']
+chat_token = os.environ.get('chat_token', default='')
+worker_addr = os.environ.get('worker_addr', default='')
+
+allow_running = int(os.environ.get('allow_running', default='1'))
+ft_list_job_url = os.environ.get('ft_console_log_url', default='')
+ft_submit_job_url = os.environ.get('ft_console_log_url', default='')
+ft_remove_job_url = os.environ.get('ft_console_log_url', default='')
+ft_console_log_url = os.environ.get('ft_console_log_url', default='')
+
+
+headers = {"User-Agent": "FastChat Client", "PRIVATE-TOKEN": chat_token}
+
+learn_more_md = """
+### License
+The service is a research preview intended for non-commercial use only, subject to the model [License](https://github.com/facebookresearch/llama/blob/main/LICENSE) of LLaMA, [Terms of Use](https://openai.com/policies/terms-of-use) of the data generated by OpenAI, and [Privacy Practices](https://chrome.google.com/webstore/detail/sharegpt-share-your-chatg/daiacboceoaocpibfodeljbdfacokfjb) of ShareGPT. Please contact us if you find any potential violation.
+"""
+
+ip_expiration_dict = defaultdict(lambda: 0)
+
+
+class State:
+    def __init__(self, model_name):
+        self.conv = get_conversation_template(model_name)
+        self.conv_id = uuid.uuid4().hex
+        self.skip_next = False
+        self.model_name = model_name
+
+        if model_name == "palm-2":
+            # According to release note, "chat-bison@001" is PaLM 2 for chat.
+            # https://cloud.google.com/vertex-ai/docs/release-notes#May_10_2023
+            self.palm_chat = init_palm_chat("chat-bison@001")
+
+    def to_gradio_chatbot(self):
+        return self.conv.to_gradio_chatbot()
+
+    def dict(self):
+        base = self.conv.dict()
+        base.update(
+            {
+                "conv_id": self.conv_id,
+                "model_name": self.model_name,
+            }
+        )
+        return base
+
+
+def get_conv_log_filename():
+    t = datetime.datetime.now()
+    name = os.path.join(LOGDIR, f"{t.year}-{t.month:02d}-{t.day:02d}-conv.json")
+    return name
+
+
+def get_model_list(midware_url):
+    ret = requests.get(midware_url, headers={"PRIVATE-TOKEN": chat_token})
+    models = ret.json()["data"]
+
+    priority = {k: f"___{i:02d}" for i, k in enumerate(model_info)}
+    models.sort(key=lambda x: priority.get(x, x))
+    logger.info(f"Models: {models}")
+    return models
+
+df_headers = [
+    "Job Name",
+    "Create By",
+    "Create At",
+    "Model",
+    "Dataset",
+    "Status",
+    "HPs"
+]
+values= [["task111", "Tom", "20230829 14:30", "Vicuna", "cat", "Done", "{\"epochs\": \"1\", \"train_batch_size\": \"2\",\"eval_batch_size\": \"3\" ,\"train_batch_size\": \"2\",\"train_batch_size\": \"2\"}"], 
+["task222", "Jerry", "20230829 15:30", "Vicuna", "dog", "Doing", "{\"train_batch_size\": \"2\", \"train_batch_size\": \"2\",\"train_batch_size\": \"2\" ,\"train_batch_size\": \"2\",\"train_batch_size\": \"2\"}"], 
+["task333", "Somebody", "20230830 15:30", "Vicuna", "cat", "Error", "{\"train_batch_size\": \"2\", \"train_batch_size\": \"2\",\"train_batch_size\": \"2\" ,\"train_batch_size\": \"2\",\"train_batch_size\": \"2\"}"]]
+
+def load_demo_single(models, url_params):
+    selected_model = models[0] if len(models) > 0 else ""
+    if "model" in url_params:
+        model = url_params["model"]
+        if model in models:
+            selected_model = model
+
+    dropdown_update = gr.Dropdown.update(
+        choices=models, value=selected_model, visible=True
+    )
+
+    state = None
+    return (
+        state,
+        dropdown_update,
+        gr.Chatbot.update(visible=True),
+        gr.Textbox.update(visible=True),
+        gr.Button.update(visible=True),
+        gr.Row.update(visible=True),
+        gr.Accordion.update(visible=True),
+    )
+
+
+def load_demo(url_params, request: gr.Request):
+    global models
+
+    ip = request.client.host
+    logger.info(f"load_demo. ip: {ip}. params: {url_params}")
+    ip_expiration_dict[ip] = time.time() + SESSION_EXPIRATION_TIME
+
+    if model_list_mode == "reload":
+        models = get_model_list(midware_url)
+
+    return load_demo_single(models, url_params)
+
+
+def vote_last_response(state, vote_type, model_selector, request: gr.Request):
+    with open(get_conv_log_filename(), "a") as fout:
+        data = {
+            "tstamp": round(time.time(), 4),
+            "type": vote_type,
+            "model": model_selector,
+            "state": state.dict(),
+            "ip": request.client.host,
+        }
+        fout.write(json.dumps(data) + "\n")
+
+
+def upvote_last_response(state, model_selector, request: gr.Request):
+    logger.info(f"upvote. ip: {request.client.host}")
+    vote_last_response(state, "upvote", model_selector, request)
+    return ("",) + (disable_btn,) * 3
+
+
+def downvote_last_response(state, model_selector, request: gr.Request):
+    logger.info(f"downvote. ip: {request.client.host}")
+    vote_last_response(state, "downvote", model_selector, request)
+    return ("",) + (disable_btn,) * 3
+
+
+def flag_last_response(state, model_selector, request: gr.Request):
+    logger.info(f"flag. ip: {request.client.host}")
+    vote_last_response(state, "flag", model_selector, request)
+    return ("",) + (disable_btn,) * 3
+
+
+def regenerate(state, request: gr.Request):
+    logger.info(f"regenerate. ip: {request.client.host}")
+    state.conv.update_last_message(None)
+    return (state, state.to_gradio_chatbot(), "") + (disable_btn,) * 5
+
+
+def clear_history(request: gr.Request):
+    logger.info(f"clear_history. ip: {request.client.host}")
+    state = None
+    return (state, [], "") + (disable_btn,) * 5
+
+
+def add_text(state, model_selector, text, request: gr.Request):
+    ip = request.client.host
+    logger.info(f"add_text. ip: {ip}. len: {len(text)}")
+
+    if state is None:
+        state = State(model_selector)
+
+    if len(text) <= 0:
+        state.skip_next = True
+        return (state, state.to_gradio_chatbot(), "") + (no_change_btn,) * 5
+
+    if ip_expiration_dict[ip] < time.time():
+        logger.info(f"inactive. ip: {request.client.host}. text: {text}")
+        state.skip_next = True
+        return (state, state.to_gradio_chatbot(), INACTIVE_MSG) + (no_change_btn,) * 5
+
+    if enable_moderation:
+        flagged = violates_moderation(text)
+        if flagged:
+            logger.info(f"violate moderation. ip: {request.client.host}. text: {text}")
+            state.skip_next = True
+            return (state, state.to_gradio_chatbot(), MODERATION_MSG) + (
+                no_change_btn,
+            ) * 5
+
+    conv = state.conv
+    if (len(conv.messages) - conv.offset) // 2 >= CONVERSATION_TURN_LIMIT:
+        logger.info(f"conversation turn limit. ip: {request.client.host}. text: {text}")
+        state.skip_next = True
+        return (state, state.to_gradio_chatbot(), CONVERSATION_LIMIT_MSG) + (
+            no_change_btn,
+        ) * 5
+
+    text = text[:INPUT_CHAR_LEN_LIMIT]  # Hard cut-off
+    conv.append_message(conv.roles[0], text)
+    conv.append_message(conv.roles[1], None)
+    return (state, state.to_gradio_chatbot(), "") + (disable_btn,) * 5
+
+
+def post_process_code(code):
+    sep = "\n```"
+    if sep in code:
+        blocks = code.split(sep)
+        if len(blocks) % 2 == 1:
+            for i in range(1, len(blocks), 2):
+                blocks[i] = blocks[i].replace("\\_", "_")
+        code = sep.join(blocks)
+    return code
+
+
+def model_worker_stream_iter(
+    conv,
+    model_name,
+    worker_addr,
+    prompt,
+    temperature,
+    repetition_penalty,
+    top_p,
+    max_new_tokens,
+):
+    # Make requests
+    gen_params = {
+        "model": model_name,
+        "prompt": prompt,
+        "temperature": temperature,
+        "repetition_penalty": repetition_penalty,
+        "top_p": top_p,
+        "max_new_tokens": max_new_tokens,
+        "stop": conv.stop_str,
+        "stop_token_ids": conv.stop_token_ids,
+        "echo": False,
+    }
+    logger.info(f"==== request ====\n{gen_params}")
+
+    # Stream output
+    response = requests.post(
+        worker_addr,
+        headers=headers,
+        json=gen_params,
+        stream=True,
+        timeout=WORKER_API_TIMEOUT,
+    )
+    for chunk in response.iter_lines(decode_unicode=False, delimiter=b"\0"):
+        if chunk:
+            data = json.loads(chunk.decode())
+            yield data
+
+
+def bot_response(state, temperature, top_p, max_new_tokens, request: gr.Request):
+    logger.info(f"bot_response. ip: {request.client.host}")
+    start_tstamp = time.time()
+    temperature = float(temperature)
+    top_p = float(top_p)
+    max_new_tokens = int(max_new_tokens)
+
+    if state.skip_next:
+        # This generate call is skipped due to invalid inputs
+        state.skip_next = False
+        yield (state, state.to_gradio_chatbot()) + (no_change_btn,) * 5
+        return
+
+    conv, model_name = state.conv, state.model_name
+    if model_name == "gpt-3.5-turbo" or model_name == "gpt-4":
+        prompt = conv.to_openai_api_messages()
+        stream_iter = openai_api_stream_iter(
+            model_name, prompt, temperature, top_p, max_new_tokens
+        )
+    elif model_name == "claude-2" or model_name == "claude-instant-1":
+        prompt = conv.get_prompt()
+        stream_iter = anthropic_api_stream_iter(
+            model_name, prompt, temperature, top_p, max_new_tokens
+        )
+    elif model_name == "palm-2":
+        stream_iter = palm_api_stream_iter(
+            state.palm_chat, conv.messages[-2][1], temperature, top_p, max_new_tokens
+        )
+    else:
+        # Get worker address
+        logger.info(f"model_name: {model_name}, worker_addr: {worker_addr}")
+        # No available worker
+        if worker_addr == "":
+            conv.update_last_message(SERVER_ERROR_MSG)
+            yield (
+                state,
+                state.to_gradio_chatbot(),
+                disable_btn,
+                disable_btn,
+                disable_btn,
+                enable_btn,
+                enable_btn,
+            )
+            return
+
+        # Construct prompt.
+        # We need to call it here, so it will not be affected by "▌".
+        prompt = conv.get_prompt()
+
+        # Set repetition_penalty
+        if "t5" in model_name:
+            repetition_penalty = 1.2
+        else:
+            repetition_penalty = 1.0
+
+        stream_iter = model_worker_stream_iter(
+            conv,
+            model_name,
+            worker_addr,
+            prompt,
+            temperature,
+            repetition_penalty,
+            top_p,
+            max_new_tokens,
+        )
+
+    conv.update_last_message("▌")
+    yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5
+
+    try:
+        for data in stream_iter:
+            if data["error_code"] == 0:
+                output = data["text"].strip()
+                if "vicuna" in model_name:
+                    output = post_process_code(output)
+                conv.update_last_message(output + "▌")
+                yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5
+            else:
+                output = data["text"] + f"\n\n(error_code: {data['error_code']})"
+                conv.update_last_message(output)
+                yield (state, state.to_gradio_chatbot()) + (
+                    disable_btn,
+                    disable_btn,
+                    disable_btn,
+                    enable_btn,
+                    enable_btn,
+                )
+                return
+            time.sleep(0.015)
+    except requests.exceptions.RequestException as e:
+        conv.update_last_message(
+            f"{SERVER_ERROR_MSG}\n\n"
+            f"(error_code: {ErrorCode.GRADIO_REQUEST_ERROR}, {e})"
+        )
+        yield (state, state.to_gradio_chatbot()) + (
+            disable_btn,
+            disable_btn,
+            disable_btn,
+            enable_btn,
+            enable_btn,
+        )
+        return
+    except Exception as e:
+        conv.update_last_message(
+            f"{SERVER_ERROR_MSG}\n\n"
+            f"(error_code: {ErrorCode.GRADIO_STREAM_UNKNOWN_ERROR}, {e})"
+        )
+        yield (state, state.to_gradio_chatbot()) + (
+            disable_btn,
+            disable_btn,
+            disable_btn,
+            enable_btn,
+            enable_btn,
+        )
+        return
+
+    # Delete "▌"
+    conv.update_last_message(conv.messages[-1][-1][:-1])
+    yield (state, state.to_gradio_chatbot()) + (enable_btn,) * 5
+
+    finish_tstamp = time.time()
+    logger.info(f"{output}")
+
+    with open(get_conv_log_filename(), "a") as fout:
+        data = {
+            "tstamp": round(finish_tstamp, 4),
+            "type": "chat",
+            "model": model_name,
+            "gen_params": {
+                "temperature": temperature,
+                "top_p": top_p,
+                "max_new_tokens": max_new_tokens,
+            },
+            "start": round(start_tstamp, 4),
+            "finish": round(finish_tstamp, 4),
+            "state": state.dict(),
+            "ip": request.client.host,
+        }
+        fout.write(json.dumps(data) + "\n")
+
+
+block_css = """
+#dialog_notice_markdown {
+    font-size: 104%
+}
+#dialog_notice_markdown th {
+    display: none;
+}
+#dialog_notice_markdown td {
+    padding-top: 6px;
+    padding-bottom: 6px;
+}
+#leaderboard_markdown {
+    font-size: 104%
+}
+#leaderboard_markdown td {
+    padding-top: 6px;
+    padding-bottom: 6px;
+}
+#leaderboard_dataframe td {
+    line-height: 0.1em;
+}
+"""
+
+
+def get_model_description_md(models):
+    model_description_md = """
+| | | |
+| ---- | ---- | ---- |
+"""
+    ct = 0
+    visited = set()
+    for i, name in enumerate(models):
+        if name in model_info:
+            minfo = model_info[name]
+            if minfo.simple_name in visited:
+                continue
+            visited.add(minfo.simple_name)
+            one_model_md = f"[{minfo.simple_name}]({minfo.link}): {minfo.description}"
+        else:
+            visited.add(name)
+            one_model_md = (
+                f"[{name}](): Add the description at fastchat/model/model_registry.py"
+            )
+
+        if ct % 3 == 0:
+            model_description_md += "|"
+        model_description_md += f" {one_model_md} |"
+        if ct % 3 == 2:
+            model_description_md += "\n"
+        ct += 1
+    return model_description_md
+
+
+def build_single_model_ui(models, add_promotion_links=False):
+    global_notice_markdown = f"""
+# Vicuna runs on Ascend
+## What does this space do
+This is a space that providing a demo for users to try vicuna big model on Ascend 910B hardware. Using this space you can chat/finetune with vicuna.
+## What is changed
+We modified some opensource libraries to make thems run well on Ascend. It includes: fastchat, torch_npu, deepspeed-npu.
+## What is not changed
+1. The Vicuna model is not changed. All the model runs here are from lmsys.
+2. All the libraries are not changed, excepet the ones mentioned above.
+## What hardware are used
+1. This web page is hosted on huggingface wih the free resource(2U16G)
+2. The chat/fietune function is hosted on a Kunpeng920(CPU) + Asend 910B(NPU) machine.
+## Useful link
+- [Ascend home page](https://www.hiascend.com/)
+- [Ascend related library](https://github.com/ascend)
+"""
+
+    dialog_notice_markdown = f"""
+# Chat with Vicuna (Ascend Backended)
+
+### Notice
+This space is originally from [FastChat](https://github.com/lm-sys/FastChat), but the backend computational hardware is Ascend.
+
+### Choose a model to chat with
+"""
+    finetune_notice_markdown = f"""
+# Finetune with Ascend
+### Finetuning with Ascend
+### Access to Finetuning
+Because of the limited computational resources, you will need a token to finetune models. Send an E-mail to [email protected] to apply for a token.
+"""
+    gr.Markdown(global_notice_markdown)
+    with gr.Column():
+        with gr.Tab("🧠 模型对话 Dialog"):
+            state = gr.State()
+            gr.Markdown(dialog_notice_markdown, elem_id="dialog_notice_markdown")
+
+            with gr.Row(elem_id="model_selector_row"):
+                model_selector = gr.Dropdown(
+                    choices=models,
+                    value=models[0] if len(models) > 0 else "",
+                    interactive=True,
+                    show_label=False,
+                    container=False,
+                )
+
+            chatbot = gr.Chatbot(
+                elem_id="chatbot",
+                label="Scroll down and start chatting",
+                visible=False,
+                height=550,
+            )
+            with gr.Row():
+                with gr.Column(scale=20):
+                    textbox = gr.Textbox(
+                        show_label=False,
+                        placeholder="Enter text and press ENTER",
+                        visible=False,
+                        container=False,
+                    )
+                with gr.Column(scale=1, min_width=50):
+                    send_btn = gr.Button(value="Send", visible=False)
+
+            with gr.Row(visible=False) as button_row:
+                upvote_btn = gr.Button(value="👍  Upvote", interactive=False)
+                downvote_btn = gr.Button(value="👎  Downvote", interactive=False)
+                flag_btn = gr.Button(value="⚠️  Flag", interactive=False)
+                regenerate_btn = gr.Button(value="🔄  Regenerate", interactive=False)
+                clear_btn = gr.Button(value="🗑️  Clear history", interactive=False)
+
+            with gr.Accordion("Parameters", open=False, visible=False) as parameter_row:
+                temperature = gr.Slider(
+                    minimum=0.0,
+                    maximum=1.0,
+                    value=0.7,
+                    step=0.1,
+                    interactive=True,
+                    label="Temperature",
+                )
+                top_p = gr.Slider(
+                    minimum=0.0,
+                    maximum=1.0,
+                    value=1.0,
+                    step=0.1,
+                    interactive=True,
+                    label="Top P",
+                )
+                max_output_tokens = gr.Slider(
+                    minimum=16,
+                    maximum=1024,
+                    value=512,
+                    step=64,
+                    interactive=True,
+                    label="Max output tokens",
+                )
+
+            gr.Markdown(learn_more_md)
+
+            # Register listeners
+            btn_list = [upvote_btn, downvote_btn, flag_btn, regenerate_btn, clear_btn]
+            upvote_btn.click(
+                upvote_last_response,
+                [state, model_selector],
+                [textbox, upvote_btn, downvote_btn, flag_btn],
+            )
+            downvote_btn.click(
+                downvote_last_response,
+                [state, model_selector],
+                [textbox, upvote_btn, downvote_btn, flag_btn],
+            )
+            flag_btn.click(
+                flag_last_response,
+                [state, model_selector],
+                [textbox, upvote_btn, downvote_btn, flag_btn],
+            )
+            regenerate_btn.click(regenerate, state, [state, chatbot, textbox] + btn_list).then(
+                bot_response,
+                [state, temperature, top_p, max_output_tokens],
+                [state, chatbot] + btn_list,
+            )
+            clear_btn.click(clear_history, None, [state, chatbot, textbox] + btn_list)
+
+            model_selector.change(clear_history, None, [state, chatbot, textbox] + btn_list)
+
+            textbox.submit(
+                add_text, [state, model_selector, textbox], [state, chatbot, textbox] + btn_list
+            ).then(
+                bot_response,
+                [state, temperature, top_p, max_output_tokens],
+                [state, chatbot] + btn_list,
+            )
+            send_btn.click(
+                add_text, [state, model_selector, textbox], [state, chatbot, textbox] + btn_list
+            ).then(
+                bot_response,
+                [state, temperature, top_p, max_output_tokens],
+                [state, chatbot] + btn_list,
+            )
+        with gr.Tab("🎚️ 模型微调 Finetune"):
+            gr.Markdown(finetune_notice_markdown)
+            ft_selected_row_data = gr.State()
+            ft_latest_running_cnt = gr.State()
+            df_headers = [
+                "Job Name",
+                "Create By",
+                "Create At",
+                "Model",
+                "Dataset",
+                "Status",
+                "HPs"
+            ]
+            values= [["task111", "Tom", "20230829 14:30", "Vicuna", "cat", "Done", "{\"epochs\": \"1\", \"train_batch_size\": \"2\",\"eval_batch_size\": \"3\" ,\"train_batch_size\": \"2\",\"train_batch_size\": \"2\"}"], 
+            ["task222", "Jerry", "20230829 15:30", "Vicuna", "dog", "Doing", "{\"train_batch_size\": \"2\", \"train_batch_size\": \"2\",\"train_batch_size\": \"2\" ,\"train_batch_size\": \"2\",\"train_batch_size\": \"2\"}"], 
+            ["task333", "Somebody", "20230830 15:30", "Vicuna", "cat", "Error", "{\"train_batch_size\": \"2\", \"train_batch_size\": \"2\",\"train_batch_size\": \"2\" ,\"train_batch_size\": \"2\",\"train_batch_size\": \"2\"}"]]
+            ft_jobs_info = gr.Dataframe(
+                headers=df_headers,
+                type='array',
+                datatype=["str", "str", "str", "str", "str", "str", "str"],
+                value=values,
+                interactive=False,
+            )
+            with gr.Row():
+                ft_show_btn = gr.Button(value="Show Logs")
+                ft_refresh_btn = gr.Button(value="Refresh")
+                ft_remove_btn = gr.Button(value="Remove Running")
+            with gr.Row():
+                with gr.Column(scale=1):
+                    ft_user_name = gr.Textbox(value="", label="User Name")
+                    ft_model = gr.Dropdown(["vicuna-7b-v1.5-16k"], value="vicuna-7b-v1.5-16k", label="Model", interactive=True)
+                    ft_dataset_name = gr.Dropdown(["cat", "dog", "bird"], value="cat", label="Dataset", interactive=True)
+                    ft_token = gr.Textbox(value="", label="Finetune token")
+                    ft_submit_btn = gr.Button(value="Submit")
+                    ft_cease_btn = gr.Button(value="Cease Streaming")
+                with gr.Column(scale=1):
+                    ft_epochs = gr.Slider(
+                        minimum=1,
+                        maximum=3,
+                        value=3,
+                        step=1,
+                        interactive=True,
+                        label="epochs",
+                    )
+                    ft_train_batch_size = gr.Textbox(value="2", label="train batch size", interactive=True)
+                    ft_eval_batch_size = gr.Textbox(value="2", label="eval batch size", interactive=True)
+                    ft_gradient_accumulation_steps = gr.Textbox(value="16", label="gradient accumulation steps", interactive=True)
+                    ft_learning_rate = gr.Textbox(value="2e-5", label="learning rate", interactive=True)
+                    ft_weight_decay = gr.Textbox(value="0.", label="weight decay", interactive=True)
+                    ft_model_max_length = gr.Textbox(value="1024", label="model max length", interactive=True)
+                with gr.Column(scale=8):
+                    ft_console = gr.Textbox(value="", lines=28, label="Console", interactive=False)
+            ft_jobs_info.select(ft_jobs_info_select, [ft_jobs_info, ft_model, ft_dataset_name, ft_epochs, ft_train_batch_size, ft_eval_batch_size, ft_gradient_accumulation_steps, ft_learning_rate, ft_weight_decay, ft_model_max_length], [ft_selected_row_data, ft_model, ft_dataset_name, ft_epochs, ft_train_batch_size, ft_eval_batch_size, ft_gradient_accumulation_steps, ft_learning_rate, ft_weight_decay, ft_model_max_length])
+            
+            ft_show_evt = ft_show_btn.click(ft_show_click, ft_selected_row_data, ft_console)
+            ft_remove_btn.click(ft_remove_click, [ft_selected_row_data, ft_token], ft_console)
+            ft_refresh_btn.click(ft_refresh_click, None, [ft_jobs_info, ft_latest_running_cnt])
+
+            ft_submit_evt = ft_submit_btn.click(ft_submit_click, [ft_latest_running_cnt, ft_user_name, ft_model, ft_dataset_name, ft_token, ft_epochs, ft_train_batch_size, ft_eval_batch_size, ft_gradient_accumulation_steps, ft_learning_rate, ft_weight_decay, ft_model_max_length], [ft_jobs_info, ft_latest_running_cnt, ft_console])
+            ft_cease_btn.click(ft_cease_click, ft_console, ft_console, cancels=[ft_submit_evt, ft_show_evt])
+            
+    return state, model_selector, chatbot, textbox, send_btn, button_row, parameter_row, ft_jobs_info, ft_latest_running_cnt
+
+
+def ft_get_job_data():
+    response = requests.get(ft_list_job_url)
+    res_lst = []
+    running = 0
+    for d in response.json():
+        if isinstance(d['status'], str) and d['status'].lower() == "running":
+            running += 1 
+        res_lst.append([d['jobName'], d['username'], d['created_at'], d['model'], d['dataset'], d['status'], d['parameter']])
+    return res_lst, running
+
+
+def ft_refresh_click():
+    return ft_get_job_data()
+
+def ft_cease_click(ft_console):
+    output = ft_console + "\n" + "** Streaming output ceased by user **"
+    return output
+
+def ft_submit_click(ft_latest_running_cnt, ft_user_name, ft_model, ft_dataset_name, ft_token, ft_epochs, ft_train_batch_size, ft_eval_batch_size, ft_gradient_accumulation_steps, ft_learning_rate, ft_weight_decay, ft_model_max_length):
+    if ft_latest_running_cnt < allow_running:
+        midware_header = {'Content-Type': 'application/json'}
+        hps_json = {
+            "epochs": str(ft_epochs),
+            "train_batch_size": ft_train_batch_size,
+            "eval_batch_size": ft_eval_batch_size,
+            "gradient_accumulation_steps": ft_gradient_accumulation_steps,
+            "learning_rate": ft_learning_rate,
+            "weight_decay": ft_weight_decay,
+            "model_max_length": ft_model_max_length
+        }
+        json_data = {
+            "dataset": ft_dataset_name,
+            "model": ft_model,
+            "parameter": json.dumps(hps_json),
+            "secret": ft_token,
+            "username": ft_user_name
+        }
+        r = requests.post(ft_submit_job_url, json=json_data, headers=midware_header)
+        gr.Info(f"Job submit success!")
+        res_lst, running = ft_get_job_data()
+        return res_lst, running, json.dumps(json_data) + "\n" + str(r.status_code) + json.dumps(r.json())
+    else:
+        gr.Info(f"Only allow {str(allow_running)} job(s) running simultaneously, please wait.")
+        return None
+
+def ft_show_click(ft_selected_row_data):
+    s = PRESET_ANSWERS * 10
+    for i in range(1000):
+        yield s[:i*40]
+        time.sleep(0.01)
+    yield s
+
+def ft_remove_click(ft_selected_row_data, ft_token):
+    status = ft_selected_row_data[5]
+    if isinstance(status, str) and status.lower() == "running":
+        if not ft_token.strip():
+            gr.Info("Remove fail, token needed.")
+        else:
+            pass
+    else:
+        gr.Info("Remove fail, can only remove a running job.")
+    return ft_selected_row_data[0]
+
+def ft_jobs_info_select(ft_jobs_info, ft_model, ft_dataset_name, ft_epochs, ft_train_batch_size, ft_eval_batch_size, ft_gradient_accumulation_steps, ft_learning_rate, ft_weight_decay, ft_model_max_length, evt: gr.SelectData):
+    selected_row = ft_jobs_info[evt.index[0]]
+    if evt.index[1] in (3, 4, 6):
+        try:
+            Hps = json.loads(selected_row[6])
+        except json.decoder.JSONDecodeError:
+            Hps = dict()
+        return [selected_row, selected_row[3], selected_row[4], Hps.get('epochs', ''), Hps.get('train_batch_size', ''), Hps.get('eval_batch_size', ''), 
+                Hps.get('gradient_accumulation_steps', ''), Hps.get('learning_rate', ''), Hps.get('weight_decay', ''), Hps.get('model_max_length', '')]
+    else:
+        return [selected_row, ft_model, ft_dataset_name, ft_epochs, ft_train_batch_size, ft_eval_batch_size, ft_gradient_accumulation_steps, ft_learning_rate, ft_weight_decay, ft_model_max_length]
+
+def build_demo(models):
+    with gr.Blocks(
+        title="Chat with Vicuna (Ascend Backended)",
+        theme=gr.themes.Base(),
+        css = block_css
+    ) as demo:
+        url_params = gr.JSON(visible=False)
+        (
+            state,
+            model_selector,
+            chatbot,
+            textbox,
+            send_btn,
+            button_row,
+            parameter_row,
+            ft_jobs_info,
+            ft_latest_running_cnt,
+        ) = build_single_model_ui(models)
+
+        if model_list_mode not in ["once", "reload"]:
+            raise ValueError(f"Unknown model list mode: {model_list_mode}")
+        demo.load(
+            load_demo,
+            [url_params],
+            [
+                state,
+                model_selector,
+                chatbot,
+                textbox,
+                send_btn,
+                button_row,
+                parameter_row,
+            ],
+            _js=get_window_url_params_js,
+        )
+        demo.load(
+            ft_get_job_data,
+            None,
+            [
+                ft_jobs_info,
+                ft_latest_running_cnt,
+            ]
+        )
+
+    return demo
+
+
+models = get_model_list(midware_url)
+
+# Launch the demo
+demo = build_demo(models)
+demo.queue(
+    concurrency_count=concurrency_count, status_update_rate=10, api_open=False
+).launch(
+    max_threads=200,
+)

requirements.txt

@@ -0,0 +1,3 @@
+requests
+fschat[model_worker,webui]
+websocket