Enhance error handling in data processing pipeline for improved robustness

app.py

@@ -1,303 +1,622 @@
-import gradio as gr
-import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, StoppingCriteria
-import gradio as gr
 import spaces
-import torch
-import numpy as np
-import torch
-import torchvision.transforms as T
-from PIL import Image
-from torchvision.transforms.functional import InterpolationMode
-from transformers import AutoModel, AutoTokenizer
-from PIL import Image, ExifTags
-
-from threading import Thread
+import argparse
+from ast import parse
+import datetime
+import json
+import os
+import time
+import hashlib
 import re
-import time 
-from PIL import Image
 import torch
-import spaces
+import gradio as gr
+import requests
+import random
+from filelock import FileLock
+from io import BytesIO
+from PIL import Image, ImageDraw, ImageFont
+from models import load_image
+from constants import LOGDIR
+from utils import (
+    build_logger,
+    server_error_msg,
+    violates_moderation,
+    moderation_msg,
+    load_image_from_base64,
+    get_log_filename,
+)
+from threading import Thread
+import traceback
+# import torch
+from conversation import Conversation
+from transformers import AutoModel, AutoTokenizer, TextIteratorStreamer
 import subprocess
-import os
 
 subprocess.run('pip install flash-attn --no-build-isolation', env={'FLASH_ATTENTION_SKIP_CUDA_BUILD': "TRUE"}, shell=True)
 
 torch.set_default_device('cuda')
 
+logger = build_logger("gradio_web_server", "gradio_web_server.log")
 
-IMAGENET_MEAN = (0.485, 0.456, 0.406)
-IMAGENET_STD = (0.229, 0.224, 0.225)
-
-def build_transform(input_size):
-    MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
-    transform = T.Compose([
-        T.Lambda(lambda img: img.convert('RGB') if img.mode != 'RGB' else img),
-        T.Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC),
-        T.ToTensor(),
-        T.Normalize(mean=MEAN, std=STD)
-    ])
-    return transform
-
-def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
-    best_ratio_diff = float('inf')
-    best_ratio = (1, 1)
-    area = width * height
-    for ratio in target_ratios:
-        target_aspect_ratio = ratio[0] / ratio[1]
-        ratio_diff = abs(aspect_ratio - target_aspect_ratio)
-        if ratio_diff < best_ratio_diff:
-            best_ratio_diff = ratio_diff
-            best_ratio = ratio
-        elif ratio_diff == best_ratio_diff:
-            if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
-                best_ratio = ratio
-    return best_ratio
-
-def dynamic_preprocess(image, min_num=1, max_num=12, image_size=448, use_thumbnail=False):
-    orig_width, orig_height = image.size
-    aspect_ratio = orig_width / orig_height
-
-    # calculate the existing image aspect ratio
-    target_ratios = set(
-        (i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if
-        i * j <= max_num and i * j >= min_num)
-    target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
-
-    # find the closest aspect ratio to the target
-    target_aspect_ratio = find_closest_aspect_ratio(
-        aspect_ratio, target_ratios, orig_width, orig_height, image_size)
-
-    # calculate the target width and height
-    target_width = image_size * target_aspect_ratio[0]
-    target_height = image_size * target_aspect_ratio[1]
-    blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
-
-    # resize the image
-    resized_img = image.resize((target_width, target_height))
-    processed_images = []
-    for i in range(blocks):
-        box = (
-            (i % (target_width // image_size)) * image_size,
-            (i // (target_width // image_size)) * image_size,
-            ((i % (target_width // image_size)) + 1) * image_size,
-            ((i // (target_width // image_size)) + 1) * image_size
-        )
-        # split the image
-        split_img = resized_img.crop(box)
-        processed_images.append(split_img)
-    assert len(processed_images) == blocks
-    if use_thumbnail and len(processed_images) != 1:
-        thumbnail_img = image.resize((image_size, image_size))
-        processed_images.append(thumbnail_img)
-    return processed_images
-
-def correct_image_orientation(image_path):
-    # Mở ảnh
-    image = Image.open(image_path)
-
-    # Kiểm tra dữ liệu Exif (nếu có)
-    try:
-        exif = image._getexif()
-        if exif is not None:
-            for tag, value in exif.items():
-                if ExifTags.TAGS.get(tag) == "Orientation":
-                    # Sửa hướng dựa trên Orientation
-                    if value == 3:
-                        image = image.rotate(180, expand=True)
-                    elif value == 6:
-                        image = image.rotate(-90, expand=True)
-                    elif value == 8:
-                        image = image.rotate(90, expand=True)
-                    break
-    except Exception as e:
-        print("Không thể xử lý Exif:", e)
+headers = {"User-Agent": "Vintern-Chat Client"}
 
-    return image
-    
-def load_image(image_file, input_size=448, max_num=12):
-    image = correct_image_orientation(image_file).convert('RGB')
-    print("Image size: ", image.size)
-    transform = build_transform(input_size=input_size)
-    images = dynamic_preprocess(image, image_size=input_size, use_thumbnail=True, max_num=max_num)
-    pixel_values = [transform(image) for image in images]
-    pixel_values = torch.stack(pixel_values)
-    return pixel_values
+no_change_btn = gr.Button()
+enable_btn = gr.Button(interactive=True)
+disable_btn = gr.Button(interactive=False)
+
+
+@spaces.GPU(duration=10)
+def make_zerogpu_happy():
+    pass
+
+
+def write2file(path, content):
+    lock = FileLock(f"{path}.lock")
+    with lock:
+        with open(path, "a") as fout:
+            fout.write(content)
+
+
+get_window_url_params = """
+function() {
+    const params = new URLSearchParams(window.location.search);
+    url_params = Object.fromEntries(params);
+    console.log(url_params);
+    return url_params;
+    }
+"""
+
+
+def init_state(state=None):
+    if state is not None:
+        del state
+    return Conversation()
+
+def vote_last_response(state, liked, request: gr.Request):
+    conv_data = {
+        "tstamp": round(time.time(), 4),
+        "like": liked,
+        "model": 'Vintern-1B-v3',
+        "state": state.dict(),
+        "ip": request.client.host,
+    }
+    write2file(get_log_filename(), json.dumps(conv_data) + "\n")
 
+
+def upvote_last_response(state, request: gr.Request):
+    logger.info(f"upvote. ip: {request.client.host}")
+    vote_last_response(state, True, request)
+    textbox = gr.MultimodalTextbox(value=None, interactive=True)
+    return (textbox,) + (disable_btn,) * 3
+
+
+def downvote_last_response(state, request: gr.Request):
+    logger.info(f"downvote. ip: {request.client.host}")
+    vote_last_response(state, False, request)
+    textbox = gr.MultimodalTextbox(value=None, interactive=True)
+    return (textbox,) + (disable_btn,) * 3
+
+
+def vote_selected_response(
+    state, request: gr.Request, data: gr.LikeData
+):
+    logger.info(
+        f"Vote: {data.liked}, index: {data.index}, value: {data.value} , ip: {request.client.host}"
+    )
+    conv_data = {
+        "tstamp": round(time.time(), 4),
+        "like": data.liked,
+        "index": data.index,
+        "model": 'Vintern-1B-v3',
+        "state": state.dict(),
+        "ip": request.client.host,
+    }
+    write2file(get_log_filename(), json.dumps(conv_data) + "\n")
+    return
+
+
+def flag_last_response(state, request: gr.Request):
+    logger.info(f"flag. ip: {request.client.host}")
+    vote_last_response(state, "flag", request)
+    textbox = gr.MultimodalTextbox(value=None, interactive=True)
+    return (textbox,) + (disable_btn,) * 3
+
+
+def regenerate(state, image_process_mode, request: gr.Request):
+    logger.info(f"regenerate. ip: {request.client.host}")
+    # state.messages[-1][-1] = None
+    state.update_message(Conversation.ASSISTANT, content='', image=None, idx=-1)
+    prev_human_msg = state.messages[-2]
+    if type(prev_human_msg[1]) in (tuple, list):
+        prev_human_msg[1] = (*prev_human_msg[1][:2], image_process_mode)
+    state.skip_next = False
+    textbox = gr.MultimodalTextbox(value=None, interactive=True)
+    return (state, state.to_gradio_chatbot(), textbox) + (disable_btn,) * 5
+
+
+def clear_history(request: gr.Request):
+    logger.info(f"clear_history. ip: {request.client.host}")
+    state = init_state()
+    textbox = gr.MultimodalTextbox(value=None, interactive=True)
+    return (state, state.to_gradio_chatbot(), textbox) + (disable_btn,) * 5
+
+
+def add_text(state, message, system_prompt, request: gr.Request):
+    print(f"state: {state}")
+    if not state:
+        state = init_state()
+    images = message.get("files", [])
+    text = message.get("text", "").strip()
+    logger.info(f"add_text. ip: {request.client.host}. len: {len(text)}")
+    # import pdb; pdb.set_trace()
+    textbox = gr.MultimodalTextbox(value=None, interactive=False)
+    if len(text) <= 0 and len(images) == 0:
+        state.skip_next = True
+        return (state, state.to_gradio_chatbot(), textbox) + (no_change_btn,) * 5
+    if args.moderate:
+        flagged = violates_moderation(text)
+        if flagged:
+            state.skip_next = True
+            textbox = gr.MultimodalTextbox(
+                value={"text": moderation_msg}, interactive=True
+            )
+            return (state, state.to_gradio_chatbot(), textbox) + (no_change_btn,) * 5
+    images = [Image.open(path).convert("RGB") for path in images]
+
+    if len(images) > 0 and len(state.get_images(source=state.USER)) > 0:
+        state = init_state(state)
+    state.set_system_message(system_prompt)
+    state.append_message(Conversation.USER, text, images)
+    state.skip_next = False
+    return (state, state.to_gradio_chatbot(), textbox) + (
+        disable_btn,
+    ) * 5
+
+model_name = "5CD-AI/Vintern-1B-v3_5"
 model = AutoModel.from_pretrained(
-    "5CD-AI/Vintern-1B-v3_5",
+    model_name,
     torch_dtype=torch.bfloat16,
     low_cpu_mem_usage=True,
     trust_remote_code=True,
 ).eval().cuda()
-tokenizer = AutoTokenizer.from_pretrained("5CD-AI/Vintern-1B-v3_5", trust_remote_code=True, use_fast=False)
-    
-@spaces.GPU
-def chat(message, history):
-    print("history",history)
-    print("message",message)
+tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True, use_fast=False)
 
-    if len(history) != 0 and len(message["files"]) != 0:
-        return """Chúng tôi hiện chỉ hổ trợ 1 ảnh ở đầu ngữ cảnh! Vui lòng tạo mới cuộc trò chuyện.
-We currently only support one image at the start of the context! Please start a new conversation."""
-    
-    if len(history) == 0 and len(message["files"]) != 0:
-        if "path" in message["files"][0]:
-            test_image = message["files"][0]["path"]
-        else:
-            test_image = message["files"][0]
-        pixel_values = load_image(test_image, max_num=6).to(torch.bfloat16).cuda()
-    elif len(history) == 0 and len(message["files"]) == 0:
-        pixel_values = None     
-    elif history[0][0][0] is not None and os.path.isfile(history[0][0][0]):
-        test_image = history[0][0][0]
-        pixel_values = load_image(test_image, max_num=6).to(torch.bfloat16).cuda()
-    else:
-        pixel_values = None 
-        
-    
-    generation_config = dict(max_new_tokens= 700, do_sample=False, num_beams = 3, repetition_penalty=2.5)
+@spaces.GPU
+def http_bot(
+    state,
+    temperature,
+    top_p,
+    repetition_penalty,
+    max_new_tokens,
+    max_input_tiles,
+    request: gr.Request,
+):
     
-    if len(history) == 0:
+    logger.info(f"http_bot. ip: {request.client.host}")
+    start_tstamp = time.time()
+    if hasattr(state, "skip_next") and state.skip_next:
+        # This generate call is skipped due to invalid inputs
+        yield (
+            state,
+            state.to_gradio_chatbot(),
+            gr.MultimodalTextbox(interactive=False),
+        ) + (no_change_btn,) * 5
+        return
+
+    if model is None:
+        # state.messages[-1][-1] = server_error_msg
+        state.update_message(Conversation.ASSISTANT, server_error_msg)
+        yield (
+            state,
+            state.to_gradio_chatbot(),
+            gr.MultimodalTextbox(interactive=False),
+            disable_btn,
+            disable_btn,
+            disable_btn,
+            enable_btn,
+            enable_btn,
+        )
+        return
+
+    all_images = state.get_images(source=state.USER)
+    all_image_paths = [state.save_image(image) for image in all_images]
+
+    state.append_message(Conversation.ASSISTANT, state.streaming_placeholder)
+    yield (
+        state,
+        state.to_gradio_chatbot(),
+        gr.MultimodalTextbox(interactive=False),
+    ) + (disable_btn,) * 5
+
+    try:
+        # Stream output
+        # response = requests.post(worker_addr, json=pload, headers=headers, stream=True, timeout=300)
+        print(f"all_image_paths: {all_image_paths}")
+
+        pixel_values = load_image(all_image_paths[0], max_num=6).to(torch.bfloat16).cuda()
+        print(f"pixel_values: {pixel_values}")
+        generation_config = dict(max_new_tokens= 700, do_sample=False, num_beams = 3, repetition_penalty=2.5)
+        message = state.get_user_message(source=state.USER)
+        print(f"######################")
+        print(f"message: {message}")
         if pixel_values is not None:
-            question = '<image>\n'+message["text"]
+            question = '<image>\n'+message
         else:
-            question = message["text"]
+            question = message
+        print("Model: ", model)
+        print("Tokenizer: ", tokenizer)
+        print("Question: ", question)
         response, conv_history = model.chat(tokenizer, pixel_values, question, generation_config, history=None, return_history=True)
-    else:
-        conv_history = []
-        if history[0][0][0] is not None and os.path.isfile(history[0][0][0]):
-            start_index = 1
-        else:
-            start_index = 0
-        
-        for i, chat_pair in enumerate(history[start_index:]):
-            if i == 0 and start_index == 1:
-                 conv_history.append(tuple(['<image>\n'+chat_pair[0],chat_pair[1]]))
-            else:
-                conv_history.append(tuple(chat_pair))
-
-            
-        print("conv_history",conv_history)
-        question = message["text"]
-        response, conv_history = model.chat(tokenizer, pixel_values, question, generation_config, history=conv_history, return_history=True)
+        print(f"AI response: {response}")
+
+
+        # streamer = TextIteratorStreamer(
+        #     tokenizer, skip_prompt=True, skip_special_tokens=True
+        # )
+        # generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=1024)
+
+        # thread = Thread(target=model.generate, kwargs=generation_kwargs)
+        # thread.start()
         
-    print(f'User: {question}\nAssistant: {response}')
-
-    # return response
-    buffer = ""
-    for new_text in response:
-      buffer += new_text
-      generated_text_without_prompt = buffer[:]
-      time.sleep(0.02)
-      yield generated_text_without_prompt
-
-CSS ="""
-#component-10 {
-  height: 70dvh !important;
-  transform-origin: top; /* Đảm bảo rằng phần tử mở rộng từ trên xuống */
-  border-style: solid;
-  overflow: hidden;
-  flex-grow: 1;
-  min-width: min(160px, 100%);
-  border-width: var(--block-border-width);
-}
-/* Đảm bảo ảnh bên trong nút hiển thị đúng cách cho các nút có aria-label chỉ định */
-button.svelte-1lcyrx4[aria-label="user's message: a file of type image/jpeg, "] img.svelte-1pijsyv {
-  width: 100%;
-  object-fit: contain;
-  height: 100%;
-  border-radius: 13px; /* Thêm bo góc cho ảnh */
-  max-width: 50vw;     /* Giới hạn chiều rộng ảnh */
-}
-/* Đặt chiều cao cho nút và cho phép chọn văn bản chỉ cho các nút có aria-label chỉ định */
-button.svelte-1lcyrx4[aria-label="user's message: a file of type image/jpeg, "] {
-  user-select: text;
-  text-align: left;
-  height: 300px;
-}
-/* Thêm bo góc và giới hạn chiều rộng cho ảnh không thuộc avatar container */
-.message-wrap.svelte-1lcyrx4 > div.svelte-1lcyrx4 .svelte-1lcyrx4:not(.avatar-container) img {
-  border-radius: 13px;
-  max-width: 50vw;
-}
-.message-wrap.svelte-1lcyrx4 .message.svelte-1lcyrx4 img {
-    margin: var(--size-2);
-    max-height: 500px;
-}
-.image-preview-close-button {
-  position: relative; /* Nếu cần định vị trí */
-  width: 5%; /* Chiều rộng nút */
-  height: 5%; /* Chiều cao nút */
-  display: flex;
-  justify-content: center;
-  align-items: center;
-  padding: 0; /* Để tránh ảnh hưởng từ padding mặc định */
-  border: none; /* Tùy chọn để loại bỏ đường viền */
-  background: none; /* Tùy chọn để loại bỏ nền */
+        # response = "This is a test response"
+        buffer = ""
+        for new_text in response:
+            buffer += new_text
+            # Remove <|im_end|> or similar tokens from the output
+            buffer = buffer.replace("<|im_end|>", "")
+                
+            state.update_message(Conversation.ASSISTANT, buffer + state.streaming_placeholder, None)
+            yield (
+                state,
+                state.to_gradio_chatbot(),
+                gr.MultimodalTextbox(interactive=False),
+            ) + (disable_btn,) * 5
+
+    except Exception as e:
+        logger.error(f"Error in http_bot: {e}")
+        traceback.print_exc()
+        state.update_message(Conversation.ASSISTANT, server_error_msg, None)
+        yield (
+            state,
+            state.to_gradio_chatbot(),
+            gr.MultimodalTextbox(interactive=True),
+        ) + (
+            disable_btn,
+            disable_btn,
+            disable_btn,
+            enable_btn,
+            enable_btn,
+        )
+        return
+
+    ai_response = state.return_last_message()
+
+    logger.info(f"==== response ====\n{ai_response}")
+   
+    state.end_of_current_turn()
+
+    yield (
+        state,
+        state.to_gradio_chatbot(),
+        gr.MultimodalTextbox(interactive=True),
+    ) + (enable_btn,) * 5
+
+    finish_tstamp = time.time()
+    logger.info(f"{buffer}")
+    data = {
+        "tstamp": round(finish_tstamp, 4),
+        "like": None,
+        "model": model_name,
+        "start": round(start_tstamp, 4),
+        "finish": round(start_tstamp, 4),
+        "state": state.dict(),
+        "images": all_image_paths,
+        "ip": request.client.host,
+    }
+    write2file(get_log_filename(), json.dumps(data) + "\n")
+
+# <h1 style="font-size: 28px; font-weight: bold;">Expanding Performance Boundaries of Open-Source Multimodal Models with Model, Data, and Test-Time Scaling</h1>
+title_html = """
+<div style="text-align: center;">
+    <img src="https://lh3.googleusercontent.com/pw/AP1GczMmW-aFQ4dNaR_LCAllh4UZLLx9fTZ1ITHeGVMWx-1bwlIWz4VsWJSGb3_9C7CQfvboqJH41y2Sbc5ToC9ZmKeV4-buf_DEevIMU0HtaLWgHAPOqBiIbG6LaE8CvDqniLZzvB9UX8TR_-YgvYzPFt2z=w1472-h832-s-no-gm?authuser=0" style="height: 100; width: 100%;">
+    <p>Vintern-1B: An Efficient Multimodal Large Language Model for Vietnamese</p>
+    <a href="https://huggingface.co/papers/2408.12480">[📖 Vintern Paper]</a>
+    <a href="https://huggingface.co/5CD-AI">[🤗 5CD-AI Huggingface]</a>
+</div>
+"""
+
+
+tos_markdown = """
+### Terms of use
+By using this service, users are required to agree to the following terms:
+It only provides limited safety measures and may generate offensive content. It must not be used for any illegal, harmful, violent, racist, or sexual purposes. The service may collect user dialogue data for future research.
+Please click the "Flag" button if you get any inappropriate answer! We will collect those to keep improving our moderator.
+For an optimal experience, please use desktop computers for this demo, as mobile devices may compromise its quality.
+"""
+
+
+# .gradio-container {margin: 5px 10px 0 10px !important};
+block_css = """
+.gradio-container {margin: 0.1% 1% 0 1% !important; max-width: 98% !important;};
+#buttons button {
+    min-width: min(120px,100%);
 }
-.example-image-container.svelte-9pi8y1 {
-    width: calc(var(--size-8) * 5);
-    height: calc(var(--size-8) * 5);
-    border-radius: var(--radius-lg);
-    overflow: hidden;
-    position: relative;
-    margin-bottom: var(--spacing-lg);
+
+.gradient-text {
+    font-size: 28px;
+    width: auto;
+    font-weight: bold;
+    background: linear-gradient(45deg, red, orange, yellow, green, blue, indigo, violet);
+    background-clip: text;
+    -webkit-background-clip: text;
+    color: transparent;
 }
-"""
 
-js = """
-function forceLightTheme() {
-    const url = new URL(window.location);
-    // Cập nhật __theme thành light nếu giá trị không đúng
-    if (url.searchParams.get('__theme') !== 'light') {
-        url.searchParams.set('__theme', 'light');
-        // Thay đổi URL mà không tải lại trang nếu cần
-        window.history.replaceState({}, '', url.href);
-    }
-    // Đảm bảo document luôn áp dụng theme light
-    document.documentElement.setAttribute('data-theme', 'light');
+.plain-text {
+    font-size: 22px;
+    width: auto;
+    font-weight: bold;
 }
 """
-from transformers import pipeline
 
-pipe = pipeline("automatic-speech-recognition", model="openai/whisper-large-v3-turbo", torch_dtype=torch.float16, device="cuda:0")
+# js = """
+# function createWaveAnimation() {
+#     const text = document.getElementById('text');
+#     var i = 0;
+#     setInterval(function() {
+#         const colors = [
+#             'red, orange, yellow, green, blue, indigo, violet, purple',
+#             'orange, yellow, green, blue, indigo, violet, purple, red',
+#             'yellow, green, blue, indigo, violet, purple, red, orange',
+#             'green, blue, indigo, violet, purple, red, orange, yellow',
+#             'blue, indigo, violet, purple, red, orange, yellow, green',
+#             'indigo, violet, purple, red, orange, yellow, green, blue',
+#             'violet, purple, red, orange, yellow, green, blue, indigo',
+#             'purple, red, orange, yellow, green, blue, indigo, violet',
+#         ];
+#         const angle = 45;
+#         const colorIndex = i % colors.length;
+#         text.style.background = `linear-gradient(${angle}deg, ${colors[colorIndex]})`;
+#         text.style.webkitBackgroundClip = 'text';
+#         text.style.backgroundClip = 'text';
+#         text.style.color = 'transparent';
+#         text.style.fontSize = '28px';
+#         text.style.width = 'auto';
+#         text.textContent = 'Vintern-1B';
+#         text.style.fontWeight = 'bold';
+#         i += 1;
+#     }, 200);
+#     const params = new URLSearchParams(window.location.search);
+#     url_params = Object.fromEntries(params);
+#     // console.log(url_params);
+#     // console.log('hello world...');
+#     // console.log(window.location.search);
+#     // console.log('hello world...');
+#     // alert(window.location.search)
+#     // alert(url_params);
+#     return url_params;
+# }
 
-@spaces.GPU
-def transcribe_speech(filepath):
-    output = pipe(
-        filepath,
-        max_new_tokens=256,
-        generate_kwargs={
-            "task": "transcribe",
-        },  
-        chunk_length_s=30,
-        batch_size=1,
+# """
+
+
+def build_demo():
+    textbox = gr.MultimodalTextbox(
+        interactive=True,
+        file_types=["image", "video"],
+        placeholder="Enter message or upload file...",
+        show_label=False,
     )
-    return output["text"]
-        
-demo = gr.Blocks(css=CSS,js=js, theme='NoCrypt/miku')
-
-with demo:
-    chat_demo_interface = gr.ChatInterface(
-        fn=chat,
-        description="""**Vintern-1B-v3.5** is the latest in the Vintern series, bringing major improvements over v2 across all benchmarks. This **continuous fine-tuning Version** enhances Vietnamese capabilities while retaining strong English performance. It excels in OCR, text recognition, and Vietnam-specific document understanding.""",
-        examples=[{"text": "Hãy viết một email giới thiệu sản phẩm trong ảnh.", "files":["./demo_3.jpg"]},
-                  {"text": "Trích xuất các thông tin từ ảnh trả về markdown.", "files":["./demo_1.jpg"]}, 
-                  {"text": "Bạn là nhân viên marketing chuyên nghiệp. Hãy viết một bài quảng cáo dài trên mạng xã hội giới thiệu về cửa hàng.", "files":["./demo_2.jpg"]},
-                  {"text": "Trích xuất thông tin kiện hàng trong ảnh và trả về dạng JSON.", "files":["./demo_4.jpg"]}],
-        title="❄️ Vintern-1B-v3.5 Demo ❄️",
-        multimodal=True,
-        css=CSS,
-        js=js,
-        theme='NoCrypt/miku'
+
+    with gr.Blocks(
+        title="Vintern-Chat",
+        theme=gr.themes.Default(),
+        css=block_css,
+    ) as demo:
+        state = gr.State()
+
+        with gr.Row():
+            with gr.Column(scale=2):
+                # gr.Image('./gallery/logo-47b364d3.jpg')
+                gr.HTML(title_html)
+
+                with gr.Accordion("Settings", open=False) as setting_row:
+                    system_prompt = gr.Textbox(
+                        value="请尽可能详细地回答用户的问题。",
+                        label="System Prompt",
+                        interactive=True,
+                    )
+                    temperature = gr.Slider(
+                        minimum=0.0,
+                        maximum=1.0,
+                        value=0.2,
+                        step=0.1,
+                        interactive=True,
+                        label="Temperature",
+                    )
+                    top_p = gr.Slider(
+                        minimum=0.0,
+                        maximum=1.0,
+                        value=0.7,
+                        step=0.1,
+                        interactive=True,
+                        label="Top P",
+                    )
+                    repetition_penalty = gr.Slider(
+                        minimum=1.0,
+                        maximum=1.5,
+                        value=1.1,
+                        step=0.02,
+                        interactive=True,
+                        label="Repetition penalty",
+                    )
+                    max_output_tokens = gr.Slider(
+                        minimum=0,
+                        maximum=4096,
+                        value=1024,
+                        step=64,
+                        interactive=True,
+                        label="Max output tokens",
+                    )
+                    max_input_tiles = gr.Slider(
+                        minimum=1,
+                        maximum=32,
+                        value=12,
+                        step=1,
+                        interactive=True,
+                        label="Max input tiles (control the image size)",
+                    )
+                examples = gr.Examples(
+                    examples=[
+                        [
+                            {
+                                "files": [
+                                    "gallery/14.jfif",
+                                ],
+                                "text": "Please help me analyze this picture.",
+                            }
+                        ],
+                        [
+                            {
+                                "files": [
+                                    "gallery/1-2.PNG",
+                                ],
+                                "text": "Implement this flow chart using python",
+                            }
+                        ],
+                        [
+                            {
+                                "files": [
+                                    "gallery/15.PNG",
+                                ],
+                                "text": "Please help me analyze this picture.",
+                            }
+                        ],
+                    ],
+                    inputs=[textbox],
+                )
+
+            with gr.Column(scale=8):
+                chatbot = gr.Chatbot(
+                    elem_id="chatbot",
+                    label="Vintern",
+                    height=580,
+                    show_copy_button=True,
+                    show_share_button=True,
+                    avatar_images=[
+                        "assets/human.png",
+                        "assets/assistant.png",
+                    ],
+                    bubble_full_width=False,
+                )
+                with gr.Row():
+                    with gr.Column(scale=8):
+                        textbox.render()
+                    with gr.Column(scale=1, min_width=50):
+                        submit_btn = gr.Button(value="Send", variant="primary")
+                with gr.Row(elem_id="buttons") as button_row:
+                    upvote_btn = gr.Button(value="👍  Upvote", interactive=False)
+                    downvote_btn = gr.Button(value="👎  Downvote", interactive=False)
+                    flag_btn = gr.Button(value="⚠️  Flag", interactive=False)
+                    # stop_btn = gr.Button(value="⏹️  Stop Generation", interactive=False)
+                    regenerate_btn = gr.Button(
+                        value="🔄  Regenerate", interactive=False
+                    )
+                    clear_btn = gr.Button(value="🗑️  Clear", interactive=False)
+
+        gr.Markdown(tos_markdown)
+        url_params = gr.JSON(visible=False)
+
+        # Register listeners
+        btn_list = [upvote_btn, downvote_btn, flag_btn, regenerate_btn, clear_btn]
+        upvote_btn.click(
+            upvote_last_response,
+            [state],
+            [textbox, upvote_btn, downvote_btn, flag_btn],
+        )
+        downvote_btn.click(
+            downvote_last_response,
+            [state],
+            [textbox, upvote_btn, downvote_btn, flag_btn],
+        )
+        chatbot.like(
+            vote_selected_response,
+            [state],
+            [],
+        )
+        flag_btn.click(
+            flag_last_response,
+            [state],
+            [textbox, upvote_btn, downvote_btn, flag_btn],
+        )
+        regenerate_btn.click(
+            regenerate,
+            [state, system_prompt],
+            [state, chatbot, textbox] + btn_list,
+        ).then(
+            http_bot,
+            [
+                state,
+                temperature,
+                top_p,
+                repetition_penalty,
+                max_output_tokens,
+                max_input_tiles,
+            ],
+            [state, chatbot, textbox] + btn_list,
+        )
+        clear_btn.click(clear_history, None, [state, chatbot, textbox] + btn_list)
+
+        textbox.submit(
+            add_text,
+            [state, textbox, system_prompt],
+            [state, chatbot, textbox] + btn_list,
+        ).then(
+            http_bot,
+            [
+                state,
+                temperature,
+                top_p,
+                repetition_penalty,
+                max_output_tokens,
+                max_input_tiles,
+            ],
+            [state, chatbot, textbox] + btn_list,
+        )
+        submit_btn.click(
+            add_text,
+            [state, textbox, system_prompt],
+            [state, chatbot, textbox] + btn_list,
+        ).then(
+            http_bot,
+            [
+                state,
+                temperature,
+                top_p,
+                repetition_penalty,
+                max_output_tokens,
+                max_input_tiles,
+            ],
+            [state, chatbot, textbox] + btn_list,
+        )
+
+    return demo
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--host", type=str, default="0.0.0.0")
+    parser.add_argument("--port", type=int, default=7860)
+    parser.add_argument("--concurrency-count", type=int, default=10)
+    parser.add_argument("--share", action="store_true")
+    parser.add_argument("--moderate", action="store_true")
+    args = parser.parse_args()
+    logger.info(f"args: {args}")
+
+    logger.info(args)
+    demo = build_demo()
+    demo.queue(api_open=False).launch(
+        server_name=args.host,
+        server_port=args.port,
+        share=args.share,
+        max_threads=args.concurrency_count,
     )
-        
-    # mic_transcribe = gr.Interface(
-    #     fn=transcribe_speech,
-    #     inputs=gr.Audio(sources="microphone", type="filepath", editable=False),
-    #     outputs=gr.components.Textbox(),
-    # )
-
-# chat_demo_interface.queue()
-demo.queue().launch()