app.py

import gradio as gr
from transformers import AutoModelForCausalLM, AutoTokenizer
from transformers.generation import GenerationConfig
import re
import copy
from pathlib import Path
import secrets
import torch
import os
import io
from io import BytesIO
from PIL import Image, ImageDraw, UnidentifiedImageError

base_url = "https://huggingface.co/spaces/Tonic1/Official-Qwen-VL-Chat"
model_name = "Qwen/Qwen-VL-Chat"
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True).eval()
model.generation_config = GenerationConfig.from_pretrained(model_name, trust_remote_code=True)

device = "cuda" if torch.cuda.is_available() else "cpu"
model.to(device)
task_history = []

BOX_TAG_PATTERN = r"<box>([\s\S]*?)</box>"
PUNCTUATION = "！？。＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」『』&#8203;``【oaicite:0】``&#8203;〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏."

import os
from PIL import Image, UnidentifiedImageError
import secrets


def save_image(image_file) -> str:
    upload_dir = os.path.abspath("uploaded_images")
    print(f"Creating upload directory at {upload_dir} if it doesn't exist.")
    os.makedirs(upload_dir, exist_ok=True)
    try:
        image = Image.open(image_file).convert("RGB")
        file_name = secrets.token_hex(10) + ".png"
        file_path = os.path.join(upload_dir, file_name)
        print(f"Generated file path: {file_path}")
        print("Saving the image.")
        image.save(file_path, format="PNG")
        print("Image saved successfully.")
        return file_path
    except UnidentifiedImageError:
        print("Error: The file is not a recognizable image.")
        return None
    except Exception as e:
        print(f"An unexpected error occurred: {e}")
        return None
        
def clean_response(response: str) -> str:
    response = re.sub(r'<ref>(.*?)</ref>(?:<box>.*?</box>)*(?:<quad>.*?</quad>)*', r'\1', response).strip()
    return response

def chat_with_model(image_path=None, text_query=None, history=None):
    if image_path:
        try:
            with Image.open(image_path) as img:
                print(f"Image {image_path} opened successfully.")
        except UnidentifiedImageError:
            print(f"Error: The file at {image_path} is not a recognizable image.")
            return "Error: Uploaded file is not a recognizable image."
        except Exception as e:
            print(f"An error occurred while processing the image: {e}")
            return "An error occurred while processing your request."
    else:
        print("No image path provided, using text-only mode.")

    text_input = text_query if text_query else ""
    query_elements = [
        {'image': image_path},
        {'text': text_input}
    ]
    try:
        query = tokenizer.from_list_format(query_elements)
        tokenized_inputs = tokenizer(query, return_tensors='pt').to(device)
        output = model.generate(**tokenized_inputs)
        response = tokenizer.decode(output[0], skip_special_tokens=True)
        cleaned_response = clean_response(response)
        return cleaned_response
    except Exception as e:
        print(f"An error occurred: {e}")
        return "An error occurred while processing your request."

def draw_boxes(image_path, response):
    image = Image.open(image_path)
    draw = ImageDraw.Draw(image)
    boxes = re.findall(r'<box>\((\d+),(\d+)\),\((\d+),(\d+)\)</box>', response)
    for box in boxes:
        x1, y1, x2, y2 = map(int, box)
        draw.rectangle([x1, y1, x2, y2], outline="red", width=3)
    return image

def process_input(text=None, file=None, task_history=None):
    if task_history is None:
        task_history = []
    image_path = None
    if file is not None:
        image_path = save_image(file)
        if image_path is None:
            return [("bot", "Error: Uploaded file is not a recognizable image.")], task_history

    response = chat_with_model(image_path=image_path, text_query=text, history=task_history)
    task_history.append((text, response))

    if "<box>" in response and image_path:
        image_with_boxes = draw_boxes(image_path, response)
        buffered = BytesIO()
        image_with_boxes.save(buffered, format="PNG")
        img_str = buffered.getvalue()
        return [("bot", response, "data:image/png;base64," + base64.b64encode(img_str).decode())], task_history
    else:
        cleaned_response = clean_response(response)
        return [("bot", cleaned_response)], task_history

with gr.Blocks() as demo:
    gr.Markdown("""
# 🙋🏻‍♂️欢迎来到🌟Tonic 的🦆Qwen-VL-Chat🤩Bot！🚀
# 🙋🏻‍♂️Welcome toTonic's Qwen-VL-Chat Bot! 
该WebUI基于Qwen-VL-Chat，实现聊天机器人功能。 但我必须解决它的很多问题，也许我也能获得一些荣誉。
Qwen-VL-Chat 是一种多模式输入模型。 您可以使用此空间来测试当前模型 [qwen/Qwen-VL-Chat](https://huggingface.co/qwen/Qwen-VL-Chat) 您也可以使用 🧑🏻‍🚀qwen/Qwen-VL -通过克隆这个空间来聊天🚀。 🧬🔬🔍 只需点击这里：[重复空间](https://huggingface.co/spaces/Tonic1/VLChat?duplicate=true)
加入我们：🌟TeamTonic🌟总是在制作很酷的演示！ 在 👻Discord 上加入我们活跃的构建者🛠️社区：[Discord](https://discord.gg/nXx5wbX9) 在 🤗Huggingface 上：[TeamTonic](https://huggingface.co/TeamTonic) 和 [MultiTransformer](https:/ /huggingface.co/MultiTransformer) 在 🌐Github 上：[Polytonic](https://github.com/tonic-ai) 并为 🌟 [PolyGPT](https://github.com/tonic-ai/polygpt-alpha) 做出贡献 ）
This WebUI is based on Qwen-VL-Chat, implementing chatbot functionalities. Qwen-VL-Chat is a multimodal input model. You can use this Space to test out the current model [qwen/Qwen-VL-Chat](https://huggingface.co/qwen/Qwen-VL-Chat) You can also use  qwen/Qwen-VL-Chat🚀 by cloning this space.   Simply click here: [Duplicate Space](https://huggingface.co/spaces/Tonic1/VLChat?duplicate=true)
Join us:  TeamTonic  is always making cool demos! Join our active builder's community on  Discord: [Discord](https://discord.gg/nXx5wbX9) On  Huggingface: [TeamTonic](https://huggingface.co/TeamTonic) & [MultiTransformer](https://huggingface.co/MultiTransformer) On  Github: [Polytonic](https://github.com/tonic-ai) & contribute to   [PolyGPT](https://github.com/tonic-ai/polygpt-alpha)
""")
    with gr.Row():
        with gr.Column(scale=1):
            chatbot = gr.Chatbot(label='Qwen-VL-Chat')
        with gr.Column(scale=1):
            with gr.Row():
                query = gr.Textbox(lines=2, label='Input', placeholder="Type your message here...")
                file_upload = gr.Image(type="filepath", label="Upload Image")
                submit_btn = gr.Button("Submit")
    
    task_history = gr.State([])

    submit_btn.click(
        fn=process_input,
        inputs=[query, file_upload, task_history],
        outputs=[chatbot, task_history]
    )
    
    gr.Markdown("""
注意：此演示受 Qwen-VL 原始许可证的约束。我们强烈建议用户不要故意生成或允许他人故意生成有害内容，
包括仇恨言论、暴力、色情、欺骗等。（注：本演示受Qwen-VL许可协议约束，强烈建议用户不要传播或允许他人传播以下内容，包括但不限于仇恨言论、暴力、色情、欺诈相关的有害信息 .)
Note: This demo is governed by the original license of Qwen-VL. We strongly advise users not to knowingly generate or allow others to knowingly generate harmful content,
including hate speech, violence, pornography, deception, etc. (Note: This demo is subject to the license agreement of Qwen-VL. We strongly advise users not to disseminate or allow others to disseminate the following content, including but not limited to hate speech, violence, pornography, and fraud-related harmful information.)
""")
    demo.queue().launch()

if __name__ == "__main__":
    demo.launch()