Tonic commited on
Commit
e78f385
·
1 Parent(s): 4b2b0f0

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +48 -111
app.py CHANGED
@@ -26,112 +26,54 @@ task_history = []
26
  BOX_TAG_PATTERN = r"<box>([\s\S]*?)</box>"
27
  PUNCTUATION = "!?。"#$%&'()*+,-/:;<=>@[\]^_`{|}~⦅⦆「」、、〃》「」『』&#8203;``【oaicite:0】``&#8203;〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏."
28
 
29
- import os
30
- from PIL import Image, UnidentifiedImageError
31
- import secrets
32
-
33
-
34
- def save_image(image_file) -> str:
35
- upload_dir = os.path.abspath("uploaded_images")
36
- print(f"Creating upload directory at {upload_dir} if it doesn't exist.")
37
- os.makedirs(upload_dir, exist_ok=True)
38
- try:
39
- image = Image.open(image_file).convert("RGB")
40
- file_name = secrets.token_hex(10) + ".png"
41
- file_path = os.path.join(upload_dir, file_name)
42
- print(f"Generated file path: {file_path}")
43
- print("Saving the image.")
44
- image.save(file_path, format="PNG")
45
- print("Image saved successfully.")
46
- return file_path
47
- except UnidentifiedImageError:
48
- print("Error: The file is not a recognizable image.")
49
- return None
50
- except Exception as e:
51
- print(f"An unexpected error occurred: {e}")
52
- return None
53
 
54
- def clean_response(response: str) -> str:
55
- response = re.sub(r'<ref>(.*?)</ref>(?:<box>.*?</box>)*(?:<quad>.*?</quad>)*', r'\1', response).strip()
56
- return response
57
-
58
- def chat_with_model(image_path=None, text_query=None, history=None):
59
- if image_path:
60
- try:
61
- with Image.open(image_path) as img:
62
- print(f"Image {image_path} opened successfully.")
63
- except UnidentifiedImageError:
64
- print(f"Error: The file at {image_path} is not a recognizable image.")
65
- return "Error: Uploaded file is not a recognizable image."
66
- except Exception as e:
67
- print(f"An error occurred while processing the image: {e}")
68
- return "An error occurred while processing your request."
69
- else:
70
- print("No image path provided, using text-only mode.")
71
-
72
- text_input = text_query if text_query else ""
73
- query_elements = [
74
- {'image': image_path},
75
- {'text': text_input}
76
- ]
77
- try:
78
- query = tokenizer.from_list_format(query_elements)
79
- tokenized_inputs = tokenizer(query, return_tensors='pt').to(device)
80
- output = model.generate(**tokenized_inputs)
81
- response = tokenizer.decode(output[0], skip_special_tokens=True)
82
- cleaned_response = clean_response(response)
83
- return cleaned_response
84
- except Exception as e:
85
- print(f"An error occurred: {e}")
86
- return "An error occurred while processing your request."
87
-
88
- def draw_boxes(image_path, response):
89
- with Image.open(image_path) as image:
90
- draw = ImageDraw.Draw(image)
91
- boxes = re.findall(r'<box>\((\d+),(\d+)\),\((\d+),(\d+)\)</box>', response)
92
- for box in boxes:
93
- x1, y1, x2, y2 = map(int, box)
94
- draw.rectangle([x1, y1, x2, y2], outline="red", width=3)
95
- with tempfile.NamedTemporaryFile(delete=False, suffix='.png', dir=os.path.abspath("uploaded_images")) as temp_file:
96
- image.save(temp_file, format="PNG")
97
- return temp_file.name
98
-
99
- def draw_boxes_with_tokenizer(response, history):
100
- image = tokenizer.draw_bbox_on_latest_picture(response, history)
101
- if image is not None:
102
- buffered = io.BytesIO()
103
- image.save(buffered, format="PNG")
104
- img_str = base64.b64encode(buffered.getvalue()).decode()
105
- return "data:image/png;base64," + img_str
106
- else:
107
- print("No box found or drawing failed.")
108
- return None
109
 
110
- def process_input(text=None, file=None, task_history=None):
111
- if task_history is None:
112
- task_history = []
113
- image_path = None
114
- if file is not None:
115
- image_path = save_image(file)
116
- if image_path is None:
117
- return [("bot", "Error: Uploaded file is not a recognizable image.")], task_history
118
 
119
- response = chat_with_model(image_path=image_path, text_query=text, history=task_history)
120
- task_history.append((text, response))
 
121
 
122
- if "<box>" in response and image_path:
123
- image_with_boxes_base64 = draw_boxes_with_tokenizer(response, task_history)
124
- if image_with_boxes_base64:
125
- cleaned_response = clean_response(response)
126
- return [("Qwen-VL_Image", image_with_boxes_base64), ("Qwen-VL_Chat", cleaned_response)], task_history
127
- else:
128
- return [("bot", "Unable to draw boxes on the image.")], task_history
129
  else:
130
- cleaned_response = clean_response(response)
131
- return [("Qwen-VL_Chat", cleaned_response)], task_history
132
 
133
  with gr.Blocks() as demo:
134
- gr.Markdown("""
 
135
  # 🙋🏻‍♂️欢迎来到🌟Tonic 的🦆Qwen-VL-Chat🤩Bot!🚀
136
  # 🙋🏻‍♂️Welcome to Tonic's🦆Qwen-VL-Chat🤩Bot!🚀
137
  该WebUI基于Qwen-VL-Chat,实现聊天机器人功能。 但我必须解决它的很多问题,也许我也能获得一些荣誉。
@@ -142,28 +84,23 @@ Join us: TeamTonic is always making cool demos! Join our active builder's comm
142
  """)
143
  with gr.Row():
144
  with gr.Column(scale=1):
145
- chatbot = gr.Chatbot(label='Qwen-VL-Chat')
146
  with gr.Column(scale=1):
147
  with gr.Row():
148
  query = gr.Textbox(lines=2, label='Input', placeholder="Type your message here...")
149
- file_upload = gr.Image(type="filepath", label="Upload Image")
150
  submit_btn = gr.Button("Submit")
151
 
152
- task_history = gr.State([])
153
-
154
  submit_btn.click(
155
- fn=process_input,
156
- inputs=[query, file_upload, task_history],
157
- outputs=[chatbot, task_history]
158
  )
159
-
160
- gr.Markdown("""
161
  注意:此演示受 Qwen-VL 原始许可证的约束。我们强烈建议用户不要故意生成或允许他人故意生成有害内容,
162
  包括仇恨言论、暴力、色情、欺骗等。(注:本演示受Qwen-VL许可协议约束,强烈建议用户不要传播或允许他人传播以下内容,包括但不限于仇恨言论、暴力、色情、欺诈相关的有害信息 .)
163
  Note: This demo is governed by the original license of Qwen-VL. We strongly advise users not to knowingly generate or allow others to knowingly generate harmful content,
164
  including hate speech, violence, pornography, deception, etc. (Note: This demo is subject to the license agreement of Qwen-VL. We strongly advise users not to disseminate or allow others to disseminate the following content, including but not limited to hate speech, violence, pornography, and fraud-related harmful information.)
165
  """)
166
- demo.queue().launch()
167
-
168
  if __name__ == "__main__":
169
- demo.launch()
 
26
  BOX_TAG_PATTERN = r"<box>([\s\S]*?)</box>"
27
  PUNCTUATION = "!?。"#$%&'()*+,-/:;<=>@[\]^_`{|}~⦅⦆「」、、〃》「」『』&#8203;``【oaicite:0】``&#8203;〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏."
28
 
29
+ class ChatBot:
30
+ def __init__(self):
31
+ self.tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen-VL-Chat", trust_remote_code=True)
32
+ self.model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen-VL-Chat", device_map="cuda", trust_remote_code=True).eval()
33
+ self.model.generation_config = GenerationConfig.from_pretrained("Qwen/Qwen-VL-Chat", trust_remote_code=True)
34
+ self.history = []
35
+
36
+ def chat(self, image_path=None, text_query=None):
37
+ query_elements = []
38
+ if image_path:
39
+ query_elements.append({'image': image_path})
40
+ if text_query:
41
+ query_elements.append({'text': text_query})
 
 
 
 
 
 
 
 
 
 
 
42
 
43
+ query = self.tokenizer.from_list_format(query_elements)
44
+ response, self.history = self.model.chat(self.tokenizer, query=query, history=self.history)
45
+ return response
46
+
47
+ def draw_boxes(self, response):
48
+ image = self.tokenizer.draw_bbox_on_latest_picture(response, self.history)
49
+ if image is not None:
50
+ buffered = io.BytesIO()
51
+ image.save(buffered, format="PNG")
52
+ img_str = base64.b64encode(buffered.getvalue()).decode()
53
+ return "data:image/png;base64," + img_str
54
+ else:
55
+ return None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
56
 
57
+ def clear_memory(self):
58
+ if torch.cuda.is_available():
59
+ torch.cuda.empty_cache()
60
+ gc.collect()
 
 
 
 
61
 
62
+ def chat_interface(chatbot, text_query, file):
63
+ image_path = file.name if file is not None else None
64
+ response = chatbot.chat(image_path=image_path, text_query=text_query)
65
 
66
+ if "<box>" in response:
67
+ image_with_boxes = chatbot.draw_boxes(response)
68
+ chatbot.clear_memory()
69
+ return [("Qwen-VL_Chat", response), ("Qwen-VL_Image", image_with_boxes)]
 
 
 
70
  else:
71
+ chatbot.clear_memory()
72
+ return [("Qwen-VL_Chat", response)]
73
 
74
  with gr.Blocks() as demo:
75
+ chatbot = ChatBot()
76
+ with gr.Markdown("""
77
  # 🙋🏻‍♂️欢迎来到🌟Tonic 的🦆Qwen-VL-Chat🤩Bot!🚀
78
  # 🙋🏻‍♂️Welcome to Tonic's🦆Qwen-VL-Chat🤩Bot!🚀
79
  该WebUI基于Qwen-VL-Chat,实现聊天机器人功能。 但我必须解决它的很多问题,也许我也能获得一些荣誉。
 
84
  """)
85
  with gr.Row():
86
  with gr.Column(scale=1):
87
+ chatbot_component = gr.Chatbot(label='🦆Qwen-VL-Chat')
88
  with gr.Column(scale=1):
89
  with gr.Row():
90
  query = gr.Textbox(lines=2, label='Input', placeholder="Type your message here...")
91
+ file_upload = gr.File(label="Upload Image")
92
  submit_btn = gr.Button("Submit")
93
 
 
 
94
  submit_btn.click(
95
+ fn=chat_interface,
96
+ inputs=[chatbot, query, file_upload],
97
+ outputs=[chatbot_component]
98
  )
99
+ with gr.Markdown("""
 
100
  注意:此演示受 Qwen-VL 原始许可证的约束。我们强烈建议用户不要故意生成或允许他人故意生成有害内容,
101
  包括仇恨言论、暴力、色情、欺骗等。(注:本演示受Qwen-VL许可协议约束,强烈建议用户不要传播或允许他人传播以下内容,包括但不限于仇恨言论、暴力、色情、欺诈相关的有害信息 .)
102
  Note: This demo is governed by the original license of Qwen-VL. We strongly advise users not to knowingly generate or allow others to knowingly generate harmful content,
103
  including hate speech, violence, pornography, deception, etc. (Note: This demo is subject to the license agreement of Qwen-VL. We strongly advise users not to disseminate or allow others to disseminate the following content, including but not limited to hate speech, violence, pornography, and fraud-related harmful information.)
104
  """)
 
 
105
  if __name__ == "__main__":
106
+ demo.launch()