Spaces:
Build error
Build error
File size: 4,939 Bytes
63e68a4 a504543 63e68a4 a504543 63e68a4 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 |
import gradio as gr
from PIL import Image
from transformers import AutoModelForCausalLM
from transformers import AutoProcessor
from transformers import TextIteratorStreamer
import time
from threading import Thread
import torch
import spaces
model_id = "microsoft/Phi-3-vision-128k-instruct"
model = AutoModelForCausalLM.from_pretrained(model_id, device_map="cuda", trust_remote_code=True, torch_dtype="auto")
processor = AutoProcessor.from_pretrained(model_id, trust_remote_code=True)
model.to("cuda:0")
PLACEHOLDER = """
<div style="padding: 30px; text-align: center; display: flex; flex-direction: column; align-items: center;">
<img src="https://cdn-thumbnails.huggingface.co/social-thumbnails/models/microsoft/Phi-3-vision-128k-instruct.png" style="width: 80%; max-width: 550px; height: auto; opacity: 0.55; ">
<h1 style="font-size: 28px; margin-bottom: 2px; opacity: 0.55;">Microsoft's Phi3-Vision-128k-Context</h1>
<p style="font-size: 18px; margin-bottom: 2px; opacity: 0.65;">Phi-3-Vision is a 4.2B parameter multimodal model that brings together language and vision capabilities.</p>
</div>
"""
@spaces.GPU
def bot_streaming(message, history):
print(f'message is - {message}')
print(f'history is - {history}')
if message["files"]:
# message["files"][-1] is a Dict or just a string
if type(message["files"][-1]) == dict:
image = message["files"][-1]["path"]
else:
image = message["files"][-1]
else:
# if there's no image uploaded for this turn, look for images in the past turns
# kept inside tuples, take the last one
for hist in history:
if type(hist[0]) == tuple:
image = hist[0][0]
try:
if image is None:
# Handle the case where image is None
raise gr.Error("You need to upload an image for FalconVLM to work. Close the error and try again with an Image.")
except NameError:
# Handle the case where 'image' is not defined at all
raise gr.Error("You need to upload an image for FalconVLM to work. Close the error and try again with an Image.")
conversation = []
flag=False
for user, assistant in history:
if assistant is None:
#pass
flag=True
conversation.extend([{"role": "user", "content":""}])
continue
if flag==True:
conversation[0]['content'] = f"<|image_1|>\n{user}"
conversation.extend([{"role": "assistant", "content": assistant}])
flag=False
continue
#conversation += f"""User:<image>\n{user} Falcon:{assistant} """
conversation.extend([{"role": "user", "content": user}, {"role": "assistant", "content": assistant}])
if len(history) == 0:
conversation.append({"role": "user", "content": f"<|image_1|>\n{message['text']}"})
else:
conversation.append({"role": "user", "content": message['text']})
print(f"prompt is -\n{conversation}")
#prompt = f"""User:<image>\n{message['text']} Falcon:"""
prompt = processor.tokenizer.apply_chat_template(conversation, tokenize=False, add_generation_prompt=True)
image = Image.open(image)
inputs = processor(prompt, image, return_tensors="pt").to("cuda:0")
#inputs = processor(prompt, image, return_tensors='pt').to(0, torch.float16)
streamer = TextIteratorStreamer(processor, **{"skip_special_tokens": True, "skip_prompt": True, 'clean_up_tokenization_spaces':False,}) # "eos_token_id":processor.tokenizer.eos_token_id})
generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=1024, do_sample=False, temperature=0.0, eos_token_id=processor.tokenizer.eos_token_id,)
thread = Thread(target=model.generate, kwargs=generation_kwargs)
thread.start()
buffer = ""
for new_text in streamer:
# find <|eot_id|> and remove it from the new_text
#if "<|eot_id|>" in new_text:
# new_text = new_text.split("<|eot_id|>")[0]
buffer += new_text
yield buffer
chatbot=gr.Chatbot(scale=1, placeholder=PLACEHOLDER)
chat_input = gr.MultimodalTextbox(interactive=True, file_types=["image"], placeholder="Enter message or upload file...", show_label=False)
with gr.Blocks(fill_height=True, ) as demo:
gr.ChatInterface(
fn=bot_streaming,
title="FalconVLM",
examples=[{"text": "What is on the flower?", "files": ["./bee.jpg"]},
{"text": "How to make this pastry?", "files": ["./baklava.png"]}],
description="Try [tiiuae/falcon-11B-VLM](https://huggingface.co/tiiuae/falcon-11B-vlm). Upload an image and start chatting about it, or simply try one of the examples below. If you don't upload an image, you will receive an error.",
stop_btn="Stop Generation",
multimodal=True,
textbox=chat_input,
chatbot=chatbot,
cache_examples=False,
)
demo.queue()
demo.launch(debug=True, quiet=True)
|