RamAnanth1 commited on
Commit
314b58e
·
1 Parent(s): 98be013

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +12 -13
app.py CHANGED
@@ -291,12 +291,12 @@ class BLIPVQA:
291
  class ConversationBot:
292
  def __init__(self):
293
  print("Initializing VisualChatGPT")
294
- self.edit = ImageEditing(device="cuda:0")
295
  self.i2t = ImageCaptioning(device="cuda:0")
296
  self.t2i = T2I(device="cuda:0")
297
  self.image2canny = image2canny()
298
  #self.canny2image = canny2image(device="cuda:0")
299
- #self.BLIPVQA = BLIPVQA(device="cuda:0")
300
  #self.pix2pix = Pix2Pix(device="cuda:0")
301
  self.memory = ConversationBufferMemory(memory_key="chat_history", output_key='output')
302
  self.tools = [
@@ -306,19 +306,19 @@ class ConversationBot:
306
  Tool(name="Generate Image From User Input Text", func=self.t2i.inference,
307
  description="useful when you want to generate an image from a user input text and save it to a file. like: generate an image of an object or something, or generate an image that includes some objects. "
308
  "The input to this tool should be a string, representing the text used to generate image. "),
309
- Tool(name="Remove Something From The Photo", func=self.edit.remove_part_of_image,
310
- description="useful when you want to remove and object or something from the photo from its description or location. "
311
- "The input to this tool should be a comma seperated string of two, representing the image_path and the object need to be removed. "),
312
- Tool(name="Replace Something From The Photo", func=self.edit.replace_part_of_image,
313
- description="useful when you want to replace an object from the object description or location with another object from its description. "
314
- "The input to this tool should be a comma seperated string of three, representing the image_path, the object to be replaced, the object to be replaced with "),
315
 
316
  #Tool(name="Instruct Image Using Text", func=self.pix2pix.inference,
317
  # description="useful when you want to the style of the image to be like the text. like: make it look like a painting. or make it like a robot. "
318
  # "The input to this tool should be a comma seperated string of two, representing the image_path and the text. "),
319
- #Tool(name="Answer Question About The Image", func=self.BLIPVQA.get_answer_from_question_and_image,
320
- # description="useful when you need an answer for a question based on an image. like: what is the background color of the last image, how many cats in this figure, what is in this figure. "
321
- # "The input to this tool should be a comma seperated string of two, representing the image_path and the question"),
322
  Tool(name="Edge Detection On Image", func=self.image2canny.inference,
323
  description="useful when you want to detect the edge of the image. like: detect the edges of this image, or canny detection on image, or peform edge detection on this image, or detect the canny image of this image. "
324
  "The input to this tool should be a string, representing the image_path"),
@@ -420,9 +420,8 @@ class ConversationBot:
420
 
421
  bot = ConversationBot()
422
  with gr.Blocks(css="#chatbot .overflow-y-auto{height:500px}") as demo:
 
423
  openai_api_key_input = gr.Textbox(type = "password", label = "Enter your OpenAI API key here")
424
-
425
-
426
  chatbot = gr.Chatbot(elem_id="chatbot", label="Visual ChatGPT")
427
  state = gr.State([])
428
 
 
291
  class ConversationBot:
292
  def __init__(self):
293
  print("Initializing VisualChatGPT")
294
+ #self.edit = ImageEditing(device="cuda:0")
295
  self.i2t = ImageCaptioning(device="cuda:0")
296
  self.t2i = T2I(device="cuda:0")
297
  self.image2canny = image2canny()
298
  #self.canny2image = canny2image(device="cuda:0")
299
+ self.BLIPVQA = BLIPVQA(device="cuda:0")
300
  #self.pix2pix = Pix2Pix(device="cuda:0")
301
  self.memory = ConversationBufferMemory(memory_key="chat_history", output_key='output')
302
  self.tools = [
 
306
  Tool(name="Generate Image From User Input Text", func=self.t2i.inference,
307
  description="useful when you want to generate an image from a user input text and save it to a file. like: generate an image of an object or something, or generate an image that includes some objects. "
308
  "The input to this tool should be a string, representing the text used to generate image. "),
309
+ #Tool(name="Remove Something From The Photo", func=self.edit.remove_part_of_image,
310
+ # description="useful when you want to remove and object or something from the photo from its description or location. "
311
+ # "The input to this tool should be a comma seperated string of two, representing the image_path and the object need to be removed. "),
312
+ #Tool(name="Replace Something From The Photo", func=self.edit.replace_part_of_image,
313
+ #description="useful when you want to replace an object from the object description or location with another object from its description. "
314
+ #"The input to this tool should be a comma seperated string of three, representing the image_path, the object to be replaced, the object to be replaced with "),
315
 
316
  #Tool(name="Instruct Image Using Text", func=self.pix2pix.inference,
317
  # description="useful when you want to the style of the image to be like the text. like: make it look like a painting. or make it like a robot. "
318
  # "The input to this tool should be a comma seperated string of two, representing the image_path and the text. "),
319
+ Tool(name="Answer Question About The Image", func=self.BLIPVQA.get_answer_from_question_and_image,
320
+ description="useful when you need an answer for a question based on an image. like: what is the background color of the last image, how many cats in this figure, what is in this figure. "
321
+ "The input to this tool should be a comma seperated string of two, representing the image_path and the question"),
322
  Tool(name="Edge Detection On Image", func=self.image2canny.inference,
323
  description="useful when you want to detect the edge of the image. like: detect the edges of this image, or canny detection on image, or peform edge detection on this image, or detect the canny image of this image. "
324
  "The input to this tool should be a string, representing the image_path"),
 
420
 
421
  bot = ConversationBot()
422
  with gr.Blocks(css="#chatbot .overflow-y-auto{height:500px}") as demo:
423
+ gr.Markdown("# Visual ChatGPT <p> Currently supports text, image captioning, image generation, and visual quesetion answering</p>'")
424
  openai_api_key_input = gr.Textbox(type = "password", label = "Enter your OpenAI API key here")
 
 
425
  chatbot = gr.Chatbot(elem_id="chatbot", label="Visual ChatGPT")
426
  state = gr.State([])
427