Update app.py

app.py

@@ -1,5 +1,7 @@
+import base64
 import os
 
+from gradio_client.utils import get_mimetype
 from openai import OpenAI
 import gradio as gr
 
@@ -15,14 +17,44 @@ MODELS = [
 ]
 
 
+def process_image(data):
+    with open(data['path'], "rb") as image_file:
+        b64image = base64.b64encode(image_file.read()).decode('utf-8')
+
+    return "data:" + data['mime_type'] + ";base64," + b64image
+
+
 def generate(message, history, model, system_prompt,
-    temperature=1.0, top_p=1.0, frequency_penalty=0.0, presence_penalty=0.0):
-    
+             temperature=1.0, top_p=1.0, frequency_penalty=0.0, presence_penalty=0.0):
+    # history
     history_openai_format = [{"role": "system", "content": system_prompt}]
     for user, assistant in history:
-        history_openai_format.append({"role": "user", "content": user})
-        history_openai_format.append({"role": "assistant", "content": assistant})
-    history_openai_format.append({"role": "user", "content": message})
+        if isinstance(user, tuple):  # there were files
+            content = [
+                {"type": "image_url",
+                 # for some reason you don't get the same image format in history as in message
+                 "image_url": {"url": process_image({'path': filepath,
+                                                     'mime_type': get_mimetype(filepath)})}}
+                for filepath in user]
+
+            history_openai_format.append(
+                {"role": "user", "content": content})
+        else:  # there was just text
+            history_openai_format.append({"role": "user", "content": user})
+
+        if assistant is not None:
+            history_openai_format.append({"role": "assistant", "content": assistant})
+
+    # new message
+    content = [{"type": "text",
+                "text": message['text']}]
+
+    for file in message['files']:
+        content.append({"type": "image_url",
+                        "image_url": {"url": process_image(file)}})
+
+    history_openai_format.append(
+        {"role": "user", "content": content})
 
     response = client.chat.completions.create(model=model,
                                               messages=history_openai_format,
@@ -40,6 +72,7 @@ def generate(message, history, model, system_prompt,
 
 
 chat_interface = gr.ChatInterface(
+    multimodal=True,
     title='💬 Private ChatGPT',
     description='Chat with OpenAI models using their official API. OpenAI <a href="https://platform.openai.com/docs/concepts">promises</a> not to train on input or output of API calls.',
     fn=generate,
@@ -76,14 +109,15 @@ chat_interface = gr.ChatInterface(
                   step=0.05,
                   value=0.),
     ],
-    textbox=gr.Textbox(container=False,
-                       show_label=False,
-                       label="Message",
-                       placeholder="Type een bericht...",
-                       scale=7),
+    textbox=gr.MultimodalTextbox(
+                            show_label=False,
+                            label="Message",
+                            placeholder="Type een bericht...",
+                            scale=7,
+                        ),
     additional_inputs_accordion=gr.Accordion(label="Instellingen", open=False),
     show_progress="full",
-    submit_btn="Genereer",
+    submit_btn=None,
     stop_btn="Stop",
     retry_btn="🔄 Opnieuw",
     undo_btn="↩️ Ongedaan maken",