Upload 6 files

.chainlit/config.toml

@@ -0,0 +1,63 @@
+[project]
+# If true (default), the app will be available to anonymous users.
+# If false, users will need to authenticate and be part of the project to use the app.
+public = true
+
+# The project ID (found on https://cloud.chainlit.io).
+# The project ID is required when public is set to false or when using the cloud database.
+#id = ""
+
+# Uncomment if you want to persist the chats.
+# local will create a database in your .chainlit directory (requires node.js installed).
+# cloud will use the Chainlit cloud database.
+# custom will load use your custom client.
+# database = "local"
+
+# Whether to enable telemetry (default: true). No personal data is collected.
+enable_telemetry = false
+
+# List of environment variables to be provided by each user to use the app.
+user_env = []
+
+# Duration (in seconds) during which the session is saved when the connection is lost
+session_timeout = 3600
+
+[UI]
+# Name of the app and chatbot.
+name = "Chatbot"
+
+# Description of the app and chatbot. This is used for HTML tags.
+# description = ""
+
+# The default value for the expand messages settings.
+default_expand_messages = false
+
+# Hide the chain of thought details from the user in the UI.
+hide_cot = false
+
+# Link to your github repo. This will add a github button in the UI's header.
+# github = ""
+
+# Override default MUI light theme. (Check theme.ts)
+[UI.theme.light]
+    #background = "#FAFAFA"
+    #paper = "#FFFFFF"
+
+    [UI.theme.light.primary]
+        #main = "#F80061"
+        #dark = "#980039"
+        #light = "#FFE7EB"
+
+# Override default MUI dark theme. (Check theme.ts)
+[UI.theme.dark]
+    #background = "#FAFAFA"
+    #paper = "#FFFFFF"
+
+    [UI.theme.dark.primary]
+        #main = "#F80061"
+        #dark = "#980039"
+        #light = "#FFE7EB"
+
+
+[meta]
+generated_by = "0.6.2"

Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.11
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:$PATH
+WORKDIR $HOME/app
+COPY --chown=user . $HOME/app
+COPY ./requirements.txt ~/app/requirements.txt
+RUN pip install -r requirements.txt
+COPY . .
+CMD ["chainlit", "run", "app.py", "--port", "7860"]

app.py

@@ -0,0 +1,146 @@
+from langchain.agents import AgentExecutor, AgentType, initialize_agent
+from langchain.agents.structured_chat.prompt import SUFFIX
+from langchain.chat_models import ChatOpenAI
+from langchain.memory import ConversationBufferMemory
+from tools import generate_image_tool, describe_image_tool, gpt_vision_call, process_images, handle_image_history
+
+import chainlit as cl
+from chainlit.action import Action
+from chainlit.input_widget import Select, Switch, Slider
+
+
+#@cl.author_rename
+def rename(orig_author):
+    """
+    Rename the author of messages as displayed in the "Thinking" section.
+
+    This is useful to make the chat look more natural, or add some fun to it!
+    """
+    mapping = {
+        "AgentExecutor": "The LLM Brain",
+        "LLMChain": "The Assistant",
+        "GenerateImage": "DALL-E 3",
+        "ChatOpenAI": "GPT-4 Turbo",
+        "Chatbot": "Coolest App",
+    }
+    return mapping.get(orig_author, orig_author)
+
+
+@cl.cache
+def get_memory():
+    """
+    This is used to track the conversation history and allow our agent to
+    remember what was said before.
+    """
+    return ConversationBufferMemory(memory_key="chat_history")
+
+
+@cl.on_chat_start
+async def start():
+    """
+    This is called when the Chainlit chat is started!
+
+    We can add some settings to our application to allow users to select the appropriate model, and more!
+    """
+    cl.user_session.set("image_history", [{"role": "system", "content": "You are a helpful assistant. You are developed with GPT-4-vision-preview, if the user uploads an image, you have the ability to understand it."}])
+ 
+    settings = await cl.ChatSettings(
+        [
+            Select(
+                id="Model",
+                label="OpenAI - Model",
+                values=["gpt-3.5-turbo", "gpt-4-1106-preview"],
+                initial_index=1,
+            ),
+            Switch(id="Streaming", label="OpenAI - Stream Tokens", initial=True),
+            Slider(
+                id="Temperature",
+                label="OpenAI - Temperature",
+                initial=0,
+                min=0,
+                max=2,
+                step=0.1,
+            ),
+        ]
+    ).send()
+    await setup_agent(settings)
+
+
+@cl.on_settings_update
+async def setup_agent(settings):
+    print("Setup agent with following settings: ", settings)
+
+    # We set up our agent with the user selected (or default) settings here.
+    llm = ChatOpenAI(
+        temperature=settings["Temperature"],
+        streaming=settings["Streaming"],
+        model=settings["Model"],
+    )
+
+    # We get our memory here, which is used to track the conversation history.
+    memory = get_memory()
+
+    # This suffix is used to provide the chat history to the prompt.
+    _SUFFIX = "Chat history:\n{chat_history}\n\n" + SUFFIX
+
+    # We initialize our agent here, which is simply being used to decide between responding with text
+    # or an image
+    agent = initialize_agent(
+        llm=llm,  # our LLM (default is GPT-4 Turbo)
+        tools=[
+            generate_image_tool,
+            describe_image_tool,
+        ],  # our custom tool used to generate images with DALL-E 3
+        agent=AgentType.STRUCTURED_CHAT_ZERO_SHOT_REACT_DESCRIPTION,  # the agent type we're using today
+        memory=memory,  # our memory!
+        agent_kwargs={
+            "suffix": _SUFFIX,  # adding our chat history suffix
+            "input_variables": ["input", "agent_scratchpad", "chat_history"],
+        },
+    )
+    cl.user_session.set("agent", agent)  # storing our agent in the user session
+
+
+@cl.on_message
+async def main(message: cl.Message):
+    """
+    This function is going to intercept all messages sent by the user, and
+    move through our agent flow to generate a response.
+
+    There are ultimately two different options for the agent to respond with:
+    1. Text
+    2. Image
+
+    If the agent responds with text, we simply send the text back to the user.
+
+    If the agent responds with an image, we need to generate the image and send
+    it back to the user.
+    """
+
+    if message.elements:
+        cl.user_session.set("image_id", message.elements[0].name)
+        handle_image_history(message)
+        message.content = message.content + ". image_id: " + message.elements[0].name
+
+    agent = cl.user_session.get("agent")
+    cl.user_session.set("generated_image", None)
+
+    res = await cl.make_async(agent.run)(
+        input=message.content, callbacks=[cl.LangchainCallbackHandler()]
+    )
+
+    elements = []
+    actions = []
+
+    generated_image_name = cl.user_session.get("generated_image")
+    generated_image = cl.user_session.get(generated_image_name)
+    if generated_image:
+        elements = [
+            cl.Image(
+                content=generated_image,
+                name=generated_image_name,
+                display="inline",
+            )
+        ]
+
+    await cl.Message(content=res, elements=elements, actions=actions).send()

requirements.txt

@@ -0,0 +1,4 @@
+langchain
+tiktoken
+chainlit
+openai

tools.py

@@ -0,0 +1,132 @@
+import io
+import os
+from openai import OpenAI
+from langchain.tools import StructuredTool, Tool
+from io import BytesIO
+import requests
+import json
+from io import BytesIO
+import base64
+import chainlit as cl
+
+
+def get_image_name():
+    """
+    We need to keep track of images we generate, so we can reference them later
+    and display them correctly to our users.
+    """
+    image_count = cl.user_session.get("image_count")
+    if image_count is None:
+        image_count = 0
+    else:
+        image_count += 1
+
+    cl.user_session.set("image_count", image_count)
+
+    return f"image-{image_count}"
+
+
+def _generate_image(prompt: str):
+    """
+    This function is used to generate an image from a text prompt using
+    DALL-E 3.
+
+    We use the OpenAI API to generate the image, and then store it in our
+    user session so we can reference it later.
+    """
+    client = OpenAI()
+
+    response = client.images.generate(
+        model="dall-e-3",
+        prompt=prompt,
+        size="1024x1024",
+        quality="standard",
+        n=1,
+    )
+
+    image_payload = requests.get(response.data[0].url, stream=True)
+
+    image_bytes = BytesIO(image_payload.content)
+
+    print(type(image_bytes))
+
+    name = get_image_name()
+    cl.user_session.set(name, image_bytes.getvalue())
+    cl.user_session.set("generated_image", name)
+    return name
+
+
+def generate_image(prompt: str):
+    image_name = _generate_image(prompt)
+    return f"Here is {image_name}."
+
+
+# this is our tool - which is what allows our agent to generate images in the first place!
+# the `description` field is of utmost imporance as it is what the LLM "brain" uses to determine
+# which tool to use for a given input.
+generate_image_format = '{{"prompt": "prompt"}}'
+generate_image_tool = Tool.from_function(
+    func=generate_image,
+    name="GenerateImage",
+    description=f"Useful to create an image from a text prompt. Input should be a single string strictly in the following JSON format: {generate_image_format}",
+    return_direct=True,
+)
+
+
+def gpt_vision_call(image_id: str):
+    #cl.user_session.set("image_id", image_id)
+    print("image_id", image_id)
+    client = OpenAI()
+    image_history = cl.user_session.get("image_history")
+    stream = client.chat.completions.create(
+        model="gpt-4-vision-preview",
+        messages=image_history,
+        max_tokens=350,
+        stream=False,
+    )
+
+    return stream
+
+def handle_image_history(msg):
+    image_history = cl.user_session.get("image_history")
+    image_base64 = None
+    image_base64 = process_images(msg)
+    
+    if image_base64:
+        # add the image to the image history
+        image_history.append(
+        {
+            "role": "user",
+            "content": [
+                    {"type": "text", "text": msg.content},
+                    {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": f"data:image/jpeg;base64,{image_base64}",
+                            "detail": "low"
+                        }
+                    },
+                ],
+            }
+        )
+        cl.user_session.set("image_history", image_history)
+
+
+def process_images(msg: cl.Message):
+    # Processing images exclusively
+    images = [file for file in msg.elements if "image" in file.mime]
+
+    # Accessing the bytes of a specific image
+    image_bytes = images[0].content # take the first image just for demo purposes
+    
+    # we need base64 encoded image
+    image_base64 = base64.b64encode(image_bytes).decode('utf-8')
+    return image_base64
+
+describe_image_format = '{{"image_id": "image_id"}}'
+describe_image_tool = Tool.from_function(
+    func=gpt_vision_call,
+    name="DescribeImage",
+    description=f"Useful to describe an image. Input should be a single string strictly in the following JSON format: {describe_image_format}",
+    return_direct=False,
+)