Initial commit

.chainlit/config.toml

@@ -0,0 +1,84 @@
+[project]
+# Whether to enable telemetry (default: true). No personal data is collected.
+enable_telemetry = true
+
+# List of environment variables to be provided by each user to use the app.
+user_env = []
+
+# Duration (in seconds) during which the session is saved when the connection is lost
+session_timeout = 3600
+
+# Enable third parties caching (e.g LangChain cache)
+cache = false
+
+# Follow symlink for asset mount (see https://github.com/Chainlit/chainlit/issues/317)
+# follow_symlink = false
+
+[features]
+# Show the prompt playground
+prompt_playground = true
+
+# Process and display HTML in messages. This can be a security risk (see https://stackoverflow.com/questions/19603097/why-is-it-dangerous-to-render-user-generated-html-or-javascript)
+unsafe_allow_html = false
+
+# Process and display mathematical expressions. This can clash with "$" characters in messages.
+latex = false
+
+# Authorize users to upload files with messages
+multi_modal = true
+
+# Allows user to use speech to text
+[features.speech_to_text]
+    enabled = false
+    # See all languages here https://github.com/JamesBrill/react-speech-recognition/blob/HEAD/docs/API.md#language-string
+    # language = "en-US"
+
+[UI]
+# Name of the app and chatbot.
+name = "RAGalicious"
+
+# Show the readme while the thread is empty.
+show_readme_as_default = true
+
+# Description of the app and chatbot. This is used for HTML tags.
+# description = ""
+
+# Large size content are by default collapsed for a cleaner ui
+default_collapse_content = true
+
+# The default value for the expand messages settings.
+default_expand_messages = false
+
+# Hide the chain of thought details from the user in the UI.
+hide_cot = false
+
+# Link to your github repo. This will add a github button in the UI's header.
+# github = "https://github.com/mickkhaw1981/ragalicious"
+
+# Specify a CSS file that can be used to customize the user interface.
+# The CSS file can be served from the public directory or via an external link.
+# custom_css = "/public/test.css"
+
+# Override default MUI light theme. (Check theme.ts)
+[UI.theme.light]
+    #background = "#FAFAFA"
+    #paper = "#FFFFFF"
+
+    [UI.theme.light.primary]
+        #main = "#F80061"
+        #dark = "#980039"
+        #light = "#FFE7EB"
+
+# Override default MUI dark theme. (Check theme.ts)
+[UI.theme.dark]
+    #background = "#FAFAFA"
+    #paper = "#FFFFFF"
+
+    [UI.theme.dark.primary]
+        #main = "#F80061"
+        #dark = "#980039"
+        #light = "#FFE7EB"
+
+
+[meta]
+generated_by = "1.0.0"

.chainlit/translations/en-US.json

@@ -0,0 +1,231 @@
+{
+    "components": {
+        "atoms": {
+            "buttons": {
+                "userButton": {
+                    "menu": {
+                        "settings": "Settings",
+                        "settingsKey": "S",
+                        "APIKeys": "API Keys",
+                        "logout": "Logout"
+                    }
+                }
+            }
+        },
+        "molecules": {
+            "newChatButton": {
+                "newChat": "New Chat"
+            },
+            "tasklist": {
+                "TaskList": {
+                    "title": "\ud83d\uddd2\ufe0f Task List",
+                    "loading": "Loading...",
+                    "error": "An error occured"
+                }
+            },
+            "attachments": {
+                "cancelUpload": "Cancel upload",
+                "removeAttachment": "Remove attachment"
+            },
+            "newChatDialog": {
+                "createNewChat": "Create new chat?",
+                "clearChat": "This will clear the current messages and start a new chat.",
+                "cancel": "Cancel",
+                "confirm": "Confirm"
+            },
+            "settingsModal": {
+                "settings": "Settings",
+                "expandMessages": "Expand Messages",
+                "hideChainOfThought": "Hide Chain of Thought",
+                "darkMode": "Dark Mode"
+            },
+            "detailsButton": {
+                "using": "Using",
+                "running": "Running",
+                "took_one": "Took {{count}} step",
+                "took_other": "Took {{count}} steps"
+            },
+            "auth": {
+                "authLogin": {
+                    "title": "Login to access the app.",
+                    "form": {
+                        "email": "Email address",
+                        "password": "Password",
+                        "noAccount": "Don't have an account?",
+                        "alreadyHaveAccount": "Already have an account?",
+                        "signup": "Sign Up",
+                        "signin": "Sign In",
+                        "or": "OR",
+                        "continue": "Continue",
+                        "forgotPassword": "Forgot password?",
+                        "passwordMustContain": "Your password must contain:",
+                        "emailRequired": "email is a required field",
+                        "passwordRequired": "password is a required field"
+                    },
+                    "error": {
+                        "default": "Unable to sign in.",
+                        "signin": "Try signing in with a different account.",
+                        "oauthsignin": "Try signing in with a different account.",
+                        "redirect_uri_mismatch": "The redirect URI is not matching the oauth app configuration.",
+                        "oauthcallbackerror": "Try signing in with a different account.",
+                        "oauthcreateaccount": "Try signing in with a different account.",
+                        "emailcreateaccount": "Try signing in with a different account.",
+                        "callback": "Try signing in with a different account.",
+                        "oauthaccountnotlinked": "To confirm your identity, sign in with the same account you used originally.",
+                        "emailsignin": "The e-mail could not be sent.",
+                        "emailverify": "Please verify your email, a new email has been sent.",
+                        "credentialssignin": "Sign in failed. Check the details you provided are correct.",
+                        "sessionrequired": "Please sign in to access this page."
+                    }
+                },
+                "authVerifyEmail": {
+                    "almostThere": "You're almost there! We've sent an email to ",
+                    "verifyEmailLink": "Please click on the link in that email to complete your signup.",
+                    "didNotReceive": "Can't find the email?",
+                    "resendEmail": "Resend email",
+                    "goBack": "Go Back",
+                    "emailSent": "Email sent successfully.",
+                    "verifyEmail": "Verify your email address"
+                },
+                "providerButton": {
+                    "continue": "Continue with {{provider}}",
+                    "signup": "Sign up with {{provider}}"
+                },
+                "authResetPassword": {
+                    "newPasswordRequired": "New password is a required field",
+                    "passwordsMustMatch": "Passwords must match",
+                    "confirmPasswordRequired": "Confirm password is a required field",
+                    "newPassword": "New password",
+                    "confirmPassword": "Confirm password",
+                    "resetPassword": "Reset Password"
+                },
+                "authForgotPassword": {
+                    "email": "Email address",
+                    "emailRequired": "email is a required field",
+                    "emailSent": "Please check the email address {{email}} for instructions to reset your password.",
+                    "enterEmail": "Enter your email address and we will send you instructions to reset your password.",
+                    "resendEmail": "Resend email",
+                    "continue": "Continue",
+                    "goBack": "Go Back"
+                }
+            }
+        },
+        "organisms": {
+            "chat": {
+                "history": {
+                    "index": {
+                        "showHistory": "Show history",
+                        "lastInputs": "Last Inputs",
+                        "noInputs": "Such empty...",
+                        "loading": "Loading..."
+                    }
+                },
+                "inputBox": {
+                    "input": {
+                        "placeholder": "Type your message here..."
+                    },
+                    "speechButton": {
+                        "start": "Start recording",
+                        "stop": "Stop recording"
+                    },
+                    "SubmitButton": {
+                        "sendMessage": "Send message",
+                        "stopTask": "Stop Task"
+                    },
+                    "UploadButton": {
+                        "attachFiles": "Attach files"
+                    },
+                    "waterMark": {
+                        "text": "Built with"
+                    }
+                },
+                "Messages": {
+                    "index": {
+                        "running": "Running",
+                        "executedSuccessfully": "executed successfully",
+                        "failed": "failed",
+                        "feedbackUpdated": "Feedback updated",
+                        "updating": "Updating"
+                    }
+                },
+                "dropScreen": {
+                    "dropYourFilesHere": "Drop your files here"
+                },
+                "index": {
+                    "failedToUpload": "Failed to upload",
+                    "cancelledUploadOf": "Cancelled upload of",
+                    "couldNotReachServer": "Could not reach the server",
+                    "continuingChat": "Continuing previous chat"
+                },
+                "settings": {
+                    "settingsPanel": "Settings panel",
+                    "reset": "Reset",
+                    "cancel": "Cancel",
+                    "confirm": "Confirm"
+                }
+            },
+            "threadHistory": {
+                "sidebar": {
+                    "filters": {
+                        "FeedbackSelect": {
+                            "feedbackAll": "Feedback: All",
+                            "feedbackPositive": "Feedback: Positive",
+                            "feedbackNegative": "Feedback: Negative"
+                        },
+                        "SearchBar": {
+                            "search": "Search"
+                        }
+                    },
+                    "DeleteThreadButton": {
+                        "confirmMessage": "This will delete the thread as well as it's messages and elements.",
+                        "cancel": "Cancel",
+                        "confirm": "Confirm",
+                        "deletingChat": "Deleting chat",
+                        "chatDeleted": "Chat deleted"
+                    },
+                    "index": {
+                        "pastChats": "Past Chats"
+                    },
+                    "ThreadList": {
+                        "empty": "Empty...",
+                        "today": "Today",
+                        "yesterday": "Yesterday",
+                        "previous7days": "Previous 7 days",
+                        "previous30days": "Previous 30 days"
+                    },
+                    "TriggerButton": {
+                        "closeSidebar": "Close sidebar",
+                        "openSidebar": "Open sidebar"
+                    }
+                },
+                "Thread": {
+                    "backToChat": "Go back to chat",
+                    "chatCreatedOn": "This chat was created on"
+                }
+            },
+            "header": {
+                "chat": "Chat",
+                "readme": "Readme"
+            }
+        }
+    },
+    "hooks": {
+        "useLLMProviders": {
+            "failedToFetchProviders": "Failed to fetch providers:"
+        }
+    },
+    "pages": {
+        "Design": {},
+        "Env": {
+            "savedSuccessfully": "Saved successfully",
+            "requiredApiKeys": "Required API Keys",
+            "requiredApiKeysInfo": "To use this app, the following API keys are required. The keys are stored on your device's local storage."
+        },
+        "Page": {
+            "notPartOfProject": "You are not part of this project."
+        },
+        "ResumeButton": {
+            "resumeChat": "Resume Chat"
+        }
+    }
+}

.env.sample

@@ -0,0 +1,7 @@
+OPENAI_API_KEY=
+MYSCALE_HOST=
+MYSCALE_USERNAME=
+MYSCALE_PASSWORD=
+TWILIO_ACCOUNT_SID=
+TWILIO_AUTH_TOKEN=
+SUPABASE_SECRET_KEY=

.gitattributes

@@ -0,0 +1,37 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.faiss filter=lfs diff=lfs merge=lfs -text
+*.weba filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,167 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Audio files
+*.weba
+.files/**/*.weba
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# VS CODE
+.vscode/

Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.10.12
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:$PATH
+WORKDIR $HOME/app
+COPY --chown=user . $HOME/app
+COPY ./requirements.txt ~/app/requirements.txt
+RUN pip install -r requirements.txt
+COPY . .
+CMD ["chainlit", "run", "app.py", "--port", "7860"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 mickkhaw1981
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,19 @@
+---
+title: RAGalicious App (AIE3)
+emoji: 👩‍🍳
+colorFrom: purple
+colorTo: blue
+sdk: docker
+pinned: false
+license: mit
+---
+
+# 😋 Welcome to RAGalicious!
+
+RAGalicious is a chatbot designed to help users discover delectable recipes from the NYTimes Cooking section. It simplifies the recipe discovery process, providing users inspiration based on ingredients, diet, occasion or equipment you have.
+
+## Example prompts
+
+- **Plan your daily meals:** E.g, "Give me ideas for making an easy weeknight dinner."
+- **Get ready to host occasions:** E.g. "What are good dishes to make for Rosh Hashanah?"
+- **Get scrappy with ingredients you already have:** E.g. "What can I make with pasta, lemon and chickpeas?"

app.py

@@ -0,0 +1,200 @@
+from io import BytesIO
+import os
+from pprint import pprint
+import uuid
+import chainlit as cl
+from chainlit.element import ElementBased
+from dotenv import load_dotenv
+
+# modules for audio processing
+import httpx
+from langchain.schema.runnable.config import RunnableConfig
+from langchain_openai.chat_models import ChatOpenAI
+from openai import AsyncOpenAI
+
+from utils.graph import generate_workflow
+
+client = AsyncOpenAI()
+
+# ---- ENV VARIABLES ---- #
+load_dotenv()
+OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
+#QDRANT_CLOUD_KEY = os.environ.get("QDRANT_CLOUD_KEY")
+#QDRANT_CLOUD_URL = "https://30591e3d-7092-41c4-95e1-4d3c7ef6e894.us-east4-0.gcp.cloud.qdrant.io"
+#ELEVENLABS_API_KEY = os.environ.get("ELEVENLABS_API_KEY")
+#ELEVENLABS_VOICE_ID = os.environ.get("ELEVENLABS_VOICE_ID")
+
+
+# -- AUGMENTED -- #
+
+# Define the LLM
+base_llm = ChatOpenAI(model="gpt-4o-mini", openai_api_key=OPENAI_API_KEY, tags=["base_llm"], temperature=0)
+power_llm = ChatOpenAI(model="gpt-4o", openai_api_key=OPENAI_API_KEY, tags=["base_llm"])
+
+
+# Conversation starters for the 1st screen
+@cl.set_starters
+async def set_starters():
+    return [
+        cl.Starter(
+            label="Plan your quick daily meals",
+            message="Give me ideas for making an easy weeknight dinner that takes less than 25 minutes to prepare",
+            icon="/public/meals4.svg",
+        ),
+        cl.Starter(
+            label="Ideas for special occasions",
+            message="What are good Middle Eastern dishes to make for Thanksgiving?",
+            icon="/public/occasion4.svg",
+        ),
+        cl.Starter(
+            label="Use ingredients you have",
+            message="Suggest Spanish recipes that are good for the summer that makes use of tomatoes",
+            icon="/public/ingredients4.svg",
+        ),
+    ]
+
+
+# Chat Start Function: Initialize a RAG (Retrieval-Augmented Generation) chain at the start of each chat session.
+@cl.on_chat_start
+async def start_chat():
+    """
+    This function will be called at the start of every user session.
+    We will build our LCEL RAG chain here, and store it in the user session.
+    The user session is a dictionary that is unique to each user session, and is stored in the memory of the server.
+    """
+
+    langgraph_chain = generate_workflow(base_llm, power_llm)
+
+    cl.user_session.set("langgraph_chain", langgraph_chain)
+    cl.user_session.set("thread_id", str(uuid.uuid4()))
+
+
+# Message Handling Function: Process and respond to user messages using the RAG chain.
+@cl.on_message
+async def main(message: cl.Message):
+    """
+    This function will be called every time a message is recieved from a session.
+    We will use the LCEL RAG chain to generate a response to the user question.
+    The LCEL RAG chain is stored in the user session, and is unique to each user session - this is why we can access it here.
+    """
+
+    langgraph_chain = cl.user_session.get("langgraph_chain")
+    thread_id = cl.user_session.get("thread_id")
+    msg = cl.Message(content="")
+    langgraph_config = {"configurable": {"thread_id": thread_id, "cl_msg": msg}}
+
+    async for output in langgraph_chain.astream({"question": message.content}, langgraph_config):
+        for key, value in output.items():
+            pprint(f"================== Node: '{key}':")
+
+    await msg.send()
+
+
+# Speech-to-Text Function: Convert audio file to text
+@cl.step(type="tool")
+async def speech_to_text(audio_file):
+    response = await client.audio.transcriptions.create(model="whisper-1", file=audio_file)
+    return response.text
+
+
+# Generate Text Answer Function: Take the output of Speech-to-Text and generate a text answer
+@cl.step(type="tool")
+async def generate_text_answer(transcription):
+    base_rag_chain = cl.user_session.get("base_rag_chain")
+    msg = cl.Message(content="")
+    async for chunk in base_rag_chain.astream(
+        {"question": transcription},
+        config=RunnableConfig(callbacks=[cl.LangchainCallbackHandler()]),
+    ):
+        if isinstance(chunk, dict) and "response" in chunk and isinstance(chunk["response"], str):
+            await msg.stream_token(chunk["response"])
+
+    return msg.content
+
+
+# Text-to-Speech Function: Take the text answer generated and convert it to an audio file
+# @cl.step(type="tool")
+# async def text_to_speech(text: str, mime_type: str):
+#     CHUNK_SIZE = 2048  # try 4096 or 8192 if getting read timeout error. the bigger the chunk size, the fewer API calls but longer wait time
+#     url = f"https://api.elevenlabs.io/v1/text-to-speech/{ELEVENLABS_VOICE_ID}"
+#     headers = {"Accept": mime_type, "Content-Type": "application/json", "xi-api-key": ELEVENLABS_API_KEY}
+#     data = {
+#         "text": text,
+#         "model_id": "eleven_monolingual_v1",
+#         "voice_settings": {"stability": 0.5, "similarity_boost": 0.5},
+#     }
+
+#     # make an async HTTP POST request to the ElevenLabs API to convert text to speech and return an audio file
+#     async with httpx.AsyncClient(timeout=60.0) as client:
+#         response = await client.post(url, json=data, headers=headers)
+#         response.raise_for_status()  # Ensure we notice bad responses
+#         buffer = BytesIO()
+#         buffer.name = f"output_audio.{mime_type.split('/')[1]}"
+#         async for chunk in response.aiter_bytes(chunk_size=CHUNK_SIZE):
+#             if chunk:
+#                 buffer.write(chunk)
+
+#         buffer.seek(0)
+#         return buffer.name, buffer.read()
+
+
+# ---- AUDIO PROCESSING ---- #
+
+
+# Audio Chunk Function: Process audio chunks as they arrive from the user's microphone
+@cl.on_audio_chunk
+async def on_audio_chunk(chunk: cl.AudioChunk):
+    if chunk.isStart:
+        buffer = BytesIO()
+        # This is required for whisper to recognize the file type
+        buffer.name = f"input_audio.{chunk.mimeType.split('/')[1]}"
+        # Initialize the session for a new audio stream
+        cl.user_session.set("audio_buffer", buffer)
+        cl.user_session.set("audio_mime_type", chunk.mimeType)
+
+    # For now, write the chunks to a buffer and transcribe the whole audio at the end
+    cl.user_session.get("audio_buffer").write(chunk.data)
+
+
+# Audio End Function: Process the audio file and generate a response
+@cl.on_audio_end
+async def on_audio_end(elements: list[ElementBased]):
+    # Get the audio buffer from the session
+    audio_buffer: BytesIO = cl.user_session.get("audio_buffer")
+    audio_buffer.seek(0)  # Move the file pointer to the beginning
+    audio_file = audio_buffer.read()
+    audio_mime_type: str = cl.user_session.get("audio_mime_type")
+    input_audio_el = cl.Audio(mime=audio_mime_type, content=audio_file, name=audio_buffer.name)
+    await cl.Message(author="You", type="user_message", content="", elements=[input_audio_el, *elements]).send()
+
+    whisper_input = (audio_buffer.name, audio_file, audio_mime_type)
+
+    transcription = await speech_to_text(whisper_input)
+    print("Transcription: ", transcription)
+
+    langgraph_chain = cl.user_session.get("langgraph_chain")
+    thread_id = cl.user_session.get("thread_id")
+    msg = cl.Message(content="")
+    langgraph_config = {"configurable": {"thread_id": thread_id, "cl_msg": msg}}
+
+    async for output in langgraph_chain.astream({"question": transcription}, langgraph_config):
+        for key, value in output.items():
+            pprint(f"================== Node: '{key}':")
+
+    await msg.send()
+
+    # text_answer = await generate_text_answer(
+    #     transcription
+    # )  # need to change this to generate answer based on base_rag_chain
+
+    # output_name, output_audio = await text_to_speech(text_answer, audio_mime_type)
+
+    # output_audio_el = cl.Audio(
+    #     name=output_name,
+    #     auto_play=True,
+    #     mime=audio_mime_type,
+    #     content=output_audio,
+    # )
+    # answer_message = await cl.Message(content="").send()
+    # answer_message.elements = [output_audio_el]
+    # await answer_message.update()

chainlit.md

@@ -0,0 +1,9 @@
+# 😋 Welcome to RAGalicious!
+
+RAGalicious is a chatbot designed to help users discover delectable recipes from the NYTimes Cooking section. It simplifies the recipe discovery process, providing users inspiration based on ingredients, diet, occasion or equipment you have.
+
+## Example prompts
+
+- **Plan your daily meals:** E.g, "Give me ideas for making an easy weeknight dinner."
+- **Get ready to host occasions:** E.g. "What are good dishes to make for Rosh Hashanah?"
+- **Get scrappy with ingredients you already have:** E.g. "What can I make with pasta, lemon and chickpeas?"

ragalicious.code-workspace

@@ -0,0 +1,7 @@
+{
+    "folders": [
+        {
+            "path": "."
+        }
+    ]
+}

requirements.in

@@ -0,0 +1,17 @@
+chainlit==1.1.306
+langchain==0.2.7
+langchain_core==0.2.15
+langchain_openai==0.1.15
+langchain-qdrant==0.1.1
+langchain_community==0.2.5
+langchain_huggingface==0.0.3
+langchain_text_splitters==0.2.1
+qdrant-client==1.10.1
+openai==1.35.13
+python-dotenv==1.0.1
+lark 
+clickhouse-connect
+langgraph
+aiosqlite
+supabase
+twilio

requirements.txt

@@ -0,0 +1,442 @@
+#
+# This file is autogenerated by pip-compile with Python 3.12
+# by the following command:
+#
+#    pip-compile requirements.in
+#
+aiofiles==23.2.1
+    # via chainlit
+aiohttp==3.9.5
+    # via
+    #   aiohttp-retry
+    #   langchain
+    #   langchain-community
+    #   twilio
+aiohttp-retry==2.8.3
+    # via twilio
+aiosignal==1.3.1
+    # via aiohttp
+aiosqlite==0.20.0
+    # via -r requirements.in
+annotated-types==0.7.0
+    # via pydantic
+anyio==3.7.1
+    # via
+    #   asyncer
+    #   httpx
+    #   openai
+    #   starlette
+    #   watchfiles
+asyncer==0.0.2
+    # via chainlit
+attrs==23.2.0
+    # via aiohttp
+bidict==0.23.1
+    # via python-socketio
+certifi==2024.7.4
+    # via
+    #   clickhouse-connect
+    #   httpcore
+    #   httpx
+    #   requests
+chainlit==1.1.306
+    # via -r requirements.in
+charset-normalizer==3.3.2
+    # via requests
+chevron==0.14.0
+    # via literalai
+click==8.1.7
+    # via
+    #   chainlit
+    #   uvicorn
+clickhouse-connect==0.7.16
+    # via -r requirements.in
+dataclasses-json==0.5.14
+    # via
+    #   chainlit
+    #   langchain-community
+deprecated==1.2.14
+    # via
+    #   opentelemetry-api
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+deprecation==2.1.0
+    # via postgrest
+distro==1.9.0
+    # via openai
+fastapi==0.110.3
+    # via chainlit
+filelock==3.15.4
+    # via
+    #   huggingface-hub
+    #   torch
+    #   transformers
+filetype==1.2.0
+    # via chainlit
+frozenlist==1.4.1
+    # via
+    #   aiohttp
+    #   aiosignal
+fsspec==2024.6.1
+    # via
+    #   huggingface-hub
+    #   torch
+googleapis-common-protos==1.63.2
+    # via
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+gotrue==2.6.0
+    # via supabase
+greenlet==3.0.3
+    # via sqlalchemy
+grpcio==1.65.1
+    # via
+    #   grpcio-tools
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   qdrant-client
+grpcio-tools==1.62.2
+    # via qdrant-client
+h11==0.14.0
+    # via
+    #   httpcore
+    #   uvicorn
+    #   wsproto
+h2==4.1.0
+    # via httpx
+hpack==4.0.0
+    # via h2
+httpcore==1.0.5
+    # via httpx
+httpx[http2]==0.27.0
+    # via
+    #   chainlit
+    #   gotrue
+    #   literalai
+    #   openai
+    #   postgrest
+    #   qdrant-client
+    #   storage3
+    #   supabase
+    #   supafunc
+huggingface-hub==0.24.0
+    # via
+    #   langchain-huggingface
+    #   sentence-transformers
+    #   tokenizers
+    #   transformers
+hyperframe==6.0.1
+    # via h2
+idna==3.7
+    # via
+    #   anyio
+    #   httpx
+    #   requests
+    #   yarl
+importlib-metadata==7.1.0
+    # via opentelemetry-api
+jinja2==3.1.4
+    # via torch
+joblib==1.4.2
+    # via scikit-learn
+jsonpatch==1.33
+    # via langchain-core
+jsonpointer==3.0.0
+    # via jsonpatch
+langchain==0.2.7
+    # via
+    #   -r requirements.in
+    #   langchain-community
+langchain-community==0.2.5
+    # via -r requirements.in
+langchain-core==0.2.15
+    # via
+    #   -r requirements.in
+    #   langchain
+    #   langchain-community
+    #   langchain-huggingface
+    #   langchain-openai
+    #   langchain-qdrant
+    #   langchain-text-splitters
+    #   langgraph
+langchain-huggingface==0.0.3
+    # via -r requirements.in
+langchain-openai==0.1.15
+    # via -r requirements.in
+langchain-qdrant==0.1.1
+    # via -r requirements.in
+langchain-text-splitters==0.2.1
+    # via
+    #   -r requirements.in
+    #   langchain
+langgraph==0.1.8
+    # via -r requirements.in
+langsmith==0.1.93
+    # via
+    #   langchain
+    #   langchain-community
+    #   langchain-core
+lark==1.1.9
+    # via -r requirements.in
+lazify==0.4.0
+    # via chainlit
+literalai==0.0.607
+    # via chainlit
+lz4==4.3.3
+    # via clickhouse-connect
+markupsafe==2.1.5
+    # via jinja2
+marshmallow==3.21.3
+    # via dataclasses-json
+mpmath==1.3.0
+    # via sympy
+multidict==6.0.5
+    # via
+    #   aiohttp
+    #   yarl
+mypy-extensions==1.0.0
+    # via typing-inspect
+nest-asyncio==1.6.0
+    # via chainlit
+networkx==3.2.1
+    # via torch
+numpy==1.26.4
+    # via
+    #   chainlit
+    #   langchain
+    #   langchain-community
+    #   qdrant-client
+    #   scikit-learn
+    #   scipy
+    #   sentence-transformers
+    #   transformers
+openai==1.35.13
+    # via
+    #   -r requirements.in
+    #   langchain-openai
+opentelemetry-api==1.25.0
+    # via
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+    #   opentelemetry-instrumentation
+    #   opentelemetry-sdk
+    #   opentelemetry-semantic-conventions
+    #   uptrace
+opentelemetry-exporter-otlp==1.25.0
+    # via uptrace
+opentelemetry-exporter-otlp-proto-common==1.25.0
+    # via
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+opentelemetry-exporter-otlp-proto-grpc==1.25.0
+    # via opentelemetry-exporter-otlp
+opentelemetry-exporter-otlp-proto-http==1.25.0
+    # via opentelemetry-exporter-otlp
+opentelemetry-instrumentation==0.46b0
+    # via uptrace
+opentelemetry-proto==1.25.0
+    # via
+    #   opentelemetry-exporter-otlp-proto-common
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+opentelemetry-sdk==1.25.0
+    # via
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+    #   uptrace
+opentelemetry-semantic-conventions==0.46b0
+    # via opentelemetry-sdk
+orjson==3.10.6
+    # via langsmith
+packaging==23.2
+    # via
+    #   chainlit
+    #   deprecation
+    #   huggingface-hub
+    #   langchain-core
+    #   literalai
+    #   marshmallow
+    #   transformers
+pillow==10.4.0
+    # via sentence-transformers
+portalocker==2.10.1
+    # via qdrant-client
+postgrest==0.16.9
+    # via supabase
+protobuf==4.25.3
+    # via
+    #   googleapis-common-protos
+    #   grpcio-tools
+    #   opentelemetry-proto
+pydantic==2.8.2
+    # via
+    #   chainlit
+    #   fastapi
+    #   gotrue
+    #   langchain
+    #   langchain-core
+    #   langsmith
+    #   literalai
+    #   openai
+    #   postgrest
+    #   qdrant-client
+pydantic-core==2.20.1
+    # via pydantic
+pyjwt==2.8.0
+    # via
+    #   chainlit
+    #   twilio
+python-dateutil==2.9.0.post0
+    # via
+    #   realtime
+    #   storage3
+python-dotenv==1.0.1
+    # via
+    #   -r requirements.in
+    #   chainlit
+python-engineio==4.9.1
+    # via python-socketio
+python-multipart==0.0.9
+    # via chainlit
+python-socketio==5.11.3
+    # via chainlit
+pytz==2024.1
+    # via clickhouse-connect
+pyyaml==6.0.1
+    # via
+    #   huggingface-hub
+    #   langchain
+    #   langchain-community
+    #   langchain-core
+    #   transformers
+qdrant-client==1.10.1
+    # via
+    #   -r requirements.in
+    #   langchain-qdrant
+realtime==1.0.6
+    # via supabase
+regex==2024.5.15
+    # via
+    #   tiktoken
+    #   transformers
+requests==2.32.3
+    # via
+    #   huggingface-hub
+    #   langchain
+    #   langchain-community
+    #   langsmith
+    #   opentelemetry-exporter-otlp-proto-http
+    #   tiktoken
+    #   transformers
+    #   twilio
+safetensors==0.4.3
+    # via transformers
+scikit-learn==1.5.1
+    # via sentence-transformers
+scipy==1.13.1
+    # via
+    #   scikit-learn
+    #   sentence-transformers
+sentence-transformers==3.0.1
+    # via langchain-huggingface
+simple-websocket==1.0.0
+    # via python-engineio
+six==1.16.0
+    # via python-dateutil
+sniffio==1.3.1
+    # via
+    #   anyio
+    #   httpx
+    #   openai
+sqlalchemy==2.0.31
+    # via
+    #   langchain
+    #   langchain-community
+starlette==0.37.2
+    # via
+    #   chainlit
+    #   fastapi
+storage3==0.7.7
+    # via supabase
+strenum==0.4.15
+    # via postgrest
+supabase==2.5.3
+    # via -r requirements.in
+supafunc==0.4.7
+    # via supabase
+sympy==1.13.1
+    # via torch
+syncer==2.0.3
+    # via chainlit
+tenacity==8.5.0
+    # via
+    #   langchain
+    #   langchain-community
+    #   langchain-core
+threadpoolctl==3.5.0
+    # via scikit-learn
+tiktoken==0.7.0
+    # via langchain-openai
+tokenizers==0.19.1
+    # via
+    #   langchain-huggingface
+    #   transformers
+tomli==2.0.1
+    # via chainlit
+torch==2.2.2
+    # via sentence-transformers
+tqdm==4.66.4
+    # via
+    #   huggingface-hub
+    #   openai
+    #   sentence-transformers
+    #   transformers
+transformers==4.42.4
+    # via
+    #   langchain-huggingface
+    #   sentence-transformers
+twilio==9.2.3
+    # via -r requirements.in
+typing-extensions==4.12.2
+    # via
+    #   aiosqlite
+    #   fastapi
+    #   huggingface-hub
+    #   openai
+    #   opentelemetry-sdk
+    #   pydantic
+    #   pydantic-core
+    #   realtime
+    #   sqlalchemy
+    #   storage3
+    #   torch
+    #   typing-inspect
+typing-inspect==0.9.0
+    # via dataclasses-json
+uptrace==1.24.0
+    # via chainlit
+urllib3==2.2.2
+    # via
+    #   clickhouse-connect
+    #   qdrant-client
+    #   requests
+uvicorn==0.25.0
+    # via chainlit
+watchfiles==0.20.0
+    # via chainlit
+websockets==12.0
+    # via realtime
+wrapt==1.16.0
+    # via
+    #   deprecated
+    #   opentelemetry-instrumentation
+wsproto==1.2.0
+    # via simple-websocket
+yarl==1.9.4
+    # via aiohttp
+zipp==3.19.2
+    # via importlib-metadata
+zstandard==0.23.0
+    # via clickhouse-connect
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

utils/db.py

@@ -0,0 +1,57 @@
+import os
+from supabase import Client, create_client
+
+SUPABASE_URL: str = "https://cfivdlyedzbcvjsztebc.supabase.co"
+SUPABASE_SECRET_KEY: str = os.environ["SUPABASE_SECRET_KEY"]
+
+
+def get_client() -> Client:
+    return create_client(SUPABASE_URL, SUPABASE_SECRET_KEY)
+
+
+def get_recipes(urls: list):
+    supabase_client = get_client()
+    response = (
+        supabase_client.table("recipes")
+        .select("url, metadata, features, md_ingredients, md_preparation, md_nutrition, md_description, time")
+        .in_("url", urls)
+        .limit(10)
+        .execute()
+    )
+
+    return [
+        {
+            "title": r["metadata"]["title"],
+            "thumbnail": r["metadata"].get("thumbnail"),
+            "url": r["url"],
+            "text": f"""
+            TITLE: \n
+            {r['metadata']['title']}
+            \n\n
+            ESTIMATED COOKING / PREPARATION TIME: {r['time']} minutes
+            \n\n
+            DESCRIPTION: \n
+            {r['md_description']}
+            \n\n
+            INGREDIENTS: \n
+            {r['md_ingredients']}
+            NUTRITIONAL INFORMATION: \n
+            {r['md_nutrition']}
+            \n\n
+            PREP INSTRUCTIONS: \n
+            {r['md_preparation']}
+
+            Source URL: {r['url']}
+            \n\n
+        """,
+        }
+        for r in response.data
+    ]
+
+
+def shortlisted_recipes_to_string(recipes):
+    output = ""
+    if recipes and isinstance(recipes, list):
+        for index, r in enumerate(recipes):
+            output += f"""Suggestion #{index+1}: {r['text']} \n\n"""
+    return output

utils/debug.py

@@ -0,0 +1,9 @@
+def retriever_output_logger(documents):
+    print("returning total results count: ", len(documents))
+    for doc in documents:
+        print(f"""*** {doc.metadata['title']}
+            > Prep Time: {doc.metadata['time']}
+            > Occasion: {doc.metadata['occasion']}
+            > Cuisine: {doc.metadata['cuisine']}
+            > Ingredients: {doc.metadata['ingredients']}""")
+    return documents

utils/graph.py

@@ -0,0 +1,376 @@
+import json
+import operator
+from pprint import pprint
+from typing import Annotated, List, TypedDict
+import chainlit as cl
+from langchain.prompts import ChatPromptTemplate
+from langchain.schema.runnable.config import RunnableConfig
+from langchain_core.messages import AIMessageChunk, FunctionMessage
+from langchain_core.utils.function_calling import convert_to_openai_function
+from langgraph.checkpoint.aiosqlite import AsyncSqliteSaver
+from langgraph.graph import END, START, StateGraph
+from langgraph.graph.message import add_messages
+from langgraph.prebuilt import ToolExecutor, ToolInvocation
+
+from utils.tools import send_text_tool
+from .db import get_recipes, shortlisted_recipes_to_string
+from .graph_chains import (
+    get_grader_chain,
+    get_question_type_chain,
+    get_recipe_url_extractor_chain,
+    get_selected_recipe,
+)
+from .retrievers import get_self_retriever
+
+
+class AgentState(TypedDict):
+    question: Annotated[str, operator.setitem]
+    question_type: str
+    generation: str
+    documents: List[str]
+    shortlisted_recipes: List[dict]
+    selected_recipe: dict
+    messages: Annotated[list, add_messages]
+
+
+def generate_workflow(base_llm, power_llm):
+    def _node_question_triage(state: AgentState):
+        print("---TRIAGE---")
+        question = state["question"]
+        messages = state["messages"]
+        last_message = messages[-1] if messages else ""
+        shortlisted_recipes = state.get("shortlisted_recipes")
+        question_type_chain = get_question_type_chain(base_llm)
+        question_type_response = question_type_chain.invoke(
+            {
+                "question": question,
+                "context": shortlisted_recipes_to_string(shortlisted_recipes),
+                "last_message": last_message,
+            }
+        )
+        question_type_response_data = sorted(
+            [
+                (question_type_response.send_text, "send_sms"),
+                (question_type_response.asking_for_recipe_suggestions, "asking_for_recipe_suggestions"),
+                (question_type_response.referring_to_shortlisted_recipes, "referring_to_shortlisted_recipes"),
+                (question_type_response.show_specific_recipe, "show_specific_recipe"),
+                (question_type_response.referring_to_specific_recipe, "referring_to_specific_recipe"),
+            ],
+            key=lambda x: x[0],
+            reverse=True,
+        )
+
+        pprint(question_type_response_data)
+        question_type = question_type_response_data[0][1]
+        selected_recipe = None
+        if shortlisted_recipes and question_type_response.specific_recipe_url:
+            selected_recipe = next(
+                (r for r in shortlisted_recipes if r["url"] == question_type_response.specific_recipe_url)
+            )
+            print("set selected recipe", question_type_response.specific_recipe_url)
+        return {"question_type": question_type, "selected_recipe": selected_recipe}
+
+    async def _node_call_retriever(state: AgentState, config):
+        print("---RETRIEVE---")
+        cl_msg = config["configurable"]["cl_msg"]
+        await cl_msg.stream_token("Searching for recipes matching your criteria ... \n\n")
+        question = state["question"]
+        vector_db_chain = get_self_retriever(base_llm)
+        # Retrieval
+        documents = vector_db_chain.invoke(question, return_only_outputs=False)
+        print("WOW: ", vector_db_chain.search_kwargs)
+        return {"documents": documents, "question": question}
+
+    async def _node_grade_recipes(state: AgentState, config):
+        print("---CHECK DOCUMENT RELEVANCE TO QUESTION---")
+        cl_msg = config["configurable"]["cl_msg"]
+        question = state["question"]
+        documents = state["documents"]
+        await cl_msg.stream_token(
+            f"Evaluating the relevance of {len(documents)} retrieved recipes based on your criteria ... \n\n"
+        )
+
+        retrieval_grader = get_grader_chain(base_llm)
+
+        # Score each doc
+        filtered_docs = []
+        for d in documents:
+            grader_output = retrieval_grader.invoke({"question": question, "document": d.page_content})
+            binary_score = grader_output.binary_score
+            score = grader_output.integer_score
+
+            if binary_score == "yes":
+                print("---GRADE: DOCUMENT RELEVANT---: ", score, d.metadata["url"])
+                d.metadata["score"] = score
+                filtered_docs.append(d)
+            else:
+                print("---GRADE: DOCUMENT NOT RELEVANT---", score, d.metadata["url"])
+                continue
+        num_eliminated_docs = len(documents) - len(filtered_docs)
+        if num_eliminated_docs > 0:
+            await cl_msg.stream_token(
+                f"Eliminated {num_eliminated_docs} recipes that were not relevant based on your criteria ... \n\n"
+            )
+        return {"documents": filtered_docs, "question": question}
+
+    async def _node_generate_response(state: AgentState, config):
+        """
+        Determines whether the retrieved recipes are relevant to the question.
+
+        Args:
+            state (messages): The current state
+
+        Returns:
+            str: A decision for whether the documents are relevant or not
+        """
+
+        print("--- GENERATING SHORTLIST ---")
+
+        question = state["question"]
+        documents = state["documents"]
+
+        # LLM with tool and validation
+        base_rag_prompt_template = """\
+            You are a friendly AI assistant. Using the provided context, 
+            please answer the user's question in a friendly, conversational tone. 
+
+            Based on the context provided, please select the top 3 receipes that best fits criteria 
+            outlined in the question. It doesn't need to be a perfect match but just get the most suitable.
+            
+            For each option, provide the following information:
+            1. A brief description of the recipe
+            2. The URL of the recipe
+            3. The ratings and number of ratings
+            Only if question includes a criteria for recipes that are good for a specific occassion, please also provide the occassion(s) of the recipe,
+            Only if question includes a criteria a type of cuisine, please also provide the cuisines associated with the recipe.
+            Only if question includes a criteria a type of diet, please also provide the diet(s) associated with the recipe.
+
+            If the context is empty, please be careful to note to the user that there are no recipes matching those specific requirements and do NOT provide any other recipes as suggestions.
+
+            Context:
+            {context}
+            
+            Question:
+            {question}
+        """
+
+        base_rag_prompt = ChatPromptTemplate.from_template(base_rag_prompt_template)
+
+        chain = base_rag_prompt | power_llm
+        full_response = ""
+        cl_msg = config["configurable"]["cl_msg"]
+        async for chunk in chain.astream(
+            {"question": question, "context": documents},
+            config=RunnableConfig(callbacks=[cl.LangchainCallbackHandler()]),
+        ):
+            if isinstance(chunk, AIMessageChunk):
+                await cl_msg.stream_token(chunk.content)
+                full_response += chunk.content
+        url_extractor = get_recipe_url_extractor_chain(base_llm)
+        url_extractor_results = url_extractor.invoke({"context": full_response})
+
+        shortlisted_recipes = None
+        if isinstance(url_extractor_results.urls, list) and len(url_extractor_results.urls):
+            shortlisted_recipes = get_recipes(url_extractor_results.urls)
+        return {
+            "documents": documents,
+            "question": question,
+            "shortlisted_recipes": shortlisted_recipes,
+            "messages": [full_response],
+        }
+
+    async def _node_shortlist_qa(state: AgentState, config):
+        print("--- Q&A with SHORTLISTED RECIPES ---")
+
+        question = state["question"]
+        shortlisted_recipes = state["shortlisted_recipes"]
+        messages = state["messages"]
+        last_message = messages[-1] if messages else ""
+
+        # LLM with tool and validation
+        base_rag_prompt_template = """\
+            You are a friendly AI assistant. Using only the provided context, 
+            please answer the user's question in a friendly, conversational tone.
+            If you don't know the answer based on the context, say you don't know.
+
+            Context:
+            {context}
+
+            Last message provided to the user:
+            {last_message}
+
+            Question:
+            {question}
+        """
+
+        base_rag_prompt = ChatPromptTemplate.from_template(base_rag_prompt_template)
+        chain = base_rag_prompt | power_llm
+
+        full_response = ""
+        cl_msg = config["configurable"]["cl_msg"]
+        async for chunk in chain.astream(
+            {
+                "question": question,
+                "context": shortlisted_recipes_to_string(shortlisted_recipes),
+                "last_message": last_message,
+            },
+            config=RunnableConfig(callbacks=[cl.LangchainCallbackHandler()]),
+        ):
+            if isinstance(chunk, AIMessageChunk):
+                await cl_msg.stream_token(chunk.content)
+                full_response += chunk.content
+
+        selected_recipe = get_selected_recipe(base_llm, question, shortlisted_recipes, messages)
+
+        return {"messages": [full_response], "selected_recipe": selected_recipe}
+
+    async def _node_single_recipe_qa(state: AgentState, config):
+        print("--- Q&A with SINGLE RECIPE ---")
+
+        question = state["question"]
+        selected_recipe = state.get("selected_recipe")
+        messages = state["messages"]
+        last_message = messages[-1] if messages else ""
+
+        # LLM with tool and validation
+        base_rag_prompt_template = """\
+            You are a friendly AI assistant. Using only the provided context, 
+            please answer the user's question in a friendly, conversational tone.
+            If you don't know the answer based on the context, say you don't know.
+
+            Context:
+            {context}
+
+            Last message provided to the user:
+            {last_message}
+
+            Question:
+            {question}
+        """
+
+        base_rag_prompt = ChatPromptTemplate.from_template(base_rag_prompt_template)
+        power_llm_with_tool = power_llm.bind_functions([convert_to_openai_function(send_text_tool)])
+        chain = base_rag_prompt | power_llm_with_tool
+        full_response = ""
+        cl_msg = config["configurable"]["cl_msg"]
+
+        async for chunk in chain.astream(
+            {"question": question, "context": selected_recipe["text"], "last_message": last_message},
+            config=RunnableConfig(callbacks=[cl.LangchainCallbackHandler()]),
+        ):
+            if isinstance(chunk, AIMessageChunk):
+                await cl_msg.stream_token(chunk.content)
+                full_response += chunk.content
+
+        return {"messages": [full_response]}
+
+    async def _node_send_sms(state: AgentState, config):
+        print("--- SEND SMS ---")
+
+        question = state["question"]
+        selected_recipe = state.get("selected_recipe")
+        messages = state["messages"]
+        last_message = messages[-1] if messages else ""
+
+        # LLM with tool and validation
+        base_rag_prompt_template = """\
+            You are a friendly AI assistant.
+            Using only the provided context and the tool,
+            please fullfill the user's request to send an SMS text
+
+            Context:
+            {context}
+
+            Last message provided to the user:
+            {last_message}
+
+            Question:
+            {question}
+        """
+
+        base_rag_prompt = ChatPromptTemplate.from_template(base_rag_prompt_template)
+        # tool_functions =
+        power_llm_with_tool = power_llm.bind_functions([convert_to_openai_function(send_text_tool)])
+        chain = base_rag_prompt | power_llm_with_tool
+
+        tool_executor = ToolExecutor([send_text_tool])
+        message = chain.invoke(
+            {
+                "question": question,
+                "context": selected_recipe.get("text") if selected_recipe else "",
+                "last_message": last_message,
+            },
+        )
+
+        print("message", message)
+
+        action = ToolInvocation(
+            tool=message.additional_kwargs["function_call"]["name"],
+            tool_input=json.loads(message.additional_kwargs["function_call"]["arguments"]),
+        )
+
+        response = tool_executor.invoke(action)
+
+        function_message = FunctionMessage(content=str(response), name=action.tool)
+
+        return {"messages": [function_message]}
+
+    workflow = StateGraph(AgentState)
+
+    # Define the nodes
+    workflow.add_node("triage", _node_question_triage)  # retrieve
+    workflow.add_node("retrieve", _node_call_retriever)  # retrieve
+    workflow.add_node("grade_recipes", _node_grade_recipes)  # grade documents
+    workflow.add_node("generate", _node_generate_response)  # generatae
+    workflow.add_node("shortlist_qa", _node_shortlist_qa)  # answer questions about shortlisted recipes
+    workflow.add_node("single_qa", _node_single_recipe_qa)  # answer questions about shortlisted recipes
+    workflow.add_node("send_sms", _node_send_sms)  # answer questions about shortlisted recipes
+
+    # Define the edges
+
+    def _edge_route_question(state: AgentState):
+        print("=======EDGE: START =====")
+        question_type = state["question_type"]
+        messages = state["messages"]
+        shortlisted_recipes = state.get("shortlisted_recipes")
+        selected_recipe = state.get("selected_recipe")
+
+        # if not shortlisted_recipes or len(shortlisted_recipes) == 0:
+        #     print("going to retrieve since no shortlisted_recipes")
+        #     return "retrieve"
+        if question_type == "asking_for_recipe_suggestions":
+            return "retrieve"
+        if question_type in ["referring_to_shortlisted_recipes", "show_specific_recipe"]:
+            return "shortlist_qa"
+        if question_type == "referring_to_specific_recipe" and selected_recipe:
+            return "single_qa"
+        if question_type == "send_sms":
+            return "send_sms"
+
+        print("defaulting to shortlist_qa")
+        return "shortlist_qa"
+
+    workflow.add_edge(START, "triage")
+    workflow.add_conditional_edges(
+        "triage",
+        _edge_route_question,
+        {
+            "shortlist_qa": "shortlist_qa",
+            "single_qa": "single_qa",
+            "retrieve": "retrieve",
+            "send_sms": "send_sms",
+        },
+    )
+
+    workflow.add_edge("retrieve", "grade_recipes")
+    workflow.add_edge("grade_recipes", "generate")
+    workflow.add_edge("generate", END)
+    workflow.add_edge("shortlist_qa", END)
+    workflow.add_edge("single_qa", END)
+    workflow.add_edge("send_sms", END)
+
+    memory = AsyncSqliteSaver.from_conn_string(":memory:")
+
+    app = workflow.compile(checkpointer=memory)
+
+    return app

utils/graph_chains.py

@@ -0,0 +1,200 @@
+from langchain.output_parsers import PydanticOutputParser
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.pydantic_v1 import BaseModel, Field
+from .db import shortlisted_recipes_to_string
+
+
+def get_grader_chain(llm_model):
+    class GradeRecipes(BaseModel):
+        """Binary score for relevance check on retrieved documents."""
+
+        binary_score: str = Field(
+            description="Document representing recipes are generally relevant to the criteria in the question, 'yes' or 'no'"
+        )
+
+        integer_score: int = Field(
+            description="Degree to which Documents are relevant to the question, integers from 1 to 100"
+        )
+
+    # LLM with function call
+    structured_llm_grader = llm_model.with_structured_output(GradeRecipes)
+
+    # Prompt
+    system = """You are a grader assessing relevance of a retrieved cooking recipe document to a user question. \n 
+        It does not need to be a stringent test. The goal is to filter out completely erroneous or irrelevant retrievals. \n
+        If the document contains keyword(s) or semantic meaning related to the user question, grade it as relevant. \n
+        Give a binary score 'yes' or 'no' score to indicate whether the recipe document is relevant to the question.
+        Also give a integer score from 1 to 100 to indicate the degree to which the recipe document is relevant to the question.
+    """
+
+    grade_prompt = ChatPromptTemplate.from_messages(
+        [
+            ("system", system),
+            ("human", "Retrieved recipe document: \n\n {document} \n\n User question: {question}"),
+        ]
+    )
+
+    retrieval_grader = grade_prompt | structured_llm_grader
+
+    return retrieval_grader
+
+
+def get_recipe_url_extractor_chain(llm_model):
+    class RecipeUrlsSchema(BaseModel):
+        urls: list[str] = Field(description="A list of urls pointing to specific recipes")
+
+    structured_llm_grader = llm_model.with_structured_output(RecipeUrlsSchema)
+
+    pydantic_parser = PydanticOutputParser(pydantic_object=RecipeUrlsSchema)
+    format_instructions = pydantic_parser.get_format_instructions()
+
+    RECIPE_SEARCH_PROMPT = """
+        Your goal is to understand and parse out the full http urls in the context corresponding to each recipe.
+
+        {format_instructions}
+
+        Context:
+        {context}
+    """
+
+    prompt = ChatPromptTemplate.from_template(
+        template=RECIPE_SEARCH_PROMPT, partial_variables={"format_instructions": format_instructions}
+    )
+
+    retriever = prompt | structured_llm_grader
+
+    return retriever
+
+
+def get_recipe_selection_chain(llm_model):
+    class RecipeSelectionSchema(BaseModel):
+        asking_for_recipe_suggestions: str = Field(
+            description="Whether the User Question is asking for recipe suggestions based on some criteria, 'yes' or 'no'"
+        )
+        referring_to_specific_recipe: str = Field(
+            description="Whether the User Question is asking about one specific recipe (but NOT asking to just show a specific recipe), 'yes' or 'no'"
+        )
+        referring_to_shortlisted_recipes: str = Field(
+            description="Whether the User Question is asking generally about the 3 shortlisted recipes, 'yes' or 'no'"
+        )
+
+        show_specific_recipe: str = Field(
+            description="Whether the User Question is asking asking to show a specific recipe, 'yes' or 'no'"
+        )
+
+        specific_recipe_url: str = Field(
+            description="URL of the specific recipe that the User Question is directed to, if any "
+        )
+
+    # LLM with function call
+    structured_llm_grader = llm_model.with_structured_output(RecipeSelectionSchema)
+    pydantic_parser = PydanticOutputParser(pydantic_object=RecipeSelectionSchema)
+    format_instructions = pydantic_parser.get_format_instructions()
+
+    # Prompt
+    RECIPE_SELECTION_PROMPT = """
+        You are a helpful assistant attempting to categorize the nature of the User question
+        based on the last message sent to he user and the provided context.
+
+        {format_instructions}
+        
+        User Question:
+        {question}
+
+        Last message provided to the user:
+        {last_message}
+
+        Context:
+        {context}
+
+        
+    """
+
+    prompt = ChatPromptTemplate.from_template(
+        template=RECIPE_SELECTION_PROMPT, partial_variables={"format_instructions": format_instructions}
+    )
+
+    chain = prompt | structured_llm_grader
+
+    return chain
+
+
+def get_question_type_chain(llm_model):
+    class RecipeSelectionChanceSchema(BaseModel):
+        asking_for_recipe_suggestions: int = Field(
+            description="The likelihood / chance that the User Question is asking for recipe suggestions based on some criteria, integers from 1 to 100"
+        )
+        referring_to_specific_recipe: int = Field(
+            description="The likelihood / chance that the User Question is asking questions about one specific full recipe, integers from 1 to 100"
+        )
+        referring_to_shortlisted_recipes: int = Field(
+            description="The likelihood / chance that the User Question is asking generally about shortlisted recipes provided in the last message, integers from 1 to 100"
+        )
+
+        show_specific_recipe: int = Field(
+            description="The likelihood / chance that the User Question is asking to show a specific recipe, integers from 1 to 100"
+        )
+        send_text: int = Field(
+            description="The likelihood / chance that the User Question is to send a SMS or text, integers from 1 to 100"
+        )
+
+        specific_recipe_url: str = Field(
+            description="URL of the specific recipe that the User Question is directed to, if any "
+        )
+
+    # LLM with function call
+    structured_llm_grader = llm_model.with_structured_output(RecipeSelectionChanceSchema)
+    pydantic_parser = PydanticOutputParser(pydantic_object=RecipeSelectionChanceSchema)
+    format_instructions = pydantic_parser.get_format_instructions()
+
+    # Prompt
+    RECIPE_SELECTION_PROMPT = """
+        You are a helpful assistant attempting to categorize the nature of the User question
+        based on the last message sent to he user and the provided context. 
+        Note that if there were recipe suggesetions in the last message provided to the user, 
+        it is highly likely that the user is asking questions referring to shortlisted recipes.
+        If the last message was a full single recipe, it is generally likely that the user 
+        is asking questions referring to specific recipe.
+
+        {format_instructions}
+        
+        User Question:
+        {question}
+
+        Last message provided to the user:
+        {last_message}
+
+        Context:
+        {context}
+
+        
+    """
+
+    prompt = ChatPromptTemplate.from_template(
+        template=RECIPE_SELECTION_PROMPT, partial_variables={"format_instructions": format_instructions}
+    )
+
+    chain = prompt | structured_llm_grader
+
+    return chain
+
+
+def get_selected_recipe(llm_model, question, shortlisted_recipes, messages):
+    selected_recipe = None
+    recipe_selection_chain = get_recipe_selection_chain(llm_model)
+    recipe_selection_response = recipe_selection_chain.invoke(
+        {
+            "question": question,
+            "context": shortlisted_recipes_to_string(shortlisted_recipes),
+            "last_message": messages[-1] if messages else "",
+        }
+    )
+
+    if (
+        recipe_selection_response.referring_to_specific_recipe == "yes"
+        and recipe_selection_response.specific_recipe_url
+    ):
+        selected_recipe = next(
+            (r for r in shortlisted_recipes if r["url"] == recipe_selection_response.specific_recipe_url)
+        )
+    return selected_recipe

utils/metadata.py

@@ -0,0 +1,431 @@
+CUISINES = [
+    "Soul Food",
+    "South American",
+    "South Asian",
+    "Southern",
+    "Southwestern",
+    "Spanish",
+    "Tex Mex",
+    "Thai",
+    "Turkish",
+    "Venezuelan",
+    "Vietnamese",
+    "West African",
+    "African",
+    "American",
+    "Asian",
+    "Australian",
+    "Austrian",
+    "Belgian",
+    "Brazilian",
+    "British",
+    "Cajun",
+    "Caribbean",
+    "Central ",
+    "Chinese",
+    "Creole",
+    "Cuban",
+    "Eastern",
+    "Filipino",
+    "French",
+    "German",
+    "Greek",
+    "Indian",
+    "Indonesian",
+    "Iranian",
+    "Irish",
+    "Italian",
+    "Italian ",
+    "Jamaican",
+    "Japanese",
+    "Jewish",
+    "Korean",
+    "Latin American",
+    "Lebanese",
+    "Malaysian",
+    "Mediterranean",
+    "Mexican",
+    "Middle Eastern",
+    "Moroccan",
+    "New England",
+    "Nordic",
+    "Pakistani",
+    "Portuguese",
+    "Russian",
+]
+
+OCCASIONS = [
+    "School Lunch",
+    "Snack",
+    "Spring",
+    "St. Patrick's Day",
+    "Summer",
+    "Super Bowl",
+    "Thanksgiving",
+    "Thanksgiving Leftover",
+    "Valentine's Day",
+    "Weekday",
+    "Weeknight",
+    "Appetizer",
+    "Barbecues",
+    "Birthday",
+    "Breakfast",
+    "Brunch",
+    "Budget",
+    "Christmas",
+    "Christmas Cookie",
+    "Date Night",
+    "Dessert",
+    "Dinner",
+    "Diwali",
+    "Easter",
+    "Fall",
+    "Father's Day",
+    "For One",
+    "For Two",
+    "Fourth Of July",
+    "Halloween",
+    "Hanukkah",
+    "Kentucky Derby",
+    "Kwanzaa",
+    "Labor Day",
+    "Lunar New Year",
+    "Lunch",
+    "Main Course",
+    "Make Ahead",
+    "Mardi Gras",
+    "Memorial Day",
+    "Mother's Day",
+    "New Year's Day",
+    "Nowruz",
+    "Party",
+    "Passover",
+    "Picnic",
+    "Plan Ahead",
+    "Quick",
+    "Ramadan",
+    "Rosh Hashana",
+]
+
+DIETS = [
+    "Vegan",
+    "Vegetarian",
+    "Dairy Free",
+    "Gluten Free",
+    "Halal",
+    "Healthy",
+    "High Fiber",
+    "High Protein",
+    "Kid Friendly",
+    "Kosher",
+    "Low Calorie",
+    "Low Carb",
+    "Low Fat",
+    "Low Sodium",
+    "Low Sugar",
+    "Nut Free",
+    "Raw",
+]
+
+EQUIPMENT = [
+    "Sheet Pan"
+    "Skillet"
+    "Slow Cooker"
+    "Stovetop"
+    "Air Fryer"
+    "Barbecue"
+    "Cast Iron"
+    "Dutch Oven"
+    "Food Processor"
+    "Freezer Friendly"
+]
+
+KEY_INGREDIENTS = [
+    "Anchovy",
+    "Andouille Sausage",
+    "Apple",
+    "Apple Cider",
+    "Applesauce",
+    "Arborio Rice",
+    "Arctic Char",
+    "Artichoke",
+    "Artichoke Heart",
+    "Arugula",
+    "Asparagus",
+    "Avocado",
+    "Baby Carrot",
+    "Bacon",
+    "Balsamic Vinegar",
+    "Banana",
+    "Basil",
+    "Bay Leaves",
+    "Bay Scallop",
+    "Beans",
+    "Beef",
+    "Beef Broth",
+    "Beer",
+    "Beet",
+    "Bell Pepper",
+    "Black Bean",
+    "Black Eyed Pea",
+    "Black Olives",
+    "Black Truffle",
+    "Blueberry",
+    "Bok Choy",
+    "Brie",
+    "Brisket",
+    "Broccoli",
+    "Broccoli Rabe",
+    "Broccolini",
+    "Brown Rice",
+    "Brussels Sprout",
+    "Butter",
+    "Butter Lettuce",
+    "Butternut Squash",
+    "Cabbage",
+    "Canned Tuna",
+    "Cannellini Bean",
+    "Cantaloupe",
+    "Caper",
+    "Carrot",
+    "Cashew",
+    "Catfish",
+    "Cauliflower",
+    "Caviar",
+    "Celery",
+    "Chanterelle Mushroom",
+    "Cheddar",
+    "Cheese",
+    "Cherry",
+    "Cherry Tomato",
+    "Chia Seed",
+    "Chicken",
+    "Chicken Breast",
+    "Chicken Broth",
+    "Chicken Wing",
+    "Chickpea",
+    "Chive",
+    "Chocolate Chip",
+    "Chorizo",
+    "Cilantro",
+    "Cinnamon",
+    "Cinnamon Stick",
+    "Citrus",
+    "Clam",
+    "Coconut Cream",
+    "Coconut Flake",
+    "Coconut Milk",
+    "Cod",
+    "Coffee",
+    "Collard Green",
+    "Corn",
+    "Corn On The Cob",
+    "Cornbread",
+    "Cornish Hen",
+    "Cottage Cheese",
+    "Couscous",
+    "Crab",
+    "Cream",
+    "Cream Cheese",
+    "Creme Fraiche",
+    "Cucumber",
+    "Daikon",
+    "Dill",
+    "Duck",
+    "Duck Breast",
+    "Duck Fat",
+    "Duck Leg",
+    "Edamame",
+    "Egg",
+    "Egg White",
+    "Egg Yolk",
+    "Eggplant",
+    "Endive",
+    "Extra Virgin Olive Oil",
+    "Farro",
+    "Feta",
+    "Fettuccine",
+    "Fish",
+    "Fruit",
+    "Garlic",
+    "Ginger",
+    "Grape",
+    "Grape Tomato",
+    "Grapefruit",
+    "Greek Yogurt",
+    "Green Olives",
+    "Ground Beef",
+    "Ground Meat",
+    "Guacamole",
+    "Halibut",
+    "Ham",
+    "Haricot Vert",
+    "Harissa",
+    "Hazelnut",
+    "Hearts Of Palm",
+    "Heirloom Tomato",
+    "Herb",
+    "Honey",
+    "Hot Dog",
+    "Hummus",
+    "Iceberg Lettuce",
+    "Italian Bread",
+    "Italian Parsley",
+    "Italian Sausage",
+    "Jalapeno",
+    "Kalamata Olive",
+    "Kale",
+    "Lamb",
+    "Lamb Chop",
+    "Leek",
+    "Lemon",
+    "Lettuce",
+    "Lime",
+    "Lime Juice",
+    "Linguine",
+    "Littleneck Clam",
+    "Lobster",
+    "Mackerel",
+    "Mango",
+    "Maple Syrup",
+    "Matcha",
+    "Mayonnaise",
+    "Melon",
+    "Mesclun",
+    "Milk",
+    "Mint",
+    "Mirin",
+    "Miso",
+    "Miso Paste",
+    "Mozzarella",
+    "Mushroom",
+    "Mustard Green",
+    "Napa Cabbage",
+    "Noodles",
+    "Octopus",
+    "Okra",
+    "Olive",
+    "Olive Oil",
+    "Onion",
+    "Orange",
+    "Oyster",
+    "Oyster Mushroom",
+    "Pancetta",
+    "Parmesan",
+    "Parsley",
+    "Parsnip",
+    "Pasta",
+    "Pea",
+    "Pea Shoot",
+    "Peach",
+    "Peanut",
+    "Peanut Butter",
+    "Pear",
+    "Pecan",
+    "Pecan Pie",
+    "Pecorino",
+    "Penne",
+    "Pepper",
+    "Pine Nut",
+    "Pineapple",
+    "Pinto Bean",
+    "Pita",
+    "Pomegranate",
+    "Pork",
+    "Pork Belly",
+    "Pork Tenderloin",
+    "Portobello",
+    "Potato",
+    "Prosciutto",
+    "Pumpkin",
+    "Quinoa",
+    "Red Onion",
+    "Rice",
+    "Ricotta",
+    "Risotto",
+    "Romaine",
+    "Rosemary",
+    "Salmon",
+    "Sausage",
+    "Sausages",
+    "Scallion",
+    "Scallop",
+    "Sea Bass",
+    "Short Rib",
+    "Shrimp",
+    "Snow Pea",
+    "Soba Noodle",
+    "Soda",
+    "Sour Cream",
+    "Spaghetti",
+    "Spaghetti Squash",
+    "Spinach",
+    "Squash",
+    "Steak",
+    "Strawberry",
+    "Sun Dried Tomato",
+    "Sunchoke",
+    "Sweet Potato",
+    "Swiss Chard",
+    "Swordfish",
+    "Thyme",
+    "Tofu",
+    "Tomatillo",
+    "Tomato",
+    "Tomato Paste",
+    "Tomato Sauce",
+    "Tuna",
+    "Turkey",
+    "Veal",
+    "Vinegar",
+    "Watermelon",
+    "Wine",
+    "Yellow Onion",
+    "Yogurt",
+    "Zucchini",
+]
+
+
+DRINK_INGREDIENTS = [
+    "Campari",
+    "Candied Ginger",
+    "Aperol",
+    "Apple Brandy",
+    "Apple Juice",
+    "Basil",
+    "Beer",
+    "Carrot Juice",
+    "Club Soda",
+    "Champagne",
+    "Cider",
+    "Cognac",
+    "Cointreau",
+    "Cynar",
+    "Dry Vermouth",
+    "Gin",
+    "Ginger Ale",
+    "Ginger Beer",
+    "Grand Marnier",
+    "Grapefruit Juice",
+    "Grappa",
+    "Irish Whiskey",
+    "Lime",
+    "Lime Juice",
+    "Mezcal",
+    "Pineapple Juice",
+    "Port",
+    "Prosecco",
+    "Ruby Port",
+    "Rum",
+    "Rye",
+    "Rye Whiskey",
+    "Scotch",
+    "Sparkling Wine",
+    "Tequila",
+    "Tonic Water",
+    "Triple Sec",
+    "Vermouth",
+    "Vodka",
+    "Whiskey",
+    "White Wine",
+]

utils/retrievers.py

@@ -0,0 +1,168 @@
+import os
+from typing import List
+from langchain_core.documents import Document
+from langchain.chains.query_constructor.base import AttributeInfo
+from langchain.retrievers import EnsembleRetriever
+from langchain.retrievers.self_query.base import SelfQueryRetriever
+from langchain_community.vectorstores import (
+    MyScale,
+    MyScaleSettings,
+)
+from langchain_community.vectorstores.qdrant import Qdrant
+from langchain_core.callbacks.manager import (
+    CallbackManagerForRetrieverRun,
+)
+from langchain_openai.embeddings import OpenAIEmbeddings
+from langchain_qdrant.vectorstores import Qdrant
+
+from .metadata import CUISINES, DIETS, EQUIPMENT, KEY_INGREDIENTS, OCCASIONS
+
+OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
+#QDRANT_CLOUD_KEY = os.environ.get("QDRANT_CLOUD_KEY")
+#QDRANT_CLOUD_URL = "https://30591e3d-7092-41c4-95e1-4d3c7ef6e894.us-east4-0.gcp.cloud.qdrant.io"
+
+
+# Define embedding model
+base_embeddings_model = OpenAIEmbeddings(model="text-embedding-3-small", openai_api_key=OPENAI_API_KEY)
+
+
+def get_ensemble_retriever():
+    # Use a Qdrant VectorStore to embed and store our data
+    qdrant_descriptions = Qdrant.from_existing_collection(
+        embedding=base_embeddings_model,
+        # 3 vector indices - recipe_descriptions, recipe_nutrition, recipe_ingredients
+        collection_name="recipe_descriptions",
+        url=QDRANT_CLOUD_URL,
+        api_key=QDRANT_CLOUD_KEY,
+    )
+
+    qdrant_nutrition = Qdrant.from_existing_collection(
+        embedding=base_embeddings_model,
+        collection_name="recipe_nutrition",
+        url=QDRANT_CLOUD_URL,
+        api_key=QDRANT_CLOUD_KEY,
+    )
+
+    qdrant_ingredients = Qdrant.from_existing_collection(
+        embedding=base_embeddings_model,
+        collection_name="recipe_ingredients",
+        url=QDRANT_CLOUD_URL,
+        api_key=QDRANT_CLOUD_KEY,
+    )
+
+    # Convert retrieved documents to JSON-serializable format
+    descriptions_retriever = qdrant_descriptions.as_retriever(search_kwargs={"k": 20})
+    nutrition_retriever = qdrant_nutrition.as_retriever(search_kwargs={"k": 20})
+    ingredients_retriever = qdrant_ingredients.as_retriever(search_kwargs={"k": 20})
+
+    ensemble_retriever = EnsembleRetriever(
+        retrievers=[
+            descriptions_retriever,
+            nutrition_retriever,
+            ingredients_retriever,
+        ],
+        weights=[
+            0.5,
+            0.25,
+            0.25,
+        ],
+    )
+
+    return ensemble_retriever
+
+
+def _list_to_string(l: list) -> str:
+    return ", ".join([f"`{item}`" for item in l])
+
+
+class ModifiedSelfQueryRetriever(SelfQueryRetriever):
+    def _get_relevant_documents(self, query: str, *, run_manager: CallbackManagerForRetrieverRun) -> List[Document]:
+        """Get documents relevant for a query.
+
+        Args:
+            query: string to find relevant documents for
+
+        Returns:
+            List of relevant documents
+        """
+        structured_query = self.query_constructor.invoke(
+            {"query": query}, config={"callbacks": run_manager.get_child()}
+        )
+        # if self.verbose:
+        #     logger.info(f"Generated Query: {structured_query}")
+
+        new_query, search_kwargs = self._prepare_query(query, structured_query)
+
+        print("search_kwargs", search_kwargs)
+        self.search_kwargs = search_kwargs
+
+        docs = self._get_docs_with_query(new_query, search_kwargs)
+        return docs
+
+
+def get_self_retriever(llm_model):
+    metadata_field_info = [
+        AttributeInfo(
+            name="cuisine",
+            description="The national / ethnic cuisine categories of the recipe."
+            f"It should be one of {_list_to_string(CUISINES)}. "
+            "It only supports contain comparisons. "
+            f"Here are some examples: contain (cuisine, '{CUISINES[0]}')",
+            type="list[string]",
+        ),
+        AttributeInfo(
+            name="diet",
+            description="The diets / dietary restrictions satisfied by this recipe."
+            f"It should be one of {_list_to_string(DIETS)}. "
+            "It only supports contain comparisons. "
+            f"Here are some examples: contain (diet, '{DIETS[0]}')",
+            type="list[string]",
+        ),
+        AttributeInfo(
+            name="equipment",
+            description="The equipment required by this recipe."
+            f"It should be one of {_list_to_string(EQUIPMENT)}. "
+            "It only supports contain comparisons. "
+            f"Here are some examples: contain (equipment, '{EQUIPMENT[0]}')",
+            type="list[string]",
+        ),
+        AttributeInfo(
+            name="occasion",
+            description="The occasions, holidays, celebrations that are well suited for this recipe."
+            f"It should be one of {_list_to_string(OCCASIONS)}. "
+            "It only supports contain comparisons. "
+            f"Here are some examples: contain (occasion, '{OCCASIONS[0]}')",
+            type="list[string]",
+        ),
+        # AttributeInfo(
+        #     name="ingredients",
+        #     description="The ingredients used to make this recipe."
+        #     f"It should be one of {_list_to_string(KEY_INGREDIENTS)}"
+        #     "It only supports contain comparisons. "
+        #     f"Here are some examples: contain (ingredients, '{KEY_INGREDIENTS[0]}')",
+        #     type="list[string]",
+        # ),
+        AttributeInfo(
+            name="time",
+            description="The estimated time in minutes required to cook and prepare the recipe",
+            type="integer",
+        ),
+    ]
+
+    config = MyScaleSettings(
+        host=os.environ["MYSCALE_HOST"],
+        port=443,
+        username=os.environ["MYSCALE_USERNAME"],
+        password=os.environ["MYSCALE_PASSWORD"],
+    )
+    vectorstore = MyScale(base_embeddings_model, config)
+
+    retriever = ModifiedSelfQueryRetriever.from_llm(
+        llm_model,
+        vectorstore,
+        "Brief summary and key attributes of a recipe, including ingredients, cooking time, occasion, cuisine and diet",
+        metadata_field_info,
+        verbose=True,
+        search_kwargs={"k": 10},
+    )
+    return retriever

utils/tools.py

@@ -0,0 +1,26 @@
+import os
+from langchain.pydantic_v1 import BaseModel, Field
+from langchain.tools import tool
+from twilio.rest import Client
+
+
+class TwilioInputSchema(BaseModel):
+    text: str = Field(description="The SMS text body")
+    number: str = Field(
+        description="The full phone number to send the SMS text to including area code. Example: +12038484755"
+    )
+
+
+@tool("send-text-tool", args_schema=TwilioInputSchema, return_direct=True)
+def send_text_tool(text: str, number: str) -> int:
+    """Send an SMS text to a specific number"""
+
+    print("SENDING text ", text, number)
+    tw_client = Client(os.environ["TWILIO_ACCOUNT_SID"], os.environ["TWILIO_AUTH_TOKEN"])
+
+    message = tw_client.messages.create(
+        from_="+18332830457",
+        body=text,
+        to=number,
+    )
+    return message.sid