Merge branch 'main' of hf.co:spaces/Ritesh-hf/Nutrigenics-chatbot

.gitattributes

@@ -36,3 +36,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 datasets/ filter=lfs diff=lfs merge=lfs -text
 datasets/sidechef/my_recipes.json filter=lfs diff=lfs merge=lfs -text
 datasets/sidechef/recipes.json filter=lfs diff=lfs merge=lfs -text
+my_recipes.json filter=lfs diff=lfs merge=lfs -text
+recipes.json filter=lfs diff=lfs merge=lfs -text

app.py

@@ -1,10 +1,7 @@
 import pandas as pd
-import json
 from PIL import Image
 import numpy as np
-
 import os
-from pathlib import Path
 
 import torch
 import torch.nn.functional as F
@@ -17,23 +14,24 @@ from transformers import StoppingCriteria, StoppingCriteriaList, TextIteratorStr
 import gradio as gr
 import spaces
 
-from langchain_core.output_parsers import StrOutputParser
-from langchain_core.prompts import ChatPromptTemplate
-from langchain_groq import ChatGroq
+from langchain.chains import ConversationChain
 from langchain_community.chat_message_histories import ChatMessageHistory
 from langchain_core.runnables import RunnableWithMessageHistory
 from langchain_core.output_parsers import StrOutputParser
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_groq import ChatGroq
 
 from dotenv import load_dotenv
 
+import json
 from openai import OpenAI
 
-
 # GROQ_API_KEY = os.getenv("GROQ_API_KEY")
 load_dotenv(".env")
 USER_AGENT = os.getenv("USER_AGENT")
 GROQ_API_KEY = os.getenv("GROQ_API_KEY")
 OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+SECRET_KEY = os.getenv("SECRET_KEY")
 
 # Set environment variables
 os.environ['USER_AGENT'] = USER_AGENT
@@ -41,7 +39,6 @@ os.environ["GROQ_API_KEY"] = GROQ_API_KEY
 os.environ['OPENAI_API_KEY'] = OPENAI_API_KEY
 os.environ["TOKENIZERS_PARALLELISM"] = 'true'
 
-
 # Initialize LLM
 llm = ChatGroq(model="llama-3.1-8b-instant", temperature=0, max_tokens=1024, max_retries=2)
 
@@ -132,7 +129,7 @@ tar_img_feats = torch.cat(tar_img_feats, dim=0)
 
 class Chat:
 
-    def __init__(self, model, transform, dataframe, tar_img_feats, device='cuda:0', stopping_criteria=None):
+    def __init__(self, model, transform, dataframe, tar_img_feats, device='cuda', stopping_criteria=None):
         self.device = device
         self.model = model
         self.transform = transform
@@ -173,17 +170,27 @@ chat = Chat(model,transform,df,tar_img_feats, device)
 print("Chat Initialized !")
 
 
+
+import secrets
+import string
+
+def generate_session_key():
+    characters = string.ascii_letters + string.digits
+    session_key = ''.join(secrets.choice(characters) for _ in range(8))
+    return session_key
+
+
 def answer_generator(formated_input, session_id):
     # QA system prompt and chain
     qa_system_prompt = """
-    You are an AI assistant developed by Nutrigenics AI, specializing in intelligent recipe information retrieval and recipe suggestions. Your purpose is to help users by recommending recipes, providing detailed nutritional values, listing ingredients, offering step-by-step cooking instructions, and filtering recipes based on provide context ans user query.
+    You are an AI assistant developed by Nutrigenics AI, specializing in intelligent recipe information retrieval and recipe suggestions. Your purpose is to help users by recommending recipes, providing detailed nutritional values, listing ingredients, offering step-by-step cooking instructions, and filtering recipes based on context and user queries.
     Operational Guidelines:
     1. Input Structure:
     - Context: You may receive contextual information related to recipes, such as specific data sets, user preferences, dietary restrictions, or previously selected dishes.
     - User Query: Users will pose questions or requests related to recipes, nutritional information, ingredient substitutions, cooking instructions, and more.
     2. Response Strategy:
     - Utilize Provided Context: If the context contains relevant information that addresses the user's query, base your response on this provided data to ensure accuracy and relevance.
-    - Respond to User Query Directly: If the context does not contain the necessary information to answer the user's query, kindly state that you do not have require information.
+    - Respond to User Query Directly: If the context does not contain the necessary information to answer the user's query, kindly state that you do not have the required information.
     Core Functionalities:
     - Nutritional Information: Accurately provide nutritional values for each recipe, including calories, macronutrients (proteins, fats, carbohydrates), and essential vitamins and minerals, using contextual data when available.
     - Ingredient Details: List all ingredients required for recipes, including substitute options for dietary restrictions or ingredient availability, utilizing context when relevant.
@@ -191,10 +198,11 @@ def answer_generator(formated_input, session_id):
     - Recipe Recommendations: Suggest dishes based on user preferences, dietary restrictions, available ingredients, and contextual data if provided.
     Additional Instructions:
     - Precision and Personalization: Always aim to provide precise, personalized, and relevant information to users based on both the provided context and their specific queries.
-    - Clarity and Coherence: Ensure that all responses are clear, well-structured, and easy to understand, facilitating a seamless user experience.
-    - Substitute Suggestions: When suggesting ingredient substitutes, consider user preferences and dietary restrictions outlined in the context or user query.
+    - Clarity and Coherence: Ensure all responses are clear, well-structured, and easy to understand, facilitating a seamless user experience.
+    - Substitute Suggestions: Consider user preferences and dietary restrictions outlined in the context or user query when suggesting ingredient substitutes.
     - Dynamic Adaptation: Adapt your responses dynamically based on whether the context is relevant to the user's current request, ensuring optimal use of available information.
-    Don't mention about context in the response, format the answer in a natural and friendly way.
+    - Don't mention about the context in the response, format the answer in a natural and friendly way.
+    
     Context:
     {context}
     """
@@ -230,8 +238,8 @@ def router_node(query):
     # Prompt
     router_instructions = """You are an expert at determining the appropriate task for a user’s question based on chat history and the current query context. You have two available tasks:
 
-        1.	Retrieval: Fetch information based on user's chat history and current query.
-        2.	Recommendation/Suggestion: Recommend recipes to users based on the query.
+        1.	Retrieval: Fetch information based on the user's chat history and current query.
+        2.	Recommendation/Suggestion: Recommend user recipes based on the query.
 
     Return a JSON response with a single key named “task” indicating either “retrieval” or “recommendation” based on your decision.
     """
@@ -256,14 +264,14 @@ def recommendation_node(query):
         "recipe_time": integer,
         "recipe_yields": string,
         "recipe_ingredients": list of ingredients,
-        "recipe_instructions": list of instruections,
+        "recipe_instructions": list of instructions,
         "recipe_image": string,
         "blogger": string,
-        "recipe_nutrients": JSON object with key value pairs such as "protein: 10g",
-        "tags": list of tags related to recipe
+        "recipe_nutrients": JSON object with key-value pairs such as "protein: 10g",
+        "tags": list of tags related to a recipe
     } \n
 
-    Here is the example of an recipe json object from the JSON data: \n
+    Here is the example of a recipe JSON object from the JSON data: \n
     {
         "recipe_name": "Asian Potato Salad with Seven Minute Egg",
         "recipe_time": 0,
@@ -313,19 +321,19 @@ def recommendation_node(query):
         ]
     } \n
 
-    Based on the user query, provide a Python function to filter the JSON data. The output of the function should be a list of json objects. \n
+    Based on the user query, provide a Python function to filter the JSON data. The output of the function should be a list of JSON objects. \n
 
     Recipe filtering instructions:
-    - If a user asked for the highest nutrient recipe such as "high protein or high calories" then filtered recipes should be the top highest recipes from all the recipes with high nutrient.
-    - sort or rearrange recipes based which recipes are more appropriate for the user.
+    - If a user asked for the highest nutrient recipe such as "high protein or high calories" then filtered recipes should be the top highest recipes from all the recipes with high nutrients.
+    - sort or rearrange recipes based on which recipes are more appropriate for the user.
 
     Your output instructions:
-    - The function name should be filter_recipes. The input to the function should be file name.
+    - The function name should be filter_recipes. The input to the function should be the file name.
     - The length of output recipes should not be more than 6.
-    - Only give me output function. Do not call the function.
-    - Give the python function as a key named "code" in a json format.
-    - Do not include any other text with the output, only give python code.
-    - If you do not follow the above given instructions, the chat may be terminated.
+    - Only give me the output function. Do not call the function.
+    - Give the Python function as a key named "code" in a JSON format.
+    - Do not include any other text with the output, only give Python code.
+    - If you do not follow the above-given instructions, the chat may be terminated.
     """
     max_tries = 3
     while True:
@@ -383,11 +391,14 @@ def answer_formatter_node(question, context):
     return res
 
 CURR_CONTEXT = ''
+CURR_SESSION_KEY = generate_session_key()
 
-# @spaces.GPU
+@spaces.GPU
 def get_answer(image=[], message='', sessionID='abc123'):
     global CURR_CONTEXT
-    if len(image) > 0:
+    global CURR_SESSION_KEY
+    sessionID = CURR_SESSION_KEY
+    if image is not None:
         try:
             # Process the image and message here
             device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
@@ -403,21 +414,21 @@ def get_answer(image=[], message='', sessionID='abc123'):
         except Exception as e:
             print(e)
             response = {'content':"An error occurred while processing your request."}
-    elif len(image) == 0 and message is not None:
-        print("I am here")
+    elif (image is None) and (message is not None):
         task = router_node(message)
-        if task == 'retrieval':
+        print(task)
+        if task == 'recommendation':
             recipes = recommendation_node(message)
             if not recipes:
-                response = {'content':"An error occurred while processing your request."}
-            response = answer_formatter_node(message, recipes)
+                response = {'content': "An error occurred while processing your request."}
+            else:
+                response = answer_formatter_node(message, recipes)
         else:
             formated_input = {
                 'input': message,
                 'context': CURR_CONTEXT
             }
             response = answer_generator(formated_input, session_id=sessionID)
-
     return response
 
 import json
@@ -432,8 +443,9 @@ session_store = {}
 def handle_message(data):
     global session_store
     global CURR_CONTEXT
+    global CURR_SESSION_KEY
+    session_id = CURR_SESSION_KEY
     context = "No data available"
-    session_id = request.sid
     if session_id not in session_store:
         session_store[session_id] = {'image_data': b"", 'message': None, 'image_received': False}
 
@@ -448,7 +460,7 @@ def handle_message(data):
 
         except Exception as e:
             print(f"Error processing image chunk: {str(e)}")
-            return
+            return "An error occurred while receiving the image chunk."
         
         if session_store[session_id]['image_data'] or session_store[session_id]['message']:
             try:
@@ -470,10 +482,11 @@ def handle_message(data):
                 }
                 # Invoke question_answer_chain and stream the response
                 response = answer_generator(formated_input, session_id=session_id)
+                return response
 
             except Exception as e:
                 print(f"Error processing image or message: {str(e)}")
-                return
+                return "An error occurred while processing your request."
             finally:
                 # Clear session data after processing
                 session_store.pop(session_id, None)
@@ -487,15 +500,15 @@ def handle_message(data):
                 'context': json.dumps(CURR_CONTEXT)
             }
             response = answer_generator(formated_input, session_id=session_id)
+            session_store.pop(session_id, None)
+            return response
         else:
             response = recommendation_node(message)
             # response = answer_formatter_node(message, recipes)
             if response is None:
                 response = {'content':"An error occurred while processing your request."}
-
-        session_store.pop(session_id, None)
-    
-
+            session_store.pop(session_id, None)
+            return response
 
 import requests
 from PIL import Image
@@ -519,17 +532,37 @@ def download_image_to_numpy(url):
         raise Exception(f"Failed to download image. Status code: {response.status_code}")
 
 def handle_message(data):
+    global CURR_SESSION_KEY
+    session_id = CURR_SESSION_KEY
     img_url = data['img_url']
     message = data['message']
     image_array = download_image_to_numpy(img_url)
-    response = get_answer(image=image_array, message=message)
+    response = get_answer(image=image_array, message=message, sessionID=session_id)
     return response
 
 
+
+# @spaces.GPU
+def respond_to_user(image, message):
+    # Process the image and message here
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    chat = Chat(model,transform,df,tar_img_feats, device)
+    chat.encode_image(image)
+    data = chat.ask()
+    formated_input = {
+        'input': message,
+        'context': data
+    }
+    try:
+        response = answer_generator(formated_input, session_id="123cnedc")
+    except Exception as e:
+        response = {'content':"An error occurred while processing your request."}
+    return response
+
 iface = gr.Interface(
-    fn=respond_to_user,
+    fn=get_answer,
     inputs=[gr.Image(), gr.Textbox(label="Ask Query")],
-    outputs=[gr.Textbox(label="Nutrition-GPT"), gr.JSON(label="context")],
+    outputs=[gr.Textbox(label="Nutrition-GPT")],
     title="Nutrition-GPT Demo",
     description="Upload an food image and ask queries!",
     css=".component-12 {background-color: red}",

my_recipes.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c06976151fba792ef9589980ba3773dec8529d7f3f9d25ce0db599aab3b7efe1
+size 20836308

recipes.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c04c9e8ba0e463d0bc09077b466085b6cbe98b1428a21f74299115b9c3c54b51
+size 23644165

requirements.txt

@@ -12,4 +12,4 @@ gradio
 langchain
 langchain-community
 langchain-groq
-openai
+openai