Update app.py

app.py

@@ -1,181 +1,392 @@
 from flask import Flask, request
+from twilio.twiml.messaging_response import MessagingResponse
+from twilio.rest import Client
 import os
-from langchain.vectorstores import Chroma
-from langchain.document_loaders import PyPDFLoader
-from langchain.embeddings import HuggingFaceEmbeddings
-from langchain.text_splitter import RecursiveCharacterTextSplitter
 import requests
-from twilio.rest import Client
+from PIL import Image
+import shutil
+
+from langchain.vectorstores.chroma import Chroma
+from langchain.prompts import ChatPromptTemplate
+from langchain_community.llms.ollama import Ollama
+from get_embedding_function import get_embedding_function
+from langchain.document_loaders.pdf import PyPDFDirectoryLoader
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain.schema.document import Document
+import tempfile
+
+# Create a temporary directory for Chroma if running in Hugging Face Spaces
+
 
 
-# Flask app
 app = Flask(__name__)
+UPLOAD_FOLDER = '/code/uploads'
+CHROMA_PATH = tempfile.mkdtemp()  # Use the same folder for Chroma
+if not os.path.exists(UPLOAD_FOLDER):
+    os.makedirs(UPLOAD_FOLDER)
+
+app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
+
+class ConversationBufferMemory:
+    def __init__(self, max_size=6):
+        self.memory = []
+        self.max_size = max_size
 
-# ChromaDB path
-CHROMA_PATH = '/code/chroma_db'
-if not os.path.exists(CHROMA_PATH):
-    os.makedirs(CHROMA_PATH)
+    def add_to_memory(self, interaction):
+        self.memory.append(interaction)
+        if len(self.memory) > self.max_size:
+            self.memory.pop(0)
 
+    def get_memory(self):
+        return self.memory
+
+conversation_memory = ConversationBufferMemory(max_size=2)
+
+account_sid = os.environ.get('TWILIO_ACCOUNT_SID')
+auth_token = os.environ.get('TWILIO_AUTH_TOKEN')
+client = Client(account_sid, auth_token)
+from_whatsapp_number = 'whatsapp:+14155238886'
+
+PROMPT_TEMPLATE = """
+Answer the question based only on the following context:
+{context}
+---
+Answer the question based on the above context: {question}
+"""
+
+from bs4 import BeautifulSoup
+import requests
+from requests.auth import HTTPBasicAuth
+from PIL import Image
+from io import BytesIO
+import pandas as pd
+from urllib.parse import urlparse
+import os
+from pypdf import PdfReader
 from ai71 import AI71
+import uuid
+
+from inference_sdk import InferenceHTTPClient
+import base64
+
+AI71_API_KEY = os.environ.get('AI71_API_KEY')
 
 def generate_response(query, chat_history):
     response = ''
-    try:
-        ai71_client = AI71(api_key=AI71_API_KEY)
-        chat_completion = ai71_client.chat.completions.create(
+    for chunk in AI71(AI71_API_KEY).chat.completions.create(
             model="tiiuae/falcon-180b-chat",
             messages=[
-                {"role": "system", "content": "You are the best agricultural assistant. Remember to give a response in not more than 2 sentences."},
-                {"role": "user", "content": f"Answer the query based on history {chat_history}: {query}"}
+                {"role": "system", "content": "You are the best agricultural assistant. Remember to give a response in not more than 2 sentences. Greet the user if the user greets you."},
+                {"role": "user", "content": f'''Answer the query based on history {chat_history}: {query}'''},
             ],
-            stream=True
-        )
-        
-        for chunk in chat_completion:
-            if chunk.choices[0].delta.content:
-                response += chunk.choices[0].delta.content
-        
-        # Clean up response text
-        response = response.replace("###", '').replace('\nUser:', '')
-        
-    except Exception as e:
-        print(f"Error generating response: {e}")
-        response = "An error occurred while generating the response."
+            stream=True,
+    ):
+        if chunk.choices[0].delta.content:
+            response += chunk.choices[0].delta.content
+    return response.replace("###", '').replace('\nUser:', '')
+
+def predict_pest(filepath):
+    CLIENT = InferenceHTTPClient(
+        api_url="https://detect.roboflow.com",
+        api_key="oF1aC4b1FBCDtK8CoKx7"
+    )
+    result = CLIENT.infer(filepath, model_id="pest-detection-ueoco/1")
+    return result['predictions'][0]
     
-    return response
 
+def predict_disease(filepath):
+    CLIENT = InferenceHTTPClient(
+        api_url="https://classify.roboflow.com",
+        api_key="oF1aC4b1FBCDtK8CoKx7"
+    )
+    result = CLIENT.infer(filepath, model_id="plant-disease-detection-iefbi/1")
+    return result['predicted_classes'][0]
 
-# Initialize ChromaDB
-def initialize_chroma():
+def convert_img(url, account_sid, auth_token):
     try:
-        embedding_function = HuggingFaceEmbeddings()
-        db = Chroma(persist_directory=CHROMA_PATH, embedding_function=embedding_function)
+        response = requests.get(url, auth=HTTPBasicAuth(account_sid, auth_token))
+        response.raise_for_status()
+
+        parsed_url = urlparse(url)
+        media_id = parsed_url.path.split('/')[-1]
+        filename = f"downloaded_media_{media_id}"
+
+        media_filepath = os.path.join(UPLOAD_FOLDER, filename)
+        with open(media_filepath, 'wb') as file:
+            file.write(response.content)
         
-        # Perform an initial operation to ensure the database is correctly initialized
-        db.similarity_search_with_score("test query", k=1)
-        print("Chroma initialized successfully.")
-    except Exception as e:
-        print(f"Error initializing Chroma: {e}")
+        print(f"Media downloaded successfully and saved as {media_filepath}")
 
-initialize_chroma()
+        with open(media_filepath, 'rb') as img_file:
+            image = Image.open(img_file)
 
-# Set AI71 API key
-AI71_API_KEY = os.environ.get('AI71_API_KEY')
-account_sid = os.environ.get('TWILIO_ACCOUNT_SID')
-auth_token = os.environ.get('TWILIO_AUTH_TOKEN')
-client = Client(account_sid, auth_token)
-from_whatsapp_number = 'whatsapp:+14155238886'
+            converted_filename = f"image.jpg"
+            converted_filepath = os.path.join(UPLOAD_FOLDER, converted_filename)
+            image.convert('RGB').save(converted_filepath, 'JPEG')
+            return converted_filepath
 
-# Download file utility
-def download_file(url, ext):
-    local_filename = f'/code/uploaded_file{ext}'
-    with requests.get(url, stream=True) as r:
-        with open(local_filename, 'wb') as f:
-            for chunk in r.iter_content(chunk_size=8192):
-                f.write(chunk)
-    return local_filename
+    except requests.exceptions.HTTPError as err:
+        print(f"HTTP error occurred: {err}")
+    except Exception as err:
+        print(f"An error occurred: {err}")
 
-# Process PDF and return text
+def get_weather(city):
+    city = city.strip().replace(' ', '+')
+    r = requests.get(f'https://www.google.com/search?q=weather+in+{city}')
+    soup = BeautifulSoup(r.text, 'html.parser')
+    temperature = soup.find('div', attrs={'class': 'BNeawe iBp4i AP7Wnd'}).text
+    return temperature
 
+from zenrows import ZenRowsClient
+Zenrow_api = os.environ.get('Zenrow_api')
+zenrows_client = ZenRowsClient(Zenrow_api)
 
-import fitz  # PyMuPDF
+def get_rates():
+    url = "https://www.kisandeals.com/mandiprices/ALL/TAMIL-NADU/ALL"
+    response = zenrows_client.get(url)
 
-def extract_text_from_pdf(pdf_filepath):
-    text = ''
-    try:
-        pdf_document = fitz.open(pdf_filepath)
-        for page_num in range(len(pdf_document)):
-            page = pdf_document.load_page(page_num)
-            text += page.get_text()
-        pdf_document.close()
-    except Exception as e:
-        print(f"Error extracting text from PDF: {e}")
-        return None
-    return text
+    if response.status_code == 200:
+        soup = BeautifulSoup(response.content, 'html.parser')
+        rows = soup.select('table tbody tr')
+        data = {}
+        for row in rows:
+            columns = row.find_all('td')
+            if len(columns) >= 2:
+                commodity = columns[0].get_text(strip=True)
+                price = columns[1].get_text(strip=True)
+                if '₹' in price:
+                    data[commodity] = price
+    return str(data) + " These are the prices for 1 kg"
 
-def query_rag(query_text: str, chat_history):
+def get_news():
+    news = []
+    url = "https://economictimes.indiatimes.com/news/economy/agriculture?from=mdr"
+    response = zenrows_client.get(url)
+
+    if response.status_code == 200:
+        soup = BeautifulSoup(response.content, 'html.parser')
+        headlines = soup.find_all("div", class_="eachStory")
+        for story in headlines:
+            headline = story.find('h3').text.strip()
+            news.append(headline)
+    return news
+
+def download_and_save_as_txt(url, account_sid, auth_token):
     try:
-        # Ensure the database is initialized
-        initialize_chroma()
+        response = requests.get(url, auth=HTTPBasicAuth(account_sid, auth_token))
+        response.raise_for_status()
 
-        embedding_function = HuggingFaceEmbeddings()
-        db = Chroma(persist_directory=CHROMA_PATH, embedding_function=embedding_function)
-        
-        # Perform similarity search
-        results = db.similarity_search_with_score(query_text, k=5)
+        parsed_url = urlparse(url)
+        media_id = parsed_url.path.split('/')[-1]
+        filename = f"pdf_file.pdf"
 
-        if not results:
-            print("No relevant information found.")
-            return "Sorry, I couldn't find any relevant information."
+        txt_filepath = os.path.join(UPLOAD_FOLDER, filename)
+        with open(txt_filepath, 'wb') as file:
+            file.write(response.content)
         
-        # Combine document contents for context
+        print(f"Media downloaded successfully and saved as {txt_filepath}")
+        return txt_filepath
+
+    except requests.exceptions.HTTPError as err:
+        print(f"HTTP error occurred: {err}")
+    except Exception as err:
+        print(f"An error occurred: {err}")
+
+
+def initialize_chroma():
+    try:
+        # Initialize Chroma
+        db = Chroma(persist_directory=CHROMA_PATH, embedding_function=get_embedding_function())
+        # Perform an initial operation to ensure it works
+        db.similarity_search_with_score("test query", k=1)
+        print("Chroma initialized successfully.")
+    except Exception as e:
+        print(f"Error initializing Chroma: {e}")
+
+initialize_chroma()
+
+def query_rag(query_text: str):
+    embedding_function = get_embedding_function()
+    db = Chroma(persist_directory=CHROMA_PATH, embedding_function=embedding_function)
+    print(query_text)
+    # Check if the query is related to a PDF
+    if "from pdf" in query_text.lower() or "in pdf" in query_text.lower():
+        # Provide some context about handling PDFs
+        response_text = "I see you're asking about a PDF-related query. Let me check the context from the PDF."
+    else:
+        # Regular RAG functionality
+        response_text = "Your query is not related to PDFs. Please make sure your question is clear."
+
+    results = db.similarity_search_with_score(query_text, k=5)
+    
+    if not results:
+        response_text = "Sorry, I couldn't find any relevant information."
+    else:
         context_text = "\n\n---\n\n".join([doc.page_content for doc, _score in results])
+        prompt_template = ChatPromptTemplate.from_template(PROMPT_TEMPLATE)
+        prompt = prompt_template.format(context=context_text, question=query_text)
         
-        # Prepare the prompt for the Falcon model
-        prompt = f"Context:\n{context_text}\n\nQuestion:\n{query_text}"
-        
-        # Generate a response using the Falcon model
-        response = generate_response(prompt, chat_history)
-        
-        return response
-    except Exception as e:
-        print(f"Error querying RAG system: {e}")
-        return "An error occurred while querying the RAG system."
+        response = ''
+        for chunk in AI71(AI71_API_KEY).chat.completions.create(
+                model="tiiuae/falcon-180b-chat",
+                messages=[
+                    {"role": "system", "content": "You are the best agricultural assistant. Remember to give a response in not more than 2 sentences."},
+                    {"role": "user", "content": f'''Answer the following query based on the given context: {prompt}'''},
+                ],
+                stream=True,
+        ):
+            if chunk.choices[0].delta.content:
+                response += chunk.choices[0].delta.content
 
+        response_text = response.replace("###", '').replace('\nUser:', '')
 
+    return response_text
+    
+def download_file(url, extension):
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+        filename = f"{uuid.uuid4()}{extension}"
+        file_path = os.path.join(UPLOAD_FOLDER, filename)
+        
+        with open(file_path, 'wb') as file:
+            file.write(response.content)
+        
+        print(f"File downloaded and saved as {file_path}")
+        return file_path
 
+    except requests.exceptions.HTTPError as err:
+        print(f"HTTP error occurred: {err}")
+    except Exception as err:
+        print(f"An error occurred: {err}")
+        return None
 def save_pdf_and_update_database(pdf_filepath):
     try:
-        text = extract_text_from_pdf(pdf_filepath)
-        if not text:
-            print("Error extracting text from PDF.")
-            return
-
+        document_loader = PyPDFDirectoryLoader(UPLOAD_FOLDER)
+        documents = document_loader.load()
+        
         text_splitter = RecursiveCharacterTextSplitter(
             chunk_size=800,
             chunk_overlap=80,
             length_function=len,
             is_separator_regex=False,
         )
-        chunks = text_splitter.split_text(text)
+        chunks = text_splitter.split_documents(documents)
+        
+        add_to_chroma(chunks)
+        print(f"PDF processed and data updated in Chroma.")
+    except Exception as e:
+        print(f"Error in processing PDF: {e}")
+
+def load_documents():
+    document_loader = PyPDFDirectoryLoader(DATA_PATH)
+    return document_loader.load()
 
-        embedding_function = HuggingFaceEmbeddings()
-        db = Chroma(persist_directory=CHROMA_PATH, embedding_function=embedding_function)
+def add_to_chroma(chunks: list[Document]):
+    try:
+        db = Chroma(persist_directory=CHROMA_PATH, embedding_function=get_embedding_function())
+        chunks_with_ids = calculate_chunk_ids(chunks)
+        existing_items = db.get(include=[])
+        existing_ids = set(existing_items["ids"])
+
+        new_chunks = [chunk for chunk in chunks_with_ids if chunk.metadata["id"] not in existing_ids]
 
-        db.add_documents(chunks)
-        db.persist()
-        print("PDF processed and data updated in Chroma.")
+        if new_chunks:
+            new_chunk_ids = [chunk.metadata["id"] for chunk in new_chunks]
+            db.add_documents(new_chunks, ids=new_chunk_ids)
+            db.persist()
+            print(f"Chunks added to Chroma.")
     except Exception as e:
-        print(f"Error processing PDF: {e}")
+        print(f"Error adding chunks to Chroma: {e}")
+def calculate_chunk_ids(chunks):
+    last_page_id = None
+    current_chunk_index = 0
+
+    for chunk in chunks:
+        source = chunk.metadata.get("source")
+        page = chunk.metadata.get("page")
+        current_page_id = f"{source}:{page}"
+
+        if current_page_id == last_page_id:
+            current_chunk_index += 1
+        else:
+            current_chunk_index = 0
+
+        last_page_id = current_page_id
+        chunk_id = f"{current_page_id}:{current_chunk_index}"
+        chunk.metadata["id"] = chunk_id
+
+    return chunks
+
 
-# Flask route to handle WhatsApp webhook
 @app.route('/whatsapp', methods=['POST'])
 def whatsapp_webhook():
     incoming_msg = request.values.get('Body', '').lower()
     sender = request.values.get('From')
     num_media = int(request.values.get('NumMedia', 0))
 
-    chat_history = []  # You need to handle chat history appropriately
+    chat_history = conversation_memory.get_memory()
 
     if num_media > 0:
         media_url = request.values.get('MediaUrl0')
         content_type = request.values.get('MediaContentType0')
 
-        if content_type == 'application/pdf':
-            filepath = download_file(media_url, ".pdf")
-            save_pdf_and_update_database(filepath)
-            response_text = "PDF has been processed. You can now ask questions related to its content."
+        if content_type.startswith('image/'):
+            # Handle image processing (disease/pest detection)
+            filepath = convert_img(media_url, account_sid, auth_token)
+            response_text = handle_image(filepath)
         else:
-            response_text = "Unsupported file type. Please upload a PDF document."
+            # Handle PDF processing
+            filepath = download_and_save_as_txt(media_url, account_sid, auth_token)
+            response_text = process_and_query_pdf(filepath)
+    elif ('weather' in incoming_msg.lower()) or ('climate' in incoming_msg.lower()) or (
+            'temperature' in incoming_msg.lower()):
+        response_text = get_weather(incoming_msg.lower())
+    elif 'bookkeeping' in incoming_msg:
+        response_text = "Please provide the details you'd like to record."
+    elif ('rates' in incoming_msg.lower()) or ('price' in incoming_msg.lower()) or (
+            'market' in incoming_msg.lower()) or ('rate' in incoming_msg.lower()) or ('prices' in incoming_msg.lower()):
+        rates = get_rates()
+        response_text = generate_response(incoming_msg + ' data is ' + rates, chat_history)
+    elif ('news' in incoming_msg.lower()) or ('information' in incoming_msg.lower()):
+        news = get_news()
+        response_text = generate_response(incoming_msg + ' data is ' + str(news), chat_history)
     else:
-        # Use RAG to generate a response based on the query
-        response_text = query_rag(incoming_msg, chat_history)
+        response_text = query_rag(incoming_msg)
 
-    # Send the response back to the sender
+    conversation_memory.add_to_memory({"user": incoming_msg, "assistant": response_text})
     send_message(sender, response_text)
     return '', 204
 
-# Function to send message
+def handle_image(filepath):
+    try:
+        disease = predict_disease(filepath)
+    except:
+        disease = None
+    try:
+        pest = predict_pest(filepath)
+    except:
+        pest = None
+
+    if disease:
+        response_text = f"Detected disease: {disease}"
+        disease_info = generate_response(f"Provide brief information about {disease} in plants", chat_history)
+        response_text += f"\n\nAdditional information: {disease_info}"
+    elif pest:
+        response_text = f"Detected pest: {pest}"
+        pest_info = generate_response(f"Provide brief information about {pest} in agriculture", chat_history)
+        response_text += f"\n\nAdditional information: {pest_info}"
+    else:
+        response_text = "Please upload another image with good quality."
+    
+    return response_text
+
+def process_and_query_pdf(filepath):
+    # Assuming the PDF processing and embedding are handled here.
+    add_to_chroma(load_documents())
+    return query_rag("from pdf")  # Replace with a more specific query if needed
+
+
 def send_message(to, body):
     try:
         message = client.messages.create(
@@ -186,12 +397,12 @@ def send_message(to, body):
         print(f"Message sent with SID: {message.sid}")
     except Exception as e:
         print(f"Error sending message: {e}")
+        
 def send_initial_message(to_number):
     send_message(
         f'whatsapp:{to_number}',
         'Welcome to the Agri AI Chatbot! How can I assist you today? You can send an image with "pest" or "disease" to classify it.'
     )
-
 if __name__ == "__main__":
     send_initial_message('919080522395')
     send_initial_message('916382792828')