Merge branch 'featue/add_ttd' into dev

.gitignore

@@ -12,6 +12,8 @@ notebooks/
 data/
 sandbox/
 
+climateqa/talk_to_data/database/
 *.db
+
 data_ingestion/
-.vscode
+.vscode

app.py

@@ -12,6 +12,7 @@ from climateqa.engine.reranker import get_reranker
 from climateqa.engine.graph import make_graph_agent,make_graph_agent_poc
 from climateqa.engine.chains.retrieve_papers import find_papers
 from climateqa.chat import start_chat, chat_stream, finish_chat
+from climateqa.engine.talk_to_data.main import ask_vanna
 
 from front.tabs import (create_config_modal, create_examples_tab, create_papers_tab, create_figures_tab, create_chat_interface, create_about_tab)
 from front.utils import process_figures
@@ -150,6 +151,10 @@ def cqa_tab(tab_name):
                                     "<h2>There are no graphs to be displayed at the moment. Try asking another question.</h2>",
                                     elem_id="graphs-container"
                                 )
+                            with gr.Tab("DRIAS", elem_id="tab-vanna", id=6) as tab_vanna:
+                                vanna_table = gr.DataFrame([], elem_id="vanna-display")
+                                vanna_display = gr.Plot()
+                                
     return {
         "chatbot": chatbot,
         "textbox": textbox,
@@ -170,7 +175,9 @@ def cqa_tab(tab_name):
         "tab_figures": tab_figures,
         "tab_graphs": tab_graphs,
         "tab_papers": tab_papers,
-        "graph_container": graphs_container
+        "graph_container": graphs_container,
+        "vanna_table" : vanna_table,
+        "vanna_display": vanna_display
     }
                                 
  
@@ -200,6 +207,9 @@ def event_handling(
     tab_graphs = main_tab_components["tab_graphs"]
     tab_papers = main_tab_components["tab_papers"]
     graphs_container = main_tab_components["graph_container"]
+    vanna_table = main_tab_components["vanna_table"]
+    vanna_display = main_tab_components["vanna_display"]
+    
     
     config_open = config_components["config_open"]
     config_modal = config_components["config_modal"]
@@ -214,8 +224,8 @@ def event_handling(
     close_config_modal = config_components["close_config_modal_button"]
     
     new_sources_hmtl = gr.State([])
-    
-    print("textbox id : ", textbox.elem_id)
+    ttd_data = gr.State([])
+
      
     for button in [config_button, close_config_modal]:
         button.click(
@@ -271,6 +281,9 @@ def event_handling(
         
 
 
+    # Drias search
+    textbox.submit(ask_vanna, [textbox], [vanna_table, vanna_display])
+
 def main_ui():
     # config_open = gr.State(True)
     with gr.Blocks(title="Climate Q&A", css_paths=os.getcwd()+ "/style.css", theme=theme, elem_id="main-component") as demo: 

climateqa/chat.py

@@ -53,6 +53,13 @@ def log_interaction_to_azure(history, output_query, sources, docs, share_client,
         print(f"Error logging on Azure Blob Storage: {e}")
         error_msg = f"ClimateQ&A Error: {str(e)[:100]} - The error has been noted, try another question and if the error remains, you can contact us :)"
         raise gr.Error(error_msg)
+
+def handle_numerical_data(event):
+    if event["name"] == "retrieve_drias_data" and event["event"] == "on_chain_end":
+        numerical_data = event["data"]["output"]["drias_data"]
+        sql_query = event["data"]["output"]["drias_sql_query"]
+        return numerical_data, sql_query
+    return None, None
     
 # Main chat function
 async def chat_stream(
@@ -121,6 +128,7 @@ async def chat_stream(
     used_documents = []
     retrieved_contents = []
     answer_message_content = ""
+    vanna_data = {}
 
     # Define processing steps
     steps_display = {
@@ -142,6 +150,14 @@ async def chat_stream(
                     history, used_documents, retrieved_contents = handle_retrieved_documents(
                         event, history, used_documents, retrieved_contents
                     )
+                # Handle Vanna retrieval
+                # if event["event"] == "on_chain_end" and event["name"] in ["retrieve_documents","retrieve_local_data"] and event["data"]["output"] != None:
+                #     df_output_vanna, sql_query = handle_numerical_data(
+                #         event
+                #     )
+                #     vanna_data = {"df_output": df_output_vanna, "sql_query": sql_query}
+                 
+                    
                 if event["event"] == "on_chain_end" and event["name"] == "answer_search" :
                     docs = event["data"]["input"]["documents"]
                     docs_html = convert_to_docs_to_html(docs)                    
@@ -184,7 +200,7 @@ async def chat_stream(
                         sub_questions = [q["question"] + "-> relevant sources : " + str(q["sources"]) for q in event["data"]["output"]["questions_list"]]
                         history[-1].content += "Decompose question into sub-questions:\n\n - " + "\n - ".join(sub_questions)
 
-            yield history, docs_html, output_query, output_language, related_contents, graphs_html 
+            yield history, docs_html, output_query, output_language, related_contents, graphs_html#, vanna_data
 
     except Exception as e:
         print(f"Event {event} has failed")
@@ -195,4 +211,4 @@ async def chat_stream(
     # Call the function to log interaction
     log_interaction_to_azure(history, output_query, sources, docs, share_client, user_id)
 
-    yield history, docs_html, output_query, output_language, related_contents, graphs_html 
+    yield history, docs_html, output_query, output_language, related_contents, graphs_html#, vanna_data

climateqa/engine/chains/drias_retriever.py

@@ -0,0 +1,16 @@
+import sys
+import os
+from climateqa.engine.talk_to_data.main import ask_vanna
+
+
+def make_drias_retriever_node(llm):
+    
+    def retrieve_drias_data(state):
+        print("---- Retrieving data from DRIAS ----")
+        query = state["query"]
+        sql_query, df = ask_vanna(query)
+        state["drias_data"] = df
+        state["drias_sql_query"] = sql_query
+        return state
+    
+    return retrieve_drias_data

climateqa/engine/chains/intent_categorization.py

@@ -57,6 +57,7 @@ def make_intent_categorization_node(llm):
     categorization_chain = make_intent_categorization_chain(llm)
 
     def categorize_message(state):
+        print("Input Message : ", state["user_input"])
         print("---- Categorize_message ----")
 
         output = categorization_chain.invoke({"input": state["user_input"]})

climateqa/engine/chains/query_transformation.py

@@ -293,6 +293,8 @@ def make_query_transform_node(llm,k_final=15):
             "n_questions":n_questions,
             "handled_questions_index":[],            
         }
+        print("New questions")
+        print(new_questions)
         return new_state
     
     return transform_query

climateqa/engine/graph.py

@@ -11,7 +11,7 @@ from typing import List, Dict
 
 import operator
 from typing import Annotated
-
+import pandas as pd
 from IPython.display import display, HTML, Image
 
 from .chains.answer_chitchat import make_chitchat_node
@@ -23,6 +23,7 @@ from .chains.retrieve_documents import make_IPx_retriever_node, make_POC_retriev
 from .chains.answer_rag import make_rag_node
 from .chains.graph_retriever import make_graph_retriever_node
 from .chains.chitchat_categorization import make_chitchat_intent_categorization_node
+from .chains.drias_retriever import make_drias_retriever_node
 # from .chains.set_defaults import set_defaults
 
 class GraphState(TypedDict):
@@ -49,6 +50,8 @@ class GraphState(TypedDict):
     recommended_content : List[Document] # OWID Graphs  # TODO merge with related_contents
     search_only : bool = False
     reports : List[str] = []
+    drias_data: pd.DataFrame
+    drias_sql_query : str
 
 def dummy(state):
     return 
@@ -72,7 +75,7 @@ def route_intent(state):
 def chitchat_route_intent(state):
     intent = state["search_graphs_chitchat"]
     if intent is True:
-        return "retrieve_graphs_chitchat"
+        return END #TODO 
     elif intent is False:
         return END
     
@@ -224,6 +227,7 @@ def make_graph_agent_poc(llm, vectorstore_ipcc, vectorstore_graphs, vectorstore_
     answer_rag = make_rag_node(llm, with_docs=True)
     answer_rag_no_docs = make_rag_node(llm, with_docs=False)
     chitchat_categorize_intent = make_chitchat_intent_categorization_node(llm)
+    retrieve_drias_data = make_drias_retriever_node(llm)
 
     # Define the nodes
     # workflow.add_node("set_defaults", set_defaults)
@@ -242,6 +246,7 @@ def make_graph_agent_poc(llm, vectorstore_ipcc, vectorstore_graphs, vectorstore_
     workflow.add_node("retrieve_documents", retrieve_documents)
     workflow.add_node("answer_rag", answer_rag)
     workflow.add_node("answer_rag_no_docs", answer_rag_no_docs)
+    workflow.add_node("retrieve_drias_data", retrieve_drias_data)
 
     # Entry point
     workflow.set_entry_point("categorize_intent")
@@ -291,6 +296,10 @@ def make_graph_agent_poc(llm, vectorstore_ipcc, vectorstore_graphs, vectorstore_
     workflow.add_edge("retrieve_local_data", "answer_search")
     workflow.add_edge("retrieve_documents", "answer_search")
 
+    workflow.add_edge("transform_query", "retrieve_drias_data")
+    workflow.add_edge("retrieve_drias_data", END)
+
+
     # Compile
     app = workflow.compile()
     return app

climateqa/engine/talk_to_data/drias_metadata.json

@@ -0,0 +1,58 @@
+{
+    "Frequency of rainy days index": {
+        "description": "Frequency_of_rainy_days_index table contains the frequency index of rainy days for each latitude longitude couple for each date in the past and the future.\nThe columns include:\n- `time`: Timestamp representing the date of the data.\n- `x`: Coordinate in the Lambert II projection for the location.\n- `y`: Coordinate in the Lambert II projection for the location.\n- `IFM40D`: Frequency index of rainy days.\n- `Lon`: Geographic longitude of the location.\n- `lat`: Geographic latitude of the location.",
+        "sql_query": "CREATE TABLE Frequency_of_rainy_days_index (\n    time TIMESTAMP,\n    x FLOAT,\n    y FLOAT,\n    IFM40D FLOAT,\n    Lon FLOAT,\n    lat FLOAT\n);"
+    },
+    "Remarkable daily precipitation total (Q99)": {
+        "description": "Remarkable_daily_precipitation_total_ total table contains the daily cumulative exceptional rainfall (Q99) for each latitude longitude couple for each date in the past and the future.\nThe columns include:\n- `time`: Timestamp representing the date of the data.\n- `x`: Coordinate in the Lambert II projection for the location.\n- `y`: Coordinate in the Lambert II projection for the location.\n- `LambertParisII`: Indicates that the x, y coordinates are in the Lambert Paris II projection.\n- `RRq99`: Cumulative exceptional rainfall.\n- `lat`: Geographic latitude of the location.\n- `lon`: Geographic longitude of the location.",
+        "sql_query": "CREATE TABLE Remarkable_daily_precipitation_total_(Q99) (\n    time TIMESTAMP,\n    x FLOAT,\n    y FLOAT,\n    LambertParisII VARCHAR(255),\n    RRq99 FLOAT,\n    lat FLOAT,\n    lon FLOAT\n);"
+    },
+    "Frequency of remarkable daily precipitation": {
+        "description": "The Frequency of remarkable daily precipitation table contains the frequency of daily exceptional rainfall in the past and the future.\nThe columns include:\n- `time`: Timestamp representing the date of the data.\n- `x`: Coordinate in the Lambert II projection for the location.\n- `y`: Coordinate in the Lambert II projection for the location.\n- `LambertParisII`: Indicates that the x, y coordinates are in the Lambert Paris II projection.\n- `RRq99refD`: Frequency of exceptional rainfall.\n- `lat`: Geographic latitude of the location.\n- `lon`: Geographic longitude of the location.",
+        "sql_query": "CREATE TABLE Frequency_of_remarkable_daily_precipitation (\n    time TIMESTAMP,\n    x FLOAT,\n    y FLOAT,\n    LambertParisII VARCHAR(255),\n    RRq99refD FLOAT,\n    lat FLOAT,\n    lon FLOAT\n);"
+    },
+    "Winter precipitation total": {
+        "description": "The Winter precipitation total table contains the cumulative winter precipitation in the past and the future.\nThe columns include:\n- `time`: Timestamp representing the date of the data.\n- `x`: Coordinate in the Lambert II projection for the location.\n- `y`: Coordinate in the Lambert II projection for the location.\n- `LambertParisII`: Indicates that the x, y coordinates are in the Lambert Paris II projection.\n- `RR`: Cumulative winter precipitation.\n- `lat`: Geographic latitude of the location.\n- `lon`: Geographic longitude of the location.",
+        "sql_query": "CREATE TABLE Winter_precipitation_total (\n    time TIMESTAMP,\n    x FLOAT,\n    y FLOAT,\n    LambertParisII VARCHAR(255),\n    RR FLOAT,\n    lat FLOAT,\n    lon FLOAT\n);"
+    },
+    "Summer precipitation total": {
+        "description": "The Summer precipitation total table contains the cumulative summer precipitation in the past and the future.\nThe columns include:\n- `time`: Timestamp representing the date of the data.\n- `x`: Coordinate in the Lambert II projection for the location.\n- `y`: Coordinate in the Lambert II projection for the location.\n- `LambertParisII`: Indicates that the x, y coordinates are in the Lambert Paris II projection.\n- `RR`: Cumulative summer precipitation.\n- `lat`: Geographic latitude of the location.\n- `lon`: Geographic longitude of the location.",
+        "sql_query": "CREATE TABLE Summer_precipitation_total (\n    time TIMESTAMP,\n    x FLOAT,\n    y FLOAT,\n    LambertParisII VARCHAR(255),\n    RR FLOAT,\n    lat FLOAT,\n    lon FLOAT\n);"
+    },
+    "Annual precipitation total": {
+        "description": "The Annual precipitation total table contains information on the cumulative annual precipitation in the past and the future.\nbased on Lambert Paris II projections.\nThe variables are as follows:\n- 'y' and 'x': Lambert Paris II coordinates for the location.\n- 'time': Timestamp indicating the observation period.\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\n- 'lat' and 'lon': Latitude and longitude of the location.\n- 'RR': Cumulative annual precipitation.",
+        "sql_query": "CREATE TABLE Annual_precipitation_total (\n    y FLOAT,\n    x FLOAT,\n    time TIMESTAMP,\n    LambertParisII VARCHAR(255),\n    lat FLOAT,\n    lon FLOAT,\n    RR FLOAT\n);"
+    },
+    "Extreme precipitation intensity": {
+        "description": "The Extreme precipitation intensity table contains information on the intensity of extreme precipitation in the past and the future,\nwhich represents the maximum value of total annual precipitation.\nThe variables are as follows:\n- 'y' and 'x': Lambert Paris II coordinates for the location.\n- 'time': Timestamp indicating the observation period.\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\n- 'lat' and 'lon': Latitude and longitude of the location.\n- 'RX1d': Intensity of extreme precipitation (maximum annual total precipitation).",
+        "sql_query": "CREATE TABLE Extreme_precipitation_intensity (\n    y FLOAT,\n    x FLOAT,\n    time TIMESTAMP,\n    LambertParisII VARCHAR(255),\n    lat FLOAT,\n    lon FLOAT,\n    RX1d FLOAT\n);"
+    },
+    "Drought index": {
+        "description": "The Drought index table contains information on the drought index based on observations over the past and the future.\nThe variables are as follows:\n- 'time': Timestamp indicating the observation period.\n- 'y' and 'x': Lambert Paris II coordinates for the location.\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\n- 'lat' and 'lon': Latitude and longitude of the location.\n- 'SWI04D': Drought index based on the analysis of precipitation and temperatures.",
+        "sql_query": "CREATE TABLE Drought_index (\n    time TIMESTAMP,\n    y FLOAT,\n    x FLOAT,\n    lat FLOAT,\n    lon FLOAT,\n    LambertParisII VARCHAR(255),\n    SWI04D FLOAT\n);"
+    },
+    "Mean winter temperature": {
+        "description": "The Mean winter temperature table contains information on the average (mean) winter temperature in the past and the future.\nThe variables are as follows:\n- 'y' and 'x': Lambert Paris II coordinates for the location.\n- 'time': Timestamp indicating the observation period.\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\n- 'lat' and 'lon': Latitude and longitude of the location.\n- 'TMm': Average winter temperature.",
+        "sql_query": "CREATE TABLE Mean_winter_temperature (\n    y FLOAT,\n    x FLOAT,\n    time TIMESTAMP,\n    LambertParisII VARCHAR(255),\n    lat FLOAT,\n    lon FLOAT,\n    TMm FLOAT\n);"
+    },
+    "Mean summer temperature": {
+        "description": "The Mean summer temperature table contains information on the average summer temperature in the past and the future.\nThe variables are as follows:\n- 'y' and 'x': Lambert Paris II coordinates for the location.\n- 'time': Timestamp indicating the observation period.\n- 'LambertParisII': Indicates that the x and y coordinates are in Lambert Paris II projection.\n- 'lat' and 'lon': Latitude and longitude of the location.\n- 'TMm': Average summer temperature.",
+        "sql_query": "CREATE TABLE Mean_summer_temperature (\n    y FLOAT,\n    x FLOAT,\n    time TIMESTAMP,\n    LambertParisII VARCHAR(255),\n    lat FLOAT,\n    lon FLOAT,\n    TMm FLOAT\n);"
+    },
+    "Number of tropical nights": {
+        "description": "The Number of tropical nights table contains information on the average summer temperature in the past and the future.\nThe variables are as follows:\n- 'y' and 'x': Lambert Paris II coordinates for the location.\n- 'time': Timestamp indicating the observation period.\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\n- 'lat' and 'lon': Latitude and longitude of the location.\n- 'TMm': Average summer temperature.",
+        "sql_query": "CREATE TABLE Number_of_tropical_nights (\n    y FLOAT,\n    x FLOAT,\n    time TIMESTAMP,\n    LambertParisII VARCHAR(255),\n    lat FLOAT,\n    lon FLOAT,\n    TR FLOAT\n);"
+    },
+    "Number of days with Tx above 30C": {
+        "description": "The Number of days with Tx above 30C table contains information on the number of days when the maximum temperature in the past and the future\nis greater than or equal to 30°C.\nThe variables are as follows:\n- 'y' and 'x': Lambert Paris II coordinates for the location.\n- 'time': Timestamp indicating the observation period.\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\n- 'lat' and 'lon': Latitude and longitude of the location.\n- 'TX30D': Number of days with Tx ≥ 30°C.",
+        "sql_query": "CREATE TABLE Number_of_days_with_Tx_above_30C (\n    y FLOAT,\n    x FLOAT,\n    time TIMESTAMP,\n    LambertParisII VARCHAR(255),\n    lat FLOAT,\n    lon FLOAT,\n    TX30D FLOAT\n);"
+    },
+    "Number of days with Tx above 35C": {
+        "description": "The Number of days with Tx above 35C table contains information on the number of days when the maximum temperature in the past and the future\nis greater than or equal to 35°C.\nThe variables are as follows:\n- 'y' and 'x': Lambert Paris II coordinates for the location.\n- 'time': Timestamp indicating the observation period.\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\n- 'lat' and 'lon': Latitude and longitude of the location.\n- 'TX35D': Number of days with Tx ≥ 35°C.",
+        "sql_query": "CREATE TABLE Number_of_days_with_Tx_above_35C (\n    y FLOAT,\n    x FLOAT,\n    time TIMESTAMP,\n    LambertParisII VARCHAR(255),\n    lat FLOAT,\n    lon FLOAT,\n    TX35D FLOAT\n);"
+    },
+    "Maximum summer temperature": {
+        "description": "The Maximum summer temperature table contains information on the maximum temperature in summer in the past and the future,\nwhich is the highest temperature recorded during the summer period.\nThe variables are as follows:\n- 'y' and 'x': Lambert Paris II coordinates for the location.\n- 'time': Timestamp indicating the observation period.\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\n- 'lat' and 'lon': Latitude and longitude of the location.\n- 'TXm': Maximum temperature recorded in summer.",
+        "sql_query": "CREATE TABLE Maximum_summer_temperature (\n    y FLOAT,\n    x FLOAT,\n    time TIMESTAMP,\n    LambertParisII VARCHAR(255),\n    lat FLOAT,\n    lon FLOAT,\n    TXm FLOAT\n);"
+    }
+}

climateqa/engine/talk_to_data/how_to_use_main.ipynb

@@ -0,0 +1,150 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Import the function in main.py"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The autoreload extension is already loaded. To reload it, use:\n",
+      "  %reload_ext autoreload\n"
+     ]
+    }
+   ],
+   "source": [
+    "import sys\n",
+    "import os\n",
+    "sys.path.append(os.path.dirname(os.path.dirname(os.getcwd())))\n",
+    "\n",
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
+    "\n",
+    "from main import ask_vanna\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Create a human query"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "query = \"what is the number of days where the temperature above 35 in 2050 in Marseille\"\n",
+    "# query = \"Compare the winter and summer precipitation in 2050 in Marseille\"\n",
+    "# query = \"What is the impact of climate  in Bordeaux?\"\n",
+    "query = \"Quelle sera la température à Marseille sur les prochaines années ?\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Call the function ask vanna, it gives an output of a the sql query and the dataframe of the result (tuple)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SQL Prompt: [{'role': 'system', 'content': \"You are a SQLite expert. Please help to generate a SQL query to answer the question. Your response should ONLY be based on the given context and follow the response guidelines and format instructions. \\n===Tables \\n\\n                CREATE TABLE Mean_winter_temperature (\\n                    y FLOAT,\\n                    x FLOAT,\\n                    year INT, \\n month INT, \\n  day INT \\n,\\n                    LambertParisII VARCHAR(255),\\n                    lat FLOAT,\\n                    lon FLOAT,\\n                    TMm FLOAT, -- Température moyenne en hiver\\n                );\\n            \\n\\n\\n                CREATE TABLE Mean_summer_temperature (\\n                    y FLOAT,\\n                    x FLOAT,\\n                    year INT, \\n month INT, \\n  day INT \\n,\\n                    LambertParisII VARCHAR(255),\\n                    lat FLOAT,\\n                    lon FLOAT,\\n                    TMm FLOAT, -- Température moyenne en été\\n                );\\n            \\n\\n\\n===Additional Context \\n\\n\\n                The Number of days with Tx above 35C table contains information on the number of days when the maximum temperature in the past and the future\\n                is greater than or equal to 35°C.\\n                The variables are as follows:\\n                - 'y' and 'x': Lambert Paris II coordinates for the location.\\n                - year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n                - 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n                - 'lat' and 'lon': Latitude and longitude of the location.\\n                - 'TX35D': Number of days with Tx ≥ 35°C.\\n                \\n\\n\\n                The Number of days with Tx above 30C table contains information on the number of days when the maximum temperature in the past and the future\\n                is greater than or equal to 30°C.\\n                The variables are as follows:\\n                - 'y' and 'x': Lambert Paris II coordinates for the location.\\n                - year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n                - 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n                - 'lat' and 'lon': Latitude and longitude of the location.\\n                - 'TX30D': Number of days with Tx ≥ 30°C.\\n                \\n\\n===Response Guidelines \\n1. If the provided context is sufficient, please generate a valid SQL query without any explanations for the question. \\n2. If the provided context is almost sufficient but requires knowledge of a specific string in a particular column, please generate an intermediate SQL query to find the distinct strings in that column. Prepend the query with a comment saying intermediate_sql \\n3. If the provided context is insufficient, please explain why it can't be generated. \\n4. Please use the most relevant table(s). \\n5. If the question has been asked and answered before, please repeat the answer exactly as it was given before. \\n6. Ensure that the output SQL is SQLite-compliant and executable, and free of syntax errors. \\n\"}, {'role': 'user', 'content': 'Quelle sera la température à lat, long : (43.2961743, 5.3699525) sur les prochaines années ?'}]\n",
+      "Using model gpt-4o-mini for 828.0 tokens (approx)\n",
+      "LLM Response: ```sql\n",
+      "SELECT year, TMm \n",
+      "FROM Mean_winter_temperature \n",
+      "WHERE lat = 43.2961743 AND lon = 5.3699525\n",
+      "UNION ALL\n",
+      "SELECT year, TMm \n",
+      "FROM Mean_summer_temperature \n",
+      "WHERE lat = 43.2961743 AND lon = 5.3699525;\n",
+      "```\n",
+      "Extracted SQL: SELECT year, TMm \n",
+      "FROM Mean_winter_temperature \n",
+      "WHERE lat = 43.2961743 AND lon = 5.3699525\n",
+      "UNION ALL\n",
+      "SELECT year, TMm \n",
+      "FROM Mean_summer_temperature \n",
+      "WHERE lat = 43.2961743 AND lon = 5.3699525;\n",
+      "Using model gpt-4o-mini for 218.5 tokens (approx)\n",
+      "[(2031, 9.952474117647114), (2031, 9.952474117647114), (2032, 10.142322941176474), (2032, 10.142322941176474), (2033, 9.907942941176486), (2033, 9.907942941176486), (2034, 9.548873529411765), (2034, 9.548873529411765), (2035, 10.284758235294191), (2035, 10.284758235294191), (2036, 10.372100000000046), (2036, 10.372100000000046), (2037, 9.98571000000004), (2037, 9.98571000000004), (2038, 10.221372352941216), (2038, 10.221372352941216), (2039, 10.222609411764722), (2039, 10.222609411764722), (2040, 10.473662941176485), (2040, 10.473662941176485), (2041, 10.427640588235306), (2041, 10.427640588235306), (2042, 10.364736470588241), (2042, 10.364736470588241), (2043, 10.112910588235309), (2043, 10.112910588235309), (2044, 10.250792352941176), (2044, 10.250792352941176), (2045, 10.166119411764669), (2045, 10.166119411764669), (2046, 10.728997647058861), (2046, 10.728997647058861), (2047, 10.347248823529412), (2047, 10.347248823529412), (2048, 10.706604117647089), (2048, 10.706604117647089), (2049, 10.59243764705883), (2049, 10.59243764705883), (2050, 10.63225529411767), (2050, 10.63225529411767), (2031, 24.061035294117687), (2031, 24.061035294117687), (2032, 24.530692941176483), (2032, 24.530692941176483), (2033, 24.722234705882386), (2033, 24.722234705882386), (2034, 23.84629176470588), (2034, 23.84629176470588), (2035, 24.231422352941195), (2035, 24.231422352941195), (2036, 24.488941764705885), (2036, 24.488941764705885), (2037, 24.79424117647062), (2037, 24.79424117647062), (2038, 24.730553529411793), (2038, 24.730553529411793), (2039, 24.44979882352942), (2039, 24.44979882352942), (2040, 24.40726882352942), (2040, 24.40726882352942), (2041, 24.768547647058824), (2041, 24.768547647058824), (2042, 24.53479647058822), (2042, 24.53479647058822), (2043, 24.769181176470624), (2043, 24.769181176470624), (2044, 24.489877058823538), (2044, 24.489877058823538), (2045, 24.448076470588262), (2045, 24.448076470588262), (2046, 25.111282352941203), (2046, 25.111282352941203), (2047, 24.72313823529413), (2047, 24.72313823529413), (2048, 25.187577058823535), (2048, 25.187577058823535), (2049, 24.829653529411814), (2049, 24.829653529411814), (2050, 25.053394117647144), (2050, 25.053394117647144)]\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "('SELECT year, TMm \\nFROM Mean_winter_temperature \\nWHERE lat = 43.166954040527344 AND lon = 5.430534839630127\\nUNION ALL\\nSELECT year, TMm \\nFROM Mean_summer_temperature \\nWHERE lat = 43.166954040527344 AND lon = 5.430534839630127;',\n",
+       "     year        TMm\n",
+       " 0   2031   9.952474\n",
+       " 1   2031   9.952474\n",
+       " 2   2032  10.142323\n",
+       " 3   2032  10.142323\n",
+       " 4   2033   9.907943\n",
+       " ..   ...        ...\n",
+       " 75  2048  25.187577\n",
+       " 76  2049  24.829654\n",
+       " 77  2049  24.829654\n",
+       " 78  2050  25.053394\n",
+       " 79  2050  25.053394\n",
+       " \n",
+       " [80 rows x 2 columns])"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "ask_vanna(query)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "climateqa",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

climateqa/engine/talk_to_data/main.py

@@ -0,0 +1,90 @@
+from climateqa.engine.talk_to_data.myVanna import MyVanna
+from climateqa.engine.talk_to_data.utils import loc2coords, detect_location_with_openai, detectTable, nearestNeighbourSQL
+import sqlite3
+import os
+import pandas as pd
+from climateqa.engine.llm import get_llm
+import ast
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
+PC_API_KEY = os.getenv('VANNA_PINECONE_API_KEY')
+INDEX_NAME = os.getenv('VANNA_INDEX_NAME')
+VANNA_MODEL = os.getenv('VANNA_MODEL')
+
+
+#Vanna object
+vn = MyVanna(config = {"temperature": 0, "api_key": OPENAI_API_KEY, 'model': VANNA_MODEL, 'pc_api_key': PC_API_KEY, 'index_name': INDEX_NAME, "top_k" : 4})
+db_vanna_path = os.path.join(os.path.dirname(__file__), "database/drias.db")
+vn.connect_to_sqlite(db_vanna_path)
+
+llm = get_llm(provider="openai")
+
+# def ask_vanna(query):
+#     location = detect_location_with_openai(OPENAI_API_KEY, query)
+#     if location:
+#         coords = loc2coords(location)
+#         user_input = query.replace(location, f"lat, long : {coords}")
+#         answer = vn.ask(user_input, print_results=False, allow_llm_to_see_data=True)
+#         table = detectTable(answer[0])
+#         coords2 = nearestNeighbourSQL(db_vanna_path, coords, table[0])
+
+#         query  = answer[0].replace(f"{coords[0]}", f"{coords2[0]}")
+#         sql_query = query.replace(f"{coords[1]}", f"{coords2[1]}")
+
+#         db = sqlite3.connect(db_vanna_path)
+#         result = db.cursor().execute(sql_query).fetchall()
+#         print(result)
+#         df = pd.DataFrame(result, columns=answer[1].columns)
+        
+#     else: 
+#         answer = vn.ask(query, visualize=True, print_results=False, allow_llm_to_see_data=True)
+#         sql_query = answer[0]
+#         df = answer[1]
+        
+#     return (sql_query, df)
+def replace_coordonates(coords, sql_query, coords_tables):
+    n = sql_query.count(str(coords[0]))
+    sql_query_new_coords = sql_query
+
+    for i in range(n):
+        sql_query_new_coords = sql_query_new_coords.replace(str(coords[0]), str(coords_tables[i][0]),1)
+        sql_query_new_coords = sql_query_new_coords.replace(str(coords[1]), str(coords_tables[i][1]),1)
+    return sql_query_new_coords
+
+def ask_vanna(query):
+    location = detect_location_with_openai(OPENAI_API_KEY, query)
+    if location:
+
+        coords = loc2coords(location)
+        user_input = query.replace(location, f"lat, long : {coords}")
+        sql_query, result_dataframe, figure = vn.ask(user_input, print_results=False, allow_llm_to_see_data=True)
+        table = detectTable(sql_query)
+        coords_tables = [nearestNeighbourSQL(db_vanna_path, coords, table[i]) for i in range(len(table))]
+        sql_query_new_coords = replace_coordonates(coords, sql_query, coords_tables)
+        sql_with_table_names = llm.invoke(f"Make the following sql query display the source table in the rows {sql_query_new_coords}. Just answer the query. The answer should not include ```sql\n").content
+        print("execute sql query : ", sql_with_table_names)
+        db = sqlite3.connect(db_vanna_path)
+        result = db.cursor().execute(sql_query_new_coords).fetchall()
+        columns = llm.invoke(f"From the given sql query, list the columns that are being selected. The answer should only be a python list. Just answer the list. The SQL query : {sql_query_new_coords}").content
+        columns_list = ast.literal_eval(columns.strip("```python\n").strip())
+        print("column list : ",columns_list)
+        df = pd.DataFrame(result, columns=columns_list)
+        
+        plotly_code = vn.generate_plotly_code(
+                            question="query",
+                            sql="sql_with_table_names",
+                            df_metadata=f"Running df.dtypes gives:\n {df.dtypes}",
+                        )
+
+        fig = vn.get_plotly_figure(plotly_code=plotly_code, df=df)
+
+        return df, fig
+    else : 
+        empty_df = pd.DataFrame()
+        empty_fig = {}
+        return empty_df, empty_fig

climateqa/engine/talk_to_data/myVanna.py

@@ -0,0 +1,13 @@
+from dotenv import load_dotenv
+from climateqa.engine.talk_to_data.vanna_class import MyCustomVectorDB
+from vanna.openai import OpenAI_Chat
+import os
+
+load_dotenv()
+
+OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
+
+class MyVanna(MyCustomVectorDB, OpenAI_Chat):
+    def __init__(self, config=None):
+        MyCustomVectorDB.__init__(self, config=config)
+        OpenAI_Chat.__init__(self, config=config)

climateqa/engine/talk_to_data/pinecone_vanna_training.ipynb

@@ -0,0 +1,592 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The autoreload extension is already loaded. To reload it, use:\n",
+      "  %reload_ext autoreload\n",
+      "{'temperature': 0.2, 'api_key': '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', 'model': 'gpt-4o-mini', 'pc_api_key': 'pcsk_5pEfJ8_GqSCikBaVhK3V6wehh4YHCegspQeshyWVesKeqmqzcmfLgkQRpWaUVJvSyTcdG', 'index_name': 'cqa-vanna'}\n",
+      "Loading embeddings model:  BAAI/bge-base-en-v1.5\n"
+     ]
+    }
+   ],
+   "source": [
+    "from vanna_class import MyCustomVectorDB\n",
+    "from vanna.openai import OpenAI_Chat\n",
+    "import os\n",
+    "from dotenv import load_dotenv\n",
+    "\n",
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
+    "\n",
+    "load_dotenv()\n",
+    "\n",
+    "OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')\n",
+    "OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')\n",
+    "PC_API_KEY = os.getenv('VANNA_PINECONE_API_KEY')\n",
+    "INDEX_NAME = os.getenv('VANNA_INDEX_NAME')\n",
+    "VANNA_MODEL = os.getenv('VANNA_MODEL')\n",
+    "\n",
+    "INDEX_NAME = \"cqa-vanna\"\n",
+    "PC_API_KEY = \"pcsk_5pEfJ8_GqSCikBaVhK3V6wehh4YHCegspQeshyWVesKeqmqzcmfLgkQRpWaUVJvSyTcdG\"\n",
+    "\n",
+    "class MyVanna(MyCustomVectorDB, OpenAI_Chat):\n",
+    "    def __init__(self, config=None):\n",
+    "        print(config)\n",
+    "        MyCustomVectorDB.__init__(self, config=config)\n",
+    "        OpenAI_Chat.__init__(self, config=config)\n",
+    "\n",
+    "vn = MyVanna(\n",
+    "    config={\n",
+    "        'temperature': 0.2,\n",
+    "        'api_key': OPENAI_API_KEY,\n",
+    "        'model': 'gpt-4o-mini',\n",
+    "        'pc_api_key': PC_API_KEY,\n",
+    "        'index_name': INDEX_NAME\n",
+    "    }\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 49,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "\n",
+    "with open('drias_metadata.json', 'r') as file:\n",
+    "    tables_info = json.load(file)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 50,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def convert(tables_info):\n",
+    "    text = \"\"\"- year: Year of the observation.\\n\n",
+    "              - month : Month of the observation.\\n\n",
+    "              - day: Day of the observation.\\n\"\"\"\n",
+    "    for table_name in tables_info:\n",
+    "        tables_info[table_name]['description'] = tables_info[table_name]['description'].replace(\"- 'time': Timestamp indicating the observation period.\", text)\n",
+    "        tables_info[table_name]['description'] = tables_info[table_name]['description'].replace(\"- `time`: Timestamp representing the date of the data.\", text)\n",
+    "        tables_info[table_name]['sql_query'] = tables_info[table_name]['sql_query'].replace(\"time TIMESTAMP\", \"year INT, \\n month INT, \\n  day INT \\n\")\n",
+    "        \n",
+    "    return tables_info"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 51,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'Frequency of rainy days index': {'description': 'The Frequency of rainy days index table contains the frequency index of rainy days for each latitude longitude couple for each date in the past and the future.\\nThe columns include:\\n- year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n- `x`: Coordinate in the Lambert II projection for the location.\\n- `y`: Coordinate in the Lambert II projection for the location.\\n- `IFM40D`: Frequency index of rainy days.\\n- `Lon`: Geographic longitude of the location.\\n- `lat`: Geographic latitude of the location.',\n",
+       "  'sql_query': 'CREATE TABLE Frequency_of_rainy_days_index (\\n    year INT, \\n month INT, \\n  day INT \\n,\\n    x FLOAT,\\n    y FLOAT,\\n    IFM40D FLOAT,\\n    Lon FLOAT,\\n    lat FLOAT\\n);'},\n",
+       " 'Remarkable daily precipitation total (Q99)': {'description': 'The Remarkable daily precipitation total table contains the daily cumulative exceptional rainfall (Q99) for each latitude longitude couple for each date in the past and the future.\\nThe columns include:\\n- year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n- `x`: Coordinate in the Lambert II projection for the location.\\n- `y`: Coordinate in the Lambert II projection for the location.\\n- `LambertParisII`: Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n- `RRq99`: Cumulative exceptional rainfall.\\n- `lat`: Geographic latitude of the location.\\n- `lon`: Geographic longitude of the location.',\n",
+       "  'sql_query': 'CREATE TABLE Remarkable_daily_precipitation_total_(Q99) (\\n    year INT, \\n month INT, \\n  day INT \\n,\\n    x FLOAT,\\n    y FLOAT,\\n    LambertParisII VARCHAR(255),\\n    RRq99 FLOAT,\\n    lat FLOAT,\\n    lon FLOAT\\n);'},\n",
+       " 'Frequency of remarkable daily precipitation': {'description': 'The Frequency of remarkable daily precipitation table contains the frequency of daily exceptional rainfall in the past and the future.\\nThe columns include:\\n- year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n- `x`: Coordinate in the Lambert II projection for the location.\\n- `y`: Coordinate in the Lambert II projection for the location.\\n- `LambertParisII`: Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n- `RRq99refD`: Frequency of exceptional rainfall.\\n- `lat`: Geographic latitude of the location.\\n- `lon`: Geographic longitude of the location.',\n",
+       "  'sql_query': 'CREATE TABLE Frequency_of_remarkable_daily_precipitation (\\n    year INT, \\n month INT, \\n  day INT \\n,\\n    x FLOAT,\\n    y FLOAT,\\n    LambertParisII VARCHAR(255),\\n    RRq99refD FLOAT,\\n    lat FLOAT,\\n    lon FLOAT\\n);'},\n",
+       " 'Winter precipitation total': {'description': 'The Winter precipitation total table contains the cumulative winter precipitation in the past and the future.\\nThe columns include:\\n- year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n- `x`: Coordinate in the Lambert II projection for the location.\\n- `y`: Coordinate in the Lambert II projection for the location.\\n- `LambertParisII`: Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n- `RR`: Cumulative winter precipitation.\\n- `lat`: Geographic latitude of the location.\\n- `lon`: Geographic longitude of the location.',\n",
+       "  'sql_query': 'CREATE TABLE Winter_precipitation_total (\\n    year INT, \\n month INT, \\n  day INT \\n,\\n    x FLOAT,\\n    y FLOAT,\\n    LambertParisII VARCHAR(255),\\n    RR FLOAT,\\n    lat FLOAT,\\n    lon FLOAT\\n);'},\n",
+       " 'Summer precipitation total': {'description': 'The Summer precipitation total table contains the cumulative summer precipitation in the past and the future.\\nThe columns include:\\n- year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n- `x`: Coordinate in the Lambert II projection for the location.\\n- `y`: Coordinate in the Lambert II projection for the location.\\n- `LambertParisII`: Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n- `RR`: Cumulative summer precipitation.\\n- `lat`: Geographic latitude of the location.\\n- `lon`: Geographic longitude of the location.',\n",
+       "  'sql_query': 'CREATE TABLE Summer_precipitation_total (\\n    year INT, \\n month INT, \\n  day INT \\n,\\n    x FLOAT,\\n    y FLOAT,\\n    LambertParisII VARCHAR(255),\\n    RR FLOAT,\\n    lat FLOAT,\\n    lon FLOAT\\n);'},\n",
+       " 'Annual precipitation total': {'description': \"The Annual precipitation total table contains information on the cumulative annual precipitation in the past and the future.\\nbased on Lambert Paris II projections.\\nThe variables are as follows:\\n- 'y' and 'x': Lambert Paris II coordinates for the location.\\n- year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n- 'lat' and 'lon': Latitude and longitude of the location.\\n- 'RR': Cumulative annual precipitation.\",\n",
+       "  'sql_query': 'CREATE TABLE Annual_precipitation_total (\\n    y FLOAT,\\n    x FLOAT,\\n    year INT, \\n month INT, \\n  day INT \\n,\\n    LambertParisII VARCHAR(255),\\n    lat FLOAT,\\n    lon FLOAT,\\n    RR FLOAT\\n);'},\n",
+       " 'Extreme precipitation intensity': {'description': \"The Extreme precipitation intensity table contains information on the intensity of extreme precipitation in the past and the future,\\nwhich represents the maximum value of total annual precipitation.\\nThe variables are as follows:\\n- 'y' and 'x': Lambert Paris II coordinates for the location.\\n- year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n- 'lat' and 'lon': Latitude and longitude of the location.\\n- 'RX1d': Intensity of extreme precipitation (maximum annual total precipitation).\",\n",
+       "  'sql_query': 'CREATE TABLE Extreme_precipitation_intensity (\\n    y FLOAT,\\n    x FLOAT,\\n    year INT, \\n month INT, \\n  day INT \\n,\\n    LambertParisII VARCHAR(255),\\n    lat FLOAT,\\n    lon FLOAT,\\n    RX1d FLOAT\\n);'},\n",
+       " 'Drought index': {'description': \"The Drought index table contains information on the drought index based on observations over the past and the future.\\nThe variables are as follows:\\n- year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n- 'y' and 'x': Lambert Paris II coordinates for the location.\\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n- 'lat' and 'lon': Latitude and longitude of the location.\\n- 'SWI04D': Drought index based on the analysis of precipitation and temperatures.\",\n",
+       "  'sql_query': 'CREATE TABLE Drought_index (\\n    year INT, \\n month INT, \\n  day INT \\n,\\n    y FLOAT,\\n    x FLOAT,\\n    lat FLOAT,\\n    lon FLOAT,\\n    LambertParisII VARCHAR(255),\\n    SWI04D FLOAT\\n);'},\n",
+       " 'Mean winter temperature': {'description': \"The Mean winter temperature table contains information on the average (mean) winter temperature in the past and the future.\\nThe variables are as follows:\\n- 'y' and 'x': Lambert Paris II coordinates for the location.\\n- year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n- 'lat' and 'lon': Latitude and longitude of the location.\\n- 'TMm': Average winter temperature.\",\n",
+       "  'sql_query': 'CREATE TABLE Mean_winter_temperature (\\n    y FLOAT,\\n    x FLOAT,\\n    year INT, \\n month INT, \\n  day INT \\n,\\n    LambertParisII VARCHAR(255),\\n    lat FLOAT,\\n    lon FLOAT,\\n    TMm FLOAT\\n);'},\n",
+       " 'Mean summer temperature': {'description': \"The Mean summer temperature table contains information on the average summer temperature in the past and the future.\\nThe variables are as follows:\\n- 'y' and 'x': Lambert Paris II coordinates for the location.\\n- year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n- 'LambertParisII': Indicates that the x and y coordinates are in Lambert Paris II projection.\\n- 'lat' and 'lon': Latitude and longitude of the location.\\n- 'TMm': Average summer temperature.\",\n",
+       "  'sql_query': 'CREATE TABLE Mean_summer_temperature (\\n    y FLOAT,\\n    x FLOAT,\\n    year INT, \\n month INT, \\n  day INT \\n,\\n    LambertParisII VARCHAR(255),\\n    lat FLOAT,\\n    lon FLOAT,\\n    TMm FLOAT\\n);'},\n",
+       " 'Number of tropical nights': {'description': \"The Number of tropical nights table contains information on the average summer temperature in the past and the future.\\nThe variables are as follows:\\n- 'y' and 'x': Lambert Paris II coordinates for the location.\\n- year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n- 'lat' and 'lon': Latitude and longitude of the location.\\n- 'TMm': Average summer temperature.\",\n",
+       "  'sql_query': 'CREATE TABLE Number_of_tropical_nights (\\n    y FLOAT,\\n    x FLOAT,\\n    year INT, \\n month INT, \\n  day INT \\n,\\n    LambertParisII VARCHAR(255),\\n    lat FLOAT,\\n    lon FLOAT,\\n    TR FLOAT\\n);'},\n",
+       " 'Number of days with Tx above 30C': {'description': \"The Number of days with Tx above 30C table contains information on the number of days when the maximum temperature in the past and the future\\nis greater than or equal to 30°C.\\nThe variables are as follows:\\n- 'y' and 'x': Lambert Paris II coordinates for the location.\\n- year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n- 'lat' and 'lon': Latitude and longitude of the location.\\n- 'TX30D': Number of days with Tx ≥ 30°C.\",\n",
+       "  'sql_query': 'CREATE TABLE Number_of_days_with_Tx_above_30C (\\n    y FLOAT,\\n    x FLOAT,\\n    year INT, \\n month INT, \\n  day INT \\n,\\n    LambertParisII VARCHAR(255),\\n    lat FLOAT,\\n    lon FLOAT,\\n    TX30D FLOAT\\n);'},\n",
+       " 'Number of days with Tx above 35C': {'description': \"The Number of days with Tx above 35C table contains information on the number of days when the maximum temperature in the past and the future\\nis greater than or equal to 35°C.\\nThe variables are as follows:\\n- 'y' and 'x': Lambert Paris II coordinates for the location.\\n- year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n- 'lat' and 'lon': Latitude and longitude of the location.\\n- 'TX35D': Number of days with Tx ≥ 35°C.\",\n",
+       "  'sql_query': 'CREATE TABLE Number_of_days_with_Tx_above_35C (\\n    y FLOAT,\\n    x FLOAT,\\n    year INT, \\n month INT, \\n  day INT \\n,\\n    LambertParisII VARCHAR(255),\\n    lat FLOAT,\\n    lon FLOAT,\\n    TX35D FLOAT\\n);'},\n",
+       " 'Maximum summer temperature': {'description': \"The Maximum summer temperature table contains information on the maximum temperature in summer in the past and the future,\\nwhich is the highest temperature recorded during the summer period.\\nThe variables are as follows:\\n- 'y' and 'x': Lambert Paris II coordinates for the location.\\n- year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n- 'lat' and 'lon': Latitude and longitude of the location.\\n- 'TXm': Maximum temperature recorded in summer.\",\n",
+       "  'sql_query': 'CREATE TABLE Maximum_summer_temperature (\\n    y FLOAT,\\n    x FLOAT,\\n    year INT, \\n month INT, \\n  day INT \\n,\\n    LambertParisII VARCHAR(255),\\n    lat FLOAT,\\n    lon FLOAT,\\n    TXm FLOAT\\n);'}}"
+      ]
+     },
+     "execution_count": 51,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "new_tables_info = convert(tables_info)\n",
+    "new_tables_info\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 52,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Adding ddl: CREATE TABLE Frequency_of_rainy_days_index (\n",
+      "    year INT, \n",
+      " month INT, \n",
+      "  day INT \n",
+      ",\n",
+      "    x FLOAT,\n",
+      "    y FLOAT,\n",
+      "    IFM40D FLOAT,\n",
+      "    Lon FLOAT,\n",
+      "    lat FLOAT\n",
+      ");\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Adding documentation....\n",
+      "Adding ddl: CREATE TABLE Remarkable_daily_precipitation_total_(Q99) (\n",
+      "    year INT, \n",
+      " month INT, \n",
+      "  day INT \n",
+      ",\n",
+      "    x FLOAT,\n",
+      "    y FLOAT,\n",
+      "    LambertParisII VARCHAR(255),\n",
+      "    RRq99 FLOAT,\n",
+      "    lat FLOAT,\n",
+      "    lon FLOAT\n",
+      ");\n",
+      "Adding documentation....\n",
+      "Adding ddl: CREATE TABLE Frequency_of_remarkable_daily_precipitation (\n",
+      "    year INT, \n",
+      " month INT, \n",
+      "  day INT \n",
+      ",\n",
+      "    x FLOAT,\n",
+      "    y FLOAT,\n",
+      "    LambertParisII VARCHAR(255),\n",
+      "    RRq99refD FLOAT,\n",
+      "    lat FLOAT,\n",
+      "    lon FLOAT\n",
+      ");\n",
+      "Adding documentation....\n",
+      "Adding ddl: CREATE TABLE Winter_precipitation_total (\n",
+      "    year INT, \n",
+      " month INT, \n",
+      "  day INT \n",
+      ",\n",
+      "    x FLOAT,\n",
+      "    y FLOAT,\n",
+      "    LambertParisII VARCHAR(255),\n",
+      "    RR FLOAT,\n",
+      "    lat FLOAT,\n",
+      "    lon FLOAT\n",
+      ");\n",
+      "Adding documentation....\n",
+      "Adding ddl: CREATE TABLE Summer_precipitation_total (\n",
+      "    year INT, \n",
+      " month INT, \n",
+      "  day INT \n",
+      ",\n",
+      "    x FLOAT,\n",
+      "    y FLOAT,\n",
+      "    LambertParisII VARCHAR(255),\n",
+      "    RR FLOAT,\n",
+      "    lat FLOAT,\n",
+      "    lon FLOAT\n",
+      ");\n",
+      "Adding documentation....\n",
+      "Adding ddl: CREATE TABLE Annual_precipitation_total (\n",
+      "    y FLOAT,\n",
+      "    x FLOAT,\n",
+      "    year INT, \n",
+      " month INT, \n",
+      "  day INT \n",
+      ",\n",
+      "    LambertParisII VARCHAR(255),\n",
+      "    lat FLOAT,\n",
+      "    lon FLOAT,\n",
+      "    RR FLOAT\n",
+      ");\n",
+      "Adding documentation....\n",
+      "Adding ddl: CREATE TABLE Extreme_precipitation_intensity (\n",
+      "    y FLOAT,\n",
+      "    x FLOAT,\n",
+      "    year INT, \n",
+      " month INT, \n",
+      "  day INT \n",
+      ",\n",
+      "    LambertParisII VARCHAR(255),\n",
+      "    lat FLOAT,\n",
+      "    lon FLOAT,\n",
+      "    RX1d FLOAT\n",
+      ");\n",
+      "Adding documentation....\n",
+      "Adding ddl: CREATE TABLE Drought_index (\n",
+      "    year INT, \n",
+      " month INT, \n",
+      "  day INT \n",
+      ",\n",
+      "    y FLOAT,\n",
+      "    x FLOAT,\n",
+      "    lat FLOAT,\n",
+      "    lon FLOAT,\n",
+      "    LambertParisII VARCHAR(255),\n",
+      "    SWI04D FLOAT\n",
+      ");\n",
+      "Adding documentation....\n",
+      "Adding ddl: CREATE TABLE Mean_winter_temperature (\n",
+      "    y FLOAT,\n",
+      "    x FLOAT,\n",
+      "    year INT, \n",
+      " month INT, \n",
+      "  day INT \n",
+      ",\n",
+      "    LambertParisII VARCHAR(255),\n",
+      "    lat FLOAT,\n",
+      "    lon FLOAT,\n",
+      "    TMm FLOAT\n",
+      ");\n",
+      "Adding documentation....\n",
+      "Adding ddl: CREATE TABLE Mean_summer_temperature (\n",
+      "    y FLOAT,\n",
+      "    x FLOAT,\n",
+      "    year INT, \n",
+      " month INT, \n",
+      "  day INT \n",
+      ",\n",
+      "    LambertParisII VARCHAR(255),\n",
+      "    lat FLOAT,\n",
+      "    lon FLOAT,\n",
+      "    TMm FLOAT\n",
+      ");\n",
+      "Adding documentation....\n",
+      "Adding ddl: CREATE TABLE Number_of_tropical_nights (\n",
+      "    y FLOAT,\n",
+      "    x FLOAT,\n",
+      "    year INT, \n",
+      " month INT, \n",
+      "  day INT \n",
+      ",\n",
+      "    LambertParisII VARCHAR(255),\n",
+      "    lat FLOAT,\n",
+      "    lon FLOAT,\n",
+      "    TR FLOAT\n",
+      ");\n",
+      "Adding documentation....\n",
+      "Adding ddl: CREATE TABLE Number_of_days_with_Tx_above_30C (\n",
+      "    y FLOAT,\n",
+      "    x FLOAT,\n",
+      "    year INT, \n",
+      " month INT, \n",
+      "  day INT \n",
+      ",\n",
+      "    LambertParisII VARCHAR(255),\n",
+      "    lat FLOAT,\n",
+      "    lon FLOAT,\n",
+      "    TX30D FLOAT\n",
+      ");\n",
+      "Adding documentation....\n",
+      "Adding ddl: CREATE TABLE Number_of_days_with_Tx_above_35C (\n",
+      "    y FLOAT,\n",
+      "    x FLOAT,\n",
+      "    year INT, \n",
+      " month INT, \n",
+      "  day INT \n",
+      ",\n",
+      "    LambertParisII VARCHAR(255),\n",
+      "    lat FLOAT,\n",
+      "    lon FLOAT,\n",
+      "    TX35D FLOAT\n",
+      ");\n",
+      "Adding documentation....\n",
+      "Adding ddl: CREATE TABLE Maximum_summer_temperature (\n",
+      "    y FLOAT,\n",
+      "    x FLOAT,\n",
+      "    year INT, \n",
+      " month INT, \n",
+      "  day INT \n",
+      ",\n",
+      "    LambertParisII VARCHAR(255),\n",
+      "    lat FLOAT,\n",
+      "    lon FLOAT,\n",
+      "    TXm FLOAT\n",
+      ");\n",
+      "Adding documentation....\n"
+     ]
+    }
+   ],
+   "source": [
+    "for table in new_tables_info:\n",
+    "    vn.train(ddl = new_tables_info[table]['sql_query'])\n",
+    "    vn.train(documentation = new_tables_info[table]['description'])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Requests"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# examples"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 56,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Loading embeddings model:  BAAI/bge-base-en-v1.5\n"
+     ]
+    }
+   ],
+   "source": [
+    "question_sql = [\n",
+    "    # ['How will the precipitations change in the coming years', \n",
+    "    #  'SELECT * FROM Annual_precipitation_total WHERE year > 2024;'],\n",
+    "    # ['What is the number of days where the temperature above 35 in year 2050 in (lat, lon) = (48.82337188720703, 2.390951633453369)', \n",
+    "    #  'SELECT TX35D FROM Number_of_days_with_Tx_above_35C WHERE year = 2050 AND lat = 48.82337188720703 AND lon = 2.390951633453369;'],\n",
+    "    # ['How will change the mean winter temperature in the coming years in lat = 43.2961743 lon = 5.3699525', \n",
+    "    #  'SELECT * FROM Mean_winter_temperature WHERE year > 2023 AND lat = 43.2961743 AND lon = 5.3699525'],\n",
+    "    # ['How will change the mean summer temperature in the coming years in lat = 43.2961743, lon = 5.3699525', \n",
+    "    #  'SELECT * FROM Mean_summer_temperature WHERE year > 2023 AND lat = 43.2961743 AND lon = 5.3699525'],\n",
+    "    # ['How will change the temperature in the coming years', \n",
+    "    #  'SELECT * FROM Mean_summer_temperature JOIN Mean_winter_temperature WHERE year > 2024']\n",
+    "    [\"Quelle sera la température à lat, long : (43.2961743, 5.3699525) sur les prochaines années ?\",\n",
+    "     'SELECT Mean_winter_temperature AS table_name, lat, lon, year, TMm \\nFROM Mean_winter_temperature \\nWHERE lat = 43.2961743 AND lon = 5.3699525\\nUNION ALL\\nSELECT \"Mean_summer_temperature\" AS table_name, lat, lon, year, TMm \\nFROM Mean_summer_temperature \\nWHERE lat = 43.2961743 AND lon = 5.3699525;']\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 58,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "de6738cb3a67eaec29a04119ad160b27acd64f2d87cf83782c5b2bdc84be445b_sql\n"
+     ]
+    }
+   ],
+   "source": [
+    "for question in question_sql:\n",
+    "    print(vn.train(question = question[0], sql = question[1]))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Delete"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'cqa-vanna'"
+      ]
+     },
+     "execution_count": 31,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "INDEX_NAME"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": []
+     },
+     "execution_count": 36,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from pinecone.grpc import PineconeGRPC as Pinecone\n",
+    "\n",
+    "pc = Pinecone(api_key=PC_API_KEY)\n",
+    "\n",
+    "# To get the unique host for an index, \n",
+    "# see https://docs.pinecone.io/guides/data/target-an-index\n",
+    "index = pc.Index(INDEX_NAME)\n",
+    "# index\n",
+    "index.delete(delete_all=True, namespace='ddl')\n",
+    "index.delete(delete_all=True, namespace='documentation')\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# OLD"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import vanna\n",
+    "from vanna.remote import VannaDefault"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# PC_API_KEY = os.getenv('VANNA_PINECONE_API_KEY')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[\n",
+       "    {\n",
+       "        \"name\": \"cqa-vanna\",\n",
+       "        \"dimension\": 768,\n",
+       "        \"metric\": \"cosine\",\n",
+       "        \"host\": \"cqa-vanna-9xlylwt.svc.aped-4627-b74a.pinecone.io\",\n",
+       "        \"spec\": {\n",
+       "            \"serverless\": {\n",
+       "                \"cloud\": \"aws\",\n",
+       "                \"region\": \"us-east-1\"\n",
+       "            }\n",
+       "        },\n",
+       "        \"status\": {\n",
+       "            \"ready\": true,\n",
+       "            \"state\": \"Ready\"\n",
+       "        },\n",
+       "        \"deletion_protection\": \"disabled\"\n",
+       "    },\n",
+       "    {\n",
+       "        \"name\": \"unepqa\",\n",
+       "        \"dimension\": 768,\n",
+       "        \"metric\": \"cosine\",\n",
+       "        \"host\": \"unepqa-9xlylwt.svc.aped-4627-b74a.pinecone.io\",\n",
+       "        \"spec\": {\n",
+       "            \"serverless\": {\n",
+       "                \"cloud\": \"aws\",\n",
+       "                \"region\": \"us-east-1\"\n",
+       "            }\n",
+       "        },\n",
+       "        \"status\": {\n",
+       "            \"ready\": true,\n",
+       "            \"state\": \"Ready\"\n",
+       "        },\n",
+       "        \"deletion_protection\": \"disabled\"\n",
+       "    }\n",
+       "]"
+      ]
+     },
+     "execution_count": 34,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from pinecone.grpc import PineconeGRPC as Pinecone\n",
+    "\n",
+    "pc = Pinecone(api_key=PC_API_KEY)\n",
+    "pc.list_indexes()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'dimension': 768,\n",
+       " 'index_fullness': 0.0,\n",
+       " 'namespaces': {'ddl': {'vector_count': 14},\n",
+       "                'documentation': {'vector_count': 14}},\n",
+       " 'total_vector_count': 28}"
+      ]
+     },
+     "execution_count": 35,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "pc.Index(\"cqa-vanna\").describe_index_stats()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "climateqa",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

climateqa/engine/talk_to_data/utils.py

@@ -0,0 +1,64 @@
+import re
+import openai
+import pandas as pd
+from geopy.geocoders import Nominatim
+import sqlite3
+
+
+def detect_location_with_openai(api_key, sentence):
+    """
+    Detects locations in a sentence using OpenAI's API.
+    """
+    openai.api_key = api_key
+
+    prompt = f"""
+    Extract all locations (cities, countries, states, or geographical areas) mentioned in the following sentence.
+    Return the result as a Python list. If no locations are mentioned, return an empty list.
+    
+    Sentence: "{sentence}"
+    """
+
+    response = openai.chat.completions.create(
+        model="gpt-4o-mini",  
+        messages=[
+            {"role": "system", "content": "You are a helpful assistant skilled in identifying locations in text."},
+            {"role": "user", "content": prompt}
+        ],
+        max_tokens=100,
+        temperature=0
+    )
+
+    return response.choices[0].message.content.split("\n")[1][2:-2]
+
+
+def detectTable(sql_query):
+    pattern = r'(?i)\bFROM\s+((?:`[^`]+`|"[^"]+"|\'[^\']+\'|\w+)(?:\.(?:`[^`]+`|"[^"]+"|\'[^\']+\'|\w+))*)'
+    matches = re.findall(pattern, sql_query)
+    return matches
+
+
+
+def loc2coords(location : str):
+    geolocator = Nominatim(user_agent="city_to_latlong")
+    location = geolocator.geocode(location)
+    return (location.latitude, location.longitude)
+
+
+def coords2loc(coords : tuple):
+    geolocator = Nominatim(user_agent="coords_to_city")
+    try:
+        location = geolocator.reverse(coords)
+        return location.address
+    except Exception as e:
+        print(f"Error: {e}")
+        return "Unknown Location"  
+
+
+def nearestNeighbourSQL(db: str, location: tuple, table : str):
+    conn = sqlite3.connect(db)
+    long = round(location[1], 3)
+    lat = round(location[0], 3)
+    cursor  = conn.cursor()
+    cursor.execute(f"SELECT lat, lon FROM {table} WHERE lat BETWEEN {lat - 0.3} AND {lat + 0.3} AND lon BETWEEN {long - 0.3} AND {long + 0.3}")
+    results = cursor.fetchall()
+    return results[0]

climateqa/engine/talk_to_data/vanna_class.py

@@ -0,0 +1,323 @@
+from vanna.base import VannaBase
+from pinecone import Pinecone
+from climateqa.engine.embeddings import get_embeddings_function
+import pandas as pd
+import hashlib
+
+class MyCustomVectorDB(VannaBase):
+    
+    """
+    VectorDB class for storing and retrieving vectors from Pinecone.
+    
+    args : 
+        config (dict) : Configuration dictionary containing the Pinecone API key and the index name :
+            - pc_api_key (str) : Pinecone API key
+            - index_name (str) : Pinecone index name
+            - top_k (int) : Number of top results to return (default = 2)
+            
+    """
+    
+    def __init__(self,config):
+        super().__init__(config = config)
+        try : 
+            self.api_key = config.get('pc_api_key')
+            self.index_name = config.get('index_name')
+        except : 
+            raise Exception("Please provide the Pinecone API key and the index name")
+        
+        self.pc = Pinecone(api_key = self.api_key)
+        self.index = self.pc.Index(self.index_name)
+        self.top_k = config.get('top_k', 2)
+        self.embeddings = get_embeddings_function()
+        
+        
+    def check_embedding(self, id, namespace):
+        fetched = self.index.fetch(ids = [id], namespace = namespace)
+        if fetched['vectors'] == {}: 
+            return False
+        return True
+    
+    def generate_hash_id(self, data: str) -> str:
+        """
+        Generate a unique hash ID for the given data.
+        
+        Args:
+            data (str): The input data to hash (e.g., a concatenated string of user attributes).
+        
+        Returns:
+            str: A unique hash ID as a hexadecimal string.
+        """
+        
+        data_bytes = data.encode('utf-8')
+        hash_object = hashlib.sha256(data_bytes)
+        hash_id = hash_object.hexdigest()
+        
+        return hash_id
+    
+    def add_ddl(self, ddl: str, **kwargs) -> str:
+        id = self.generate_hash_id(ddl) + '_ddl'
+        
+        if self.check_embedding(id, 'ddl'):
+            print(f"DDL having id {id} already exists")
+            return id
+        
+        self.index.upsert(
+            vectors = [(id, self.embeddings.embed_query(ddl), {'ddl': ddl})],
+            namespace = 'ddl'
+        )
+        
+        return id
+
+    def add_documentation(self, doc: str, **kwargs) -> str:
+        id = self.generate_hash_id(doc) + '_doc'
+        
+        if self.check_embedding(id, 'documentation'):
+            print(f"Documentation having id {id} already exists")
+            return id
+        
+        self.index.upsert(
+            vectors = [(id, self.embeddings.embed_query(doc), {'doc': doc})],
+            namespace = 'documentation'
+        )
+        
+        return id
+
+    def add_question_sql(self, question: str, sql: str, **kwargs) -> str:
+        id = self.generate_hash_id(question) + '_sql'
+        
+        if self.check_embedding(id, 'question_sql'):
+            print(f"Question-SQL pair having id {id} already exists")
+            return id
+        
+        self.index.upsert(
+            vectors = [(id, self.embeddings.embed_query(question + sql), {'question': question, 'sql': sql})],
+            namespace = 'question_sql'
+        )
+        
+        return id
+
+    def get_related_ddl(self, question: str, **kwargs) -> list:
+        res = self.index.query(
+            vector=self.embeddings.embed_query(question),
+            top_k=self.top_k,
+            namespace='ddl',
+            include_metadata=True
+        )
+        
+        return [match['metadata']['ddl'] for match in res['matches']]
+
+    def get_related_documentation(self, question: str, **kwargs) -> list:
+        res = self.index.query(
+            vector=self.embeddings.embed_query(question),
+            top_k=self.top_k,
+            namespace='documentation',
+            include_metadata=True
+        )
+        
+        return [match['metadata']['doc'] for match in res['matches']]
+
+    def get_similar_question_sql(self, question: str, **kwargs) -> list:
+        res = self.index.query(
+            vector=self.embeddings.embed_query(question),
+            top_k=self.top_k,
+            namespace='question_sql',
+            include_metadata=True
+        )
+        
+        return [(match['metadata']['question'], match['metadata']['sql']) for match in res['matches']]
+
+    def get_training_data(self, **kwargs) -> pd.DataFrame:
+        
+        list_of_data = []
+        
+        namespaces = ['ddl', 'documentation', 'question_sql']
+        
+        for namespace in namespaces:
+            
+            data = self.index.query(
+            top_k=10000,
+            namespace=namespace,
+            include_metadata=True,
+            include_values=False
+            )
+            
+            for match in data['matches']:
+                list_of_data.append(match['metadata'])
+                
+        return pd.DataFrame(list_of_data)
+            
+
+
+    def remove_training_data(self, id: str, **kwargs) -> bool:
+        if id.endswith("_ddl"):
+            self.Index.delete(ids=[id], namespace="_ddl")
+            return True
+        if id.endswith("_sql"):
+            self.index.delete(ids=[id], namespace="_sql")
+            return True
+        
+        if id.endswith("_doc"):
+            self.Index.delete(ids=[id], namespace="_doc")
+            return True
+        
+        return False
+    
+    def generate_embedding(self, text, **kwargs):
+        # Implement the method here
+        pass
+
+
+    def get_sql_prompt(
+            self,
+            initial_prompt : str,
+            question: str,
+            question_sql_list: list,
+            ddl_list: list,
+            doc_list: list,
+            **kwargs,
+        ):
+            """
+            Example:
+            ```python
+            vn.get_sql_prompt(
+                question="What are the top 10 customers by sales?",
+                question_sql_list=[{"question": "What are the top 10 customers by sales?", "sql": "SELECT * FROM customers ORDER BY sales DESC LIMIT 10"}],
+                ddl_list=["CREATE TABLE customers (id INT, name TEXT, sales DECIMAL)"],
+                doc_list=["The customers table contains information about customers and their sales."],
+            )
+
+            ```
+
+            This method is used to generate a prompt for the LLM to generate SQL.
+
+            Args:
+                question (str): The question to generate SQL for.
+                question_sql_list (list): A list of questions and their corresponding SQL statements.
+                ddl_list (list): A list of DDL statements.
+                doc_list (list): A list of documentation.
+
+            Returns:
+                any: The prompt for the LLM to generate SQL.
+            """
+
+            if initial_prompt is None:
+                initial_prompt = f"You are a {self.dialect} expert. " + \
+                "Please help to generate a SQL query to answer the question. Your response should ONLY be based on the given context and follow the response guidelines and format instructions. "
+
+            initial_prompt = self.add_ddl_to_prompt(
+                initial_prompt, ddl_list, max_tokens=self.max_tokens
+            )
+
+            if self.static_documentation != "":
+                doc_list.append(self.static_documentation)
+
+            initial_prompt = self.add_documentation_to_prompt(
+                initial_prompt, doc_list, max_tokens=self.max_tokens
+            )
+
+            # initial_prompt = self.add_sql_to_prompt(
+            #     initial_prompt, question_sql_list, max_tokens=self.max_tokens
+            # )
+
+
+            initial_prompt += (
+                "===Response Guidelines \n"
+                "1. If the provided context is sufficient, please generate a valid SQL query without any explanations for the question. \n"
+                "2. If the provided context is almost sufficient but requires knowledge of a specific string in a particular column, please generate an intermediate SQL query to find the distinct strings in that column. Prepend the query with a comment saying intermediate_sql \n"
+                "3. If the provided context is insufficient, please give a sql query based on your knowledge and the context provided. \n"
+                "4. Please use the most relevant table(s). \n"
+                "5. If the question has been asked and answered before, please repeat the answer exactly as it was given before. \n"
+                f"6. Ensure that the output SQL is {self.dialect}-compliant and executable, and free of syntax errors. \n"
+                f"7. Add a description of the table in the result of the sql query, and latitude, logitude if relevant. \n"
+                # "7. Add a description of the table in the result of the sql query."
+                # "7. If the question is about a specific latitude, longitude, query an interval of 0.3 and keep only the first set of coordinate. \n"
+                # "7. Table names should be included in the result of the sql query. Use for example Mean_winter_temperature AS table_name in the query \n"
+            )
+
+
+            message_log = [self.system_message(initial_prompt)]
+
+            for example in question_sql_list:
+                if example is None:
+                    print("example is None")
+                else:
+                    if example is not None and "question" in example and "sql" in example:
+                        message_log.append(self.user_message(example["question"]))
+                        message_log.append(self.assistant_message(example["sql"]))
+
+            message_log.append(self.user_message(question))
+
+            return message_log
+        
+
+# def get_sql_prompt(
+#         self,
+#         initial_prompt : str,
+#         question: str,
+#         question_sql_list: list,
+#         ddl_list: list,
+#         doc_list: list,
+#         **kwargs,
+#     ):
+#         """
+#         Example:
+#         ```python
+#         vn.get_sql_prompt(
+#             question="What are the top 10 customers by sales?",
+#             question_sql_list=[{"question": "What are the top 10 customers by sales?", "sql": "SELECT * FROM customers ORDER BY sales DESC LIMIT 10"}],
+#             ddl_list=["CREATE TABLE customers (id INT, name TEXT, sales DECIMAL)"],
+#             doc_list=["The customers table contains information about customers and their sales."],
+#         )
+
+#         ```
+
+#         This method is used to generate a prompt for the LLM to generate SQL.
+
+#         Args:
+#             question (str): The question to generate SQL for.
+#             question_sql_list (list): A list of questions and their corresponding SQL statements.
+#             ddl_list (list): A list of DDL statements.
+#             doc_list (list): A list of documentation.
+
+#         Returns:
+#             any: The prompt for the LLM to generate SQL.
+#         """
+
+#         if initial_prompt is None:
+#             initial_prompt = f"You are a {self.dialect} expert. " + \
+#             "Please help to generate a SQL query to answer the question. Your response should ONLY be based on the given context and follow the response guidelines and format instructions. "
+
+#         initial_prompt = self.add_ddl_to_prompt(
+#             initial_prompt, ddl_list, max_tokens=self.max_tokens
+#         )
+
+#         if self.static_documentation != "":
+#             doc_list.append(self.static_documentation)
+
+#         initial_prompt = self.add_documentation_to_prompt(
+#             initial_prompt, doc_list, max_tokens=self.max_tokens
+#         )
+
+#         initial_prompt += (
+#             "===Response Guidelines \n"
+#             "1. If the provided context is sufficient, please generate a valid SQL query without any explanations for the question. \n"
+#             "2. If the provided context is almost sufficient but requires knowledge of a specific string in a particular column, please generate an intermediate SQL query to find the distinct strings in that column. Prepend the query with a comment saying intermediate_sql \n"
+#             "3. If the provided context is insufficient, please explain why it can't be generated. \n"
+#             "4. Please use the most relevant table(s). \n"
+#             "5. If the question has been asked and answered before, please repeat the answer exactly as it was given before. \n"
+#             f"6. Ensure that the output SQL is {self.dialect}-compliant and executable, and free of syntax errors. \n"
+#         )
+
+#         message_log = [self.system_message(initial_prompt)]
+
+#         for example in question_sql_list:
+#             if example is None:
+#                 print("example is None")
+#             else:
+#                 if example is not None and "question" in example and "sql" in example:
+#                     message_log.append(self.user_message(example["question"]))
+#                     message_log.append(self.assistant_message(example["sql"]))
+
+#         message_log.append(self.user_message(question))
+
+#         return message_log

requirements.txt

@@ -19,3 +19,5 @@ langchain-community==0.2
 msal==1.31
 matplotlib==3.9.2
 gradio-modal==0.0.4
+vanna==0.7.5
+geopy==2.4.1

style.css

@@ -606,3 +606,8 @@ a {
 #checkbox-config:checked {
     display: block;
 }
+
+#vanna-display {
+    height: 400px;
+    overflow-y: auto;
+}