wip

.gitignore

@@ -13,4 +13,6 @@ data/
 sandbox/
 
 climateqa/talk_to_data/database/
-*.db
+*.db
+
+data_ingestion/

app.py

@@ -12,6 +12,7 @@ from climateqa.engine.reranker import get_reranker
 from climateqa.engine.graph import make_graph_agent
 from climateqa.engine.chains.retrieve_papers import find_papers
 from climateqa.chat import start_chat, chat_stream, finish_chat
+from climateqa.engine.talk_to_data.main import ask_vanna
 
 from front.tabs import (create_config_modal, create_examples_tab, create_papers_tab, create_figures_tab, create_chat_interface, create_about_tab)
 from front.utils import process_figures
@@ -141,7 +142,8 @@ def cqa_tab(tab_name):
                                     elem_id="graphs-container"
                                 )
                             with gr.Tab("Vanna", elem_id="tab-vanna", id=6) as tab_vanna:
-                                vanna_display = gr.DataFrame([], elem_id="vanna-display")
+                                vanna_table = gr.DataFrame([], elem_id="vanna-display")
+                                vanna_display = gr.Plot()
                                 
     return {
         "chatbot": chatbot,
@@ -164,6 +166,7 @@ def cqa_tab(tab_name):
         "tab_graphs": tab_graphs,
         "tab_papers": tab_papers,
         "graph_container": graphs_container,
+        "vanna_table" : vanna_table,
         "vanna_display": vanna_display
     }
                                 
@@ -194,8 +197,10 @@ def event_handling(
     tab_graphs = main_tab_components["tab_graphs"]
     tab_papers = main_tab_components["tab_papers"]
     graphs_container = main_tab_components["graph_container"]
+    vanna_table = main_tab_components["vanna_table"]
     vanna_display = main_tab_components["vanna_display"]
     
+    
     config_open = config_components["config_open"]
     config_modal = config_components["config_modal"]
     dropdown_sources = config_components["dropdown_sources"]
@@ -244,7 +249,8 @@ def event_handling(
     for component in [textbox, examples_hidden]:
         component.submit(find_papers, [component, after, dropdown_external_sources], [papers_html, citations_network, papers_summary])
 
-    ttd_data.change(lambda x: x["df_output"], inputs=[ttd_data], outputs=[vanna_display])
+    # ttd_data.change(lambda x: x["df_output"], inputs=[ttd_data], outputs=[vanna_display])
+    textbox.submit(ask_vanna, [textbox], [vanna_table, vanna_display])
 
 def main_ui():
     # config_open = gr.State(True)

climateqa/engine/chains/drias_retriever.py

@@ -1,6 +1,6 @@
 import sys
 import os
-from climateqa.talk_to_data.main import ask_vanna
+from climateqa.engine.talk_to_data.main import ask_vanna
 
 
 def make_drias_retriever_node(llm):

climateqa/engine/talk_to_data/drias_metadata.json

@@ -0,0 +1,58 @@
+{
+    "Frequency of rainy days index": {
+        "description": "Frequency_of_rainy_days_index table contains the frequency index of rainy days for each latitude longitude couple for each date in the past and the future.\nThe columns include:\n- `time`: Timestamp representing the date of the data.\n- `x`: Coordinate in the Lambert II projection for the location.\n- `y`: Coordinate in the Lambert II projection for the location.\n- `IFM40D`: Frequency index of rainy days.\n- `Lon`: Geographic longitude of the location.\n- `lat`: Geographic latitude of the location.",
+        "sql_query": "CREATE TABLE Frequency_of_rainy_days_index (\n    time TIMESTAMP,\n    x FLOAT,\n    y FLOAT,\n    IFM40D FLOAT,\n    Lon FLOAT,\n    lat FLOAT\n);"
+    },
+    "Remarkable daily precipitation total (Q99)": {
+        "description": "Remarkable_daily_precipitation_total_ total table contains the daily cumulative exceptional rainfall (Q99) for each latitude longitude couple for each date in the past and the future.\nThe columns include:\n- `time`: Timestamp representing the date of the data.\n- `x`: Coordinate in the Lambert II projection for the location.\n- `y`: Coordinate in the Lambert II projection for the location.\n- `LambertParisII`: Indicates that the x, y coordinates are in the Lambert Paris II projection.\n- `RRq99`: Cumulative exceptional rainfall.\n- `lat`: Geographic latitude of the location.\n- `lon`: Geographic longitude of the location.",
+        "sql_query": "CREATE TABLE Remarkable_daily_precipitation_total_(Q99) (\n    time TIMESTAMP,\n    x FLOAT,\n    y FLOAT,\n    LambertParisII VARCHAR(255),\n    RRq99 FLOAT,\n    lat FLOAT,\n    lon FLOAT\n);"
+    },
+    "Frequency of remarkable daily precipitation": {
+        "description": "The Frequency of remarkable daily precipitation table contains the frequency of daily exceptional rainfall in the past and the future.\nThe columns include:\n- `time`: Timestamp representing the date of the data.\n- `x`: Coordinate in the Lambert II projection for the location.\n- `y`: Coordinate in the Lambert II projection for the location.\n- `LambertParisII`: Indicates that the x, y coordinates are in the Lambert Paris II projection.\n- `RRq99refD`: Frequency of exceptional rainfall.\n- `lat`: Geographic latitude of the location.\n- `lon`: Geographic longitude of the location.",
+        "sql_query": "CREATE TABLE Frequency_of_remarkable_daily_precipitation (\n    time TIMESTAMP,\n    x FLOAT,\n    y FLOAT,\n    LambertParisII VARCHAR(255),\n    RRq99refD FLOAT,\n    lat FLOAT,\n    lon FLOAT\n);"
+    },
+    "Winter precipitation total": {
+        "description": "The Winter precipitation total table contains the cumulative winter precipitation in the past and the future.\nThe columns include:\n- `time`: Timestamp representing the date of the data.\n- `x`: Coordinate in the Lambert II projection for the location.\n- `y`: Coordinate in the Lambert II projection for the location.\n- `LambertParisII`: Indicates that the x, y coordinates are in the Lambert Paris II projection.\n- `RR`: Cumulative winter precipitation.\n- `lat`: Geographic latitude of the location.\n- `lon`: Geographic longitude of the location.",
+        "sql_query": "CREATE TABLE Winter_precipitation_total (\n    time TIMESTAMP,\n    x FLOAT,\n    y FLOAT,\n    LambertParisII VARCHAR(255),\n    RR FLOAT,\n    lat FLOAT,\n    lon FLOAT\n);"
+    },
+    "Summer precipitation total": {
+        "description": "The Summer precipitation total table contains the cumulative summer precipitation in the past and the future.\nThe columns include:\n- `time`: Timestamp representing the date of the data.\n- `x`: Coordinate in the Lambert II projection for the location.\n- `y`: Coordinate in the Lambert II projection for the location.\n- `LambertParisII`: Indicates that the x, y coordinates are in the Lambert Paris II projection.\n- `RR`: Cumulative summer precipitation.\n- `lat`: Geographic latitude of the location.\n- `lon`: Geographic longitude of the location.",
+        "sql_query": "CREATE TABLE Summer_precipitation_total (\n    time TIMESTAMP,\n    x FLOAT,\n    y FLOAT,\n    LambertParisII VARCHAR(255),\n    RR FLOAT,\n    lat FLOAT,\n    lon FLOAT\n);"
+    },
+    "Annual precipitation total": {
+        "description": "The Annual precipitation total table contains information on the cumulative annual precipitation in the past and the future.\nbased on Lambert Paris II projections.\nThe variables are as follows:\n- 'y' and 'x': Lambert Paris II coordinates for the location.\n- 'time': Timestamp indicating the observation period.\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\n- 'lat' and 'lon': Latitude and longitude of the location.\n- 'RR': Cumulative annual precipitation.",
+        "sql_query": "CREATE TABLE Annual_precipitation_total (\n    y FLOAT,\n    x FLOAT,\n    time TIMESTAMP,\n    LambertParisII VARCHAR(255),\n    lat FLOAT,\n    lon FLOAT,\n    RR FLOAT\n);"
+    },
+    "Extreme precipitation intensity": {
+        "description": "The Extreme precipitation intensity table contains information on the intensity of extreme precipitation in the past and the future,\nwhich represents the maximum value of total annual precipitation.\nThe variables are as follows:\n- 'y' and 'x': Lambert Paris II coordinates for the location.\n- 'time': Timestamp indicating the observation period.\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\n- 'lat' and 'lon': Latitude and longitude of the location.\n- 'RX1d': Intensity of extreme precipitation (maximum annual total precipitation).",
+        "sql_query": "CREATE TABLE Extreme_precipitation_intensity (\n    y FLOAT,\n    x FLOAT,\n    time TIMESTAMP,\n    LambertParisII VARCHAR(255),\n    lat FLOAT,\n    lon FLOAT,\n    RX1d FLOAT\n);"
+    },
+    "Drought index": {
+        "description": "The Drought index table contains information on the drought index based on observations over the past and the future.\nThe variables are as follows:\n- 'time': Timestamp indicating the observation period.\n- 'y' and 'x': Lambert Paris II coordinates for the location.\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\n- 'lat' and 'lon': Latitude and longitude of the location.\n- 'SWI04D': Drought index based on the analysis of precipitation and temperatures.",
+        "sql_query": "CREATE TABLE Drought_index (\n    time TIMESTAMP,\n    y FLOAT,\n    x FLOAT,\n    lat FLOAT,\n    lon FLOAT,\n    LambertParisII VARCHAR(255),\n    SWI04D FLOAT\n);"
+    },
+    "Mean winter temperature": {
+        "description": "The Mean winter temperature table contains information on the average (mean) winter temperature in the past and the future.\nThe variables are as follows:\n- 'y' and 'x': Lambert Paris II coordinates for the location.\n- 'time': Timestamp indicating the observation period.\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\n- 'lat' and 'lon': Latitude and longitude of the location.\n- 'TMm': Average winter temperature.",
+        "sql_query": "CREATE TABLE Mean_winter_temperature (\n    y FLOAT,\n    x FLOAT,\n    time TIMESTAMP,\n    LambertParisII VARCHAR(255),\n    lat FLOAT,\n    lon FLOAT,\n    TMm FLOAT\n);"
+    },
+    "Mean summer temperature": {
+        "description": "The Mean summer temperature table contains information on the average summer temperature in the past and the future.\nThe variables are as follows:\n- 'y' and 'x': Lambert Paris II coordinates for the location.\n- 'time': Timestamp indicating the observation period.\n- 'LambertParisII': Indicates that the x and y coordinates are in Lambert Paris II projection.\n- 'lat' and 'lon': Latitude and longitude of the location.\n- 'TMm': Average summer temperature.",
+        "sql_query": "CREATE TABLE Mean_summer_temperature (\n    y FLOAT,\n    x FLOAT,\n    time TIMESTAMP,\n    LambertParisII VARCHAR(255),\n    lat FLOAT,\n    lon FLOAT,\n    TMm FLOAT\n);"
+    },
+    "Number of tropical nights": {
+        "description": "The Number of tropical nights table contains information on the average summer temperature in the past and the future.\nThe variables are as follows:\n- 'y' and 'x': Lambert Paris II coordinates for the location.\n- 'time': Timestamp indicating the observation period.\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\n- 'lat' and 'lon': Latitude and longitude of the location.\n- 'TMm': Average summer temperature.",
+        "sql_query": "CREATE TABLE Number_of_tropical_nights (\n    y FLOAT,\n    x FLOAT,\n    time TIMESTAMP,\n    LambertParisII VARCHAR(255),\n    lat FLOAT,\n    lon FLOAT,\n    TR FLOAT\n);"
+    },
+    "Number of days with Tx above 30C": {
+        "description": "The Number of days with Tx above 30C table contains information on the number of days when the maximum temperature in the past and the future\nis greater than or equal to 30°C.\nThe variables are as follows:\n- 'y' and 'x': Lambert Paris II coordinates for the location.\n- 'time': Timestamp indicating the observation period.\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\n- 'lat' and 'lon': Latitude and longitude of the location.\n- 'TX30D': Number of days with Tx ≥ 30°C.",
+        "sql_query": "CREATE TABLE Number_of_days_with_Tx_above_30C (\n    y FLOAT,\n    x FLOAT,\n    time TIMESTAMP,\n    LambertParisII VARCHAR(255),\n    lat FLOAT,\n    lon FLOAT,\n    TX30D FLOAT\n);"
+    },
+    "Number of days with Tx above 35C": {
+        "description": "The Number of days with Tx above 35C table contains information on the number of days when the maximum temperature in the past and the future\nis greater than or equal to 35°C.\nThe variables are as follows:\n- 'y' and 'x': Lambert Paris II coordinates for the location.\n- 'time': Timestamp indicating the observation period.\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\n- 'lat' and 'lon': Latitude and longitude of the location.\n- 'TX35D': Number of days with Tx ≥ 35°C.",
+        "sql_query": "CREATE TABLE Number_of_days_with_Tx_above_35C (\n    y FLOAT,\n    x FLOAT,\n    time TIMESTAMP,\n    LambertParisII VARCHAR(255),\n    lat FLOAT,\n    lon FLOAT,\n    TX35D FLOAT\n);"
+    },
+    "Maximum summer temperature": {
+        "description": "The Maximum summer temperature table contains information on the maximum temperature in summer in the past and the future,\nwhich is the highest temperature recorded during the summer period.\nThe variables are as follows:\n- 'y' and 'x': Lambert Paris II coordinates for the location.\n- 'time': Timestamp indicating the observation period.\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\n- 'lat' and 'lon': Latitude and longitude of the location.\n- 'TXm': Maximum temperature recorded in summer.",
+        "sql_query": "CREATE TABLE Maximum_summer_temperature (\n    y FLOAT,\n    x FLOAT,\n    time TIMESTAMP,\n    LambertParisII VARCHAR(255),\n    lat FLOAT,\n    lon FLOAT,\n    TXm FLOAT\n);"
+    }
+}

climateqa/{talk_to_data → engine/talk_to_data}/how_to_use_main.ipynb

@@ -9,7 +9,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [
     {
@@ -17,8 +17,7 @@
      "output_type": "stream",
      "text": [
       "The autoreload extension is already loaded. To reload it, use:\n",
-      "  %reload_ext autoreload\n",
-      "Loading embeddings model:  BAAI/bge-base-en-v1.5\n"
+      "  %reload_ext autoreload\n"
      ]
     }
    ],
@@ -42,7 +41,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -61,44 +60,58 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "SQL Prompt: [{'role': 'system', 'content': \"You are a SQLite expert. Please help to generate a SQL query to answer the question. Your response should ONLY be based on the given context and follow the response guidelines and format instructions. \\n===Tables \\n\\n                CREATE TABLE Mean_winter_temperature (\\n                    y FLOAT,\\n                    x FLOAT,\\n                    year INT, \\n month INT, \\n  day INT \\n,\\n                    LambertParisII VARCHAR(255),\\n                    lat FLOAT,\\n                    lon FLOAT,\\n                    TMm FLOAT, -- Température moyenne en hiver\\n                );\\n            \\n\\n\\n                CREATE TABLE Mean_summer_temperature (\\n                    y FLOAT,\\n                    x FLOAT,\\n                    year INT, \\n month INT, \\n  day INT \\n,\\n                    LambertParisII VARCHAR(255),\\n                    lat FLOAT,\\n                    lon FLOAT,\\n                    TMm FLOAT, -- Température moyenne en été\\n                );\\n            \\n\\n\\n===Additional Context \\n\\n\\n                This table contains information on the number of days when the maximum temperature in the past and the future\\n                is greater than or equal to 35°C.\\n                The variables are as follows:\\n                - 'y' and 'x': Lambert Paris II coordinates for the location.\\n                - year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n                - 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n                - 'lat' and 'lon': Latitude and longitude of the location.\\n                - 'TX35D': Number of days with Tx ≥ 35°C.\\n                \\n\\n\\n                This table contains information on the number of days when the maximum temperature in the past and the future\\n                is greater than or equal to 30°C.\\n                The variables are as follows:\\n                - 'y' and 'x': Lambert Paris II coordinates for the location.\\n                - year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n                - 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n                - 'lat' and 'lon': Latitude and longitude of the location.\\n                - 'TX30D': Number of days with Tx ≥ 30°C.\\n                \\n\\n===Response Guidelines \\n1. If the provided context is sufficient, please generate a valid SQL query without any explanations for the question. \\n2. If the provided context is almost sufficient but requires knowledge of a specific string in a particular column, please generate an intermediate SQL query to find the distinct strings in that column. Prepend the query with a comment saying intermediate_sql \\n3. If the provided context is insufficient, please explain why it can't be generated. \\n4. Please use the most relevant table(s). \\n5. If the question has been asked and answered before, please repeat the answer exactly as it was given before. \\n6. Ensure that the output SQL is SQLite-compliant and executable, and free of syntax errors. \\n\"}, {'role': 'user', 'content': 'Quelle sera la température à lat, long : (43.2961743, 5.3699525) sur les prochaines années ?'}]\n",
-      "Using model gpt-4o-mini for 812.0 tokens (approx)\n",
-      "LLM Response: intermediate_sql\n",
-      "```sql\n",
-      "SELECT DISTINCT year FROM Mean_winter_temperature WHERE lat = 43.2961743 AND lon = 5.3699525\n",
-      "UNION\n",
-      "SELECT DISTINCT year FROM Mean_summer_temperature WHERE lat = 43.2961743 AND lon = 5.3699525;\n",
+      "SQL Prompt: [{'role': 'system', 'content': \"You are a SQLite expert. Please help to generate a SQL query to answer the question. Your response should ONLY be based on the given context and follow the response guidelines and format instructions. \\n===Tables \\n\\n                CREATE TABLE Mean_winter_temperature (\\n                    y FLOAT,\\n                    x FLOAT,\\n                    year INT, \\n month INT, \\n  day INT \\n,\\n                    LambertParisII VARCHAR(255),\\n                    lat FLOAT,\\n                    lon FLOAT,\\n                    TMm FLOAT, -- Température moyenne en hiver\\n                );\\n            \\n\\n\\n                CREATE TABLE Mean_summer_temperature (\\n                    y FLOAT,\\n                    x FLOAT,\\n                    year INT, \\n month INT, \\n  day INT \\n,\\n                    LambertParisII VARCHAR(255),\\n                    lat FLOAT,\\n                    lon FLOAT,\\n                    TMm FLOAT, -- Température moyenne en été\\n                );\\n            \\n\\n\\n===Additional Context \\n\\n\\n                The Number of days with Tx above 35C table contains information on the number of days when the maximum temperature in the past and the future\\n                is greater than or equal to 35°C.\\n                The variables are as follows:\\n                - 'y' and 'x': Lambert Paris II coordinates for the location.\\n                - year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n                - 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n                - 'lat' and 'lon': Latitude and longitude of the location.\\n                - 'TX35D': Number of days with Tx ≥ 35°C.\\n                \\n\\n\\n                The Number of days with Tx above 30C table contains information on the number of days when the maximum temperature in the past and the future\\n                is greater than or equal to 30°C.\\n                The variables are as follows:\\n                - 'y' and 'x': Lambert Paris II coordinates for the location.\\n                - year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n                - 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n                - 'lat' and 'lon': Latitude and longitude of the location.\\n                - 'TX30D': Number of days with Tx ≥ 30°C.\\n                \\n\\n===Response Guidelines \\n1. If the provided context is sufficient, please generate a valid SQL query without any explanations for the question. \\n2. If the provided context is almost sufficient but requires knowledge of a specific string in a particular column, please generate an intermediate SQL query to find the distinct strings in that column. Prepend the query with a comment saying intermediate_sql \\n3. If the provided context is insufficient, please explain why it can't be generated. \\n4. Please use the most relevant table(s). \\n5. If the question has been asked and answered before, please repeat the answer exactly as it was given before. \\n6. Ensure that the output SQL is SQLite-compliant and executable, and free of syntax errors. \\n\"}, {'role': 'user', 'content': 'Quelle sera la température à lat, long : (43.2961743, 5.3699525) sur les prochaines années ?'}]\n",
+      "Using model gpt-4o-mini for 828.0 tokens (approx)\n",
+      "LLM Response: ```sql\n",
+      "SELECT year, TMm \n",
+      "FROM Mean_winter_temperature \n",
+      "WHERE lat = 43.2961743 AND lon = 5.3699525\n",
+      "UNION ALL\n",
+      "SELECT year, TMm \n",
+      "FROM Mean_summer_temperature \n",
+      "WHERE lat = 43.2961743 AND lon = 5.3699525;\n",
       "```\n",
-      "Extracted SQL: SELECT DISTINCT year FROM Mean_winter_temperature WHERE lat = 43.2961743 AND lon = 5.3699525\n",
-      "UNION\n",
-      "SELECT DISTINCT year FROM Mean_summer_temperature WHERE lat = 43.2961743 AND lon = 5.3699525;\n",
-      "Running Intermediate SQL: SELECT DISTINCT year FROM Mean_winter_temperature WHERE lat = 43.2961743 AND lon = 5.3699525\n",
-      "UNION\n",
-      "SELECT DISTINCT year FROM Mean_summer_temperature WHERE lat = 43.2961743 AND lon = 5.3699525;\n",
-      "Final SQL Prompt: [{'role': 'system', 'content': \"You are a SQLite expert. Please help to generate a SQL query to answer the question. Your response should ONLY be based on the given context and follow the response guidelines and format instructions. \\n===Tables \\n\\n                CREATE TABLE Mean_winter_temperature (\\n                    y FLOAT,\\n                    x FLOAT,\\n                    year INT, \\n month INT, \\n  day INT \\n,\\n                    LambertParisII VARCHAR(255),\\n                    lat FLOAT,\\n                    lon FLOAT,\\n                    TMm FLOAT, -- Température moyenne en hiver\\n                );\\n            \\n\\n\\n                CREATE TABLE Mean_summer_temperature (\\n                    y FLOAT,\\n                    x FLOAT,\\n                    year INT, \\n month INT, \\n  day INT \\n,\\n                    LambertParisII VARCHAR(255),\\n                    lat FLOAT,\\n                    lon FLOAT,\\n                    TMm FLOAT, -- Température moyenne en été\\n                );\\n            \\n\\n\\n===Additional Context \\n\\n\\n                This table contains information on the number of days when the maximum temperature in the past and the future\\n                is greater than or equal to 35°C.\\n                The variables are as follows:\\n                - 'y' and 'x': Lambert Paris II coordinates for the location.\\n                - year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n                - 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n                - 'lat' and 'lon': Latitude and longitude of the location.\\n                - 'TX35D': Number of days with Tx ≥ 35°C.\\n                \\n\\n\\n                This table contains information on the number of days when the maximum temperature in the past and the future\\n                is greater than or equal to 30°C.\\n                The variables are as follows:\\n                - 'y' and 'x': Lambert Paris II coordinates for the location.\\n                - year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n                - 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n                - 'lat' and 'lon': Latitude and longitude of the location.\\n                - 'TX30D': Number of days with Tx ≥ 30°C.\\n                \\n\\nThe following is a pandas DataFrame with the results of the intermediate SQL query SELECT DISTINCT year FROM Mean_winter_temperature WHERE lat = 43.2961743 AND lon = 5.3699525\\nUNION\\nSELECT DISTINCT year FROM Mean_summer_temperature WHERE lat = 43.2961743 AND lon = 5.3699525;: \\n| year   |\\n|--------|\\n\\n===Response Guidelines \\n1. If the provided context is sufficient, please generate a valid SQL query without any explanations for the question. \\n2. If the provided context is almost sufficient but requires knowledge of a specific string in a particular column, please generate an intermediate SQL query to find the distinct strings in that column. Prepend the query with a comment saying intermediate_sql \\n3. If the provided context is insufficient, please explain why it can't be generated. \\n4. Please use the most relevant table(s). \\n5. If the question has been asked and answered before, please repeat the answer exactly as it was given before. \\n6. Ensure that the output SQL is SQLite-compliant and executable, and free of syntax errors. \\n\"}, {'role': 'user', 'content': 'Quelle sera la température à lat, long : (43.2961743, 5.3699525) sur les prochaines années ?'}]\n",
-      "Using model gpt-4o-mini for 887.25 tokens (approx)\n",
-      "LLM Response: La context fourni ne contient pas d'informations sur les prévisions de température pour les prochaines années. Par conséquent, je ne peux pas générer de requête SQL pour répondre à cette question.\n",
-      "Couldn't run sql:  Execution failed on sql 'La context fourni ne contient pas d'informations sur les prévisions de température pour les prochaines années. Par conséquent, je ne peux pas générer de requête SQL pour répondre à cette question.': near \"La\": syntax error\n"
+      "Extracted SQL: SELECT year, TMm \n",
+      "FROM Mean_winter_temperature \n",
+      "WHERE lat = 43.2961743 AND lon = 5.3699525\n",
+      "UNION ALL\n",
+      "SELECT year, TMm \n",
+      "FROM Mean_summer_temperature \n",
+      "WHERE lat = 43.2961743 AND lon = 5.3699525;\n",
+      "Using model gpt-4o-mini for 218.5 tokens (approx)\n",
+      "[(2031, 9.952474117647114), (2031, 9.952474117647114), (2032, 10.142322941176474), (2032, 10.142322941176474), (2033, 9.907942941176486), (2033, 9.907942941176486), (2034, 9.548873529411765), (2034, 9.548873529411765), (2035, 10.284758235294191), (2035, 10.284758235294191), (2036, 10.372100000000046), (2036, 10.372100000000046), (2037, 9.98571000000004), (2037, 9.98571000000004), (2038, 10.221372352941216), (2038, 10.221372352941216), (2039, 10.222609411764722), (2039, 10.222609411764722), (2040, 10.473662941176485), (2040, 10.473662941176485), (2041, 10.427640588235306), (2041, 10.427640588235306), (2042, 10.364736470588241), (2042, 10.364736470588241), (2043, 10.112910588235309), (2043, 10.112910588235309), (2044, 10.250792352941176), (2044, 10.250792352941176), (2045, 10.166119411764669), (2045, 10.166119411764669), (2046, 10.728997647058861), (2046, 10.728997647058861), (2047, 10.347248823529412), (2047, 10.347248823529412), (2048, 10.706604117647089), (2048, 10.706604117647089), (2049, 10.59243764705883), (2049, 10.59243764705883), (2050, 10.63225529411767), (2050, 10.63225529411767), (2031, 24.061035294117687), (2031, 24.061035294117687), (2032, 24.530692941176483), (2032, 24.530692941176483), (2033, 24.722234705882386), (2033, 24.722234705882386), (2034, 23.84629176470588), (2034, 23.84629176470588), (2035, 24.231422352941195), (2035, 24.231422352941195), (2036, 24.488941764705885), (2036, 24.488941764705885), (2037, 24.79424117647062), (2037, 24.79424117647062), (2038, 24.730553529411793), (2038, 24.730553529411793), (2039, 24.44979882352942), (2039, 24.44979882352942), (2040, 24.40726882352942), (2040, 24.40726882352942), (2041, 24.768547647058824), (2041, 24.768547647058824), (2042, 24.53479647058822), (2042, 24.53479647058822), (2043, 24.769181176470624), (2043, 24.769181176470624), (2044, 24.489877058823538), (2044, 24.489877058823538), (2045, 24.448076470588262), (2045, 24.448076470588262), (2046, 25.111282352941203), (2046, 25.111282352941203), (2047, 24.72313823529413), (2047, 24.72313823529413), (2048, 25.187577058823535), (2048, 25.187577058823535), (2049, 24.829653529411814), (2049, 24.829653529411814), (2050, 25.053394117647144), (2050, 25.053394117647144)]\n"
      ]
     },
     {
-     "ename": "IndexError",
-     "evalue": "list index out of range",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
-      "Cell \u001b[0;32mIn[4], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mask_vanna\u001b[49m\u001b[43m(\u001b[49m\u001b[43mquery\u001b[49m\u001b[43m)\u001b[49m\n",
-      "File \u001b[0;32m~/ai4s/climate_qa/climate-question-answering/climateqa/talk_to_data/main.py:28\u001b[0m, in \u001b[0;36mask_vanna\u001b[0;34m(query)\u001b[0m\n\u001b[1;32m     26\u001b[0m answer \u001b[38;5;241m=\u001b[39m vn\u001b[38;5;241m.\u001b[39mask(user_input, print_results\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m, allow_llm_to_see_data\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m     27\u001b[0m table \u001b[38;5;241m=\u001b[39m detectTable(answer[\u001b[38;5;241m0\u001b[39m])\n\u001b[0;32m---> 28\u001b[0m coords2 \u001b[38;5;241m=\u001b[39m nearestNeighbourSQL(db_vanna_path, coords, \u001b[43mtable\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m)\n\u001b[1;32m     30\u001b[0m query  \u001b[38;5;241m=\u001b[39m answer[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;241m.\u001b[39mreplace(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mcoords[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mcoords2[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m     31\u001b[0m sql_query \u001b[38;5;241m=\u001b[39m query\u001b[38;5;241m.\u001b[39mreplace(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mcoords[\u001b[38;5;241m1\u001b[39m]\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mcoords2[\u001b[38;5;241m1\u001b[39m]\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n",
-      "\u001b[0;31mIndexError\u001b[0m: list index out of range"
-     ]
+     "data": {
+      "text/plain": [
+       "('SELECT year, TMm \\nFROM Mean_winter_temperature \\nWHERE lat = 43.166954040527344 AND lon = 5.430534839630127\\nUNION ALL\\nSELECT year, TMm \\nFROM Mean_summer_temperature \\nWHERE lat = 43.166954040527344 AND lon = 5.430534839630127;',\n",
+       "     year        TMm\n",
+       " 0   2031   9.952474\n",
+       " 1   2031   9.952474\n",
+       " 2   2032  10.142323\n",
+       " 3   2032  10.142323\n",
+       " 4   2033   9.907943\n",
+       " ..   ...        ...\n",
+       " 75  2048  25.187577\n",
+       " 76  2049  24.829654\n",
+       " 77  2049  24.829654\n",
+       " 78  2050  25.053394\n",
+       " 79  2050  25.053394\n",
+       " \n",
+       " [80 rows x 2 columns])"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
     }
    ],
    "source": [

climateqa/engine/talk_to_data/main.py

@@ -0,0 +1,91 @@
+from climateqa.engine.talk_to_data.myVanna import MyVanna
+from climateqa.engine.talk_to_data.utils import loc2coords, detect_location_with_openai, detectTable, nearestNeighbourSQL
+import sqlite3
+import os
+import pandas as pd
+from climateqa.engine.llm import get_llm
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
+PC_API_KEY = os.getenv('VANNA_PINECONE_API_KEY')
+INDEX_NAME = os.getenv('VANNA_INDEX_NAME')
+VANNA_MODEL = os.getenv('VANNA_MODEL')
+
+
+#Vanna object
+vn = MyVanna(config = {"temperature": 0, "api_key": OPENAI_API_KEY, 'model': VANNA_MODEL, 'pc_api_key': PC_API_KEY, 'index_name': INDEX_NAME})
+db_vanna_path = os.path.join(os.path.dirname(__file__), "database/drias.db")
+vn.connect_to_sqlite(db_vanna_path)
+
+llm = get_llm(provider="openai")
+
+# def ask_vanna(query):
+#     location = detect_location_with_openai(OPENAI_API_KEY, query)
+#     if location:
+#         coords = loc2coords(location)
+#         user_input = query.replace(location, f"lat, long : {coords}")
+#         answer = vn.ask(user_input, print_results=False, allow_llm_to_see_data=True)
+#         table = detectTable(answer[0])
+#         coords2 = nearestNeighbourSQL(db_vanna_path, coords, table[0])
+
+#         query  = answer[0].replace(f"{coords[0]}", f"{coords2[0]}")
+#         sql_query = query.replace(f"{coords[1]}", f"{coords2[1]}")
+
+#         db = sqlite3.connect(db_vanna_path)
+#         result = db.cursor().execute(sql_query).fetchall()
+#         print(result)
+#         df = pd.DataFrame(result, columns=answer[1].columns)
+        
+#     else: 
+#         answer = vn.ask(query, visualize=True, print_results=False, allow_llm_to_see_data=True)
+#         sql_query = answer[0]
+#         df = answer[1]
+        
+#     return (sql_query, df)
+def replace_coordonates(coords, sql_query, coords_tables):
+    n = sql_query.count(str(coords[0]))
+    sql_query_new_coords = sql_query
+
+    for i in range(n):
+        sql_query_new_coords = sql_query_new_coords.replace(str(coords[0]), str(coords_tables[i][0]),1)
+        sql_query_new_coords = sql_query_new_coords.replace(str(coords[1]), str(coords_tables[i][1]),1)
+    return sql_query_new_coords
+
+def ask_vanna(query):
+    location = detect_location_with_openai(OPENAI_API_KEY, query)
+    if location:
+
+        coords = loc2coords(location)
+        user_input = query.replace(location, f"lat, long : {coords}")
+        sql_query, result_dataframe, figure = vn.ask(user_input, print_results=False, allow_llm_to_see_data=True)
+        table = detectTable(sql_query)
+        coords_tables = [nearestNeighbourSQL(db_vanna_path, coords, table[i]) for i in range(len(table))]
+        sql_query_new_coords = replace_coordonates(coords, sql_query, coords_tables)
+        sql_with_table_names = llm.invoke(f"Make the following sql query display the source table in the rows {sql_query_new_coords}. Just answer the query. The answer should not include ```sql\n").content
+        print("execute sql query : ", sql_with_table_names)
+        db = sqlite3.connect(db_vanna_path)
+        # if "lat" not in sql_with_table_names:
+        # sql_with_table_names = sql_with_table_names.replace("SELECT", "SELECT lat, lon,")
+        # result = db.cursor().execute(sql_with_table_names).fetchall()
+        result = db.cursor().execute(sql_query_new_coords).fetchall()
+        # df = pd.DataFrame(result, columns = list(result_dataframe.columns))
+        # df = pd.DataFrame(result, columns=["data_name"] + list(result_dataframe.columns))
+        df = pd.DataFrame(result)
+        
+        plotly_code = vn.generate_plotly_code(
+                            question="query",
+                            sql="sql_with_table_names",
+                            df_metadata=f"Running df.dtypes gives:\n {df.dtypes}",
+                        )
+
+        fig = vn.get_plotly_figure(plotly_code=plotly_code, df=df)
+
+        return df, fig
+    else : 
+        empty_df = pd.DataFrame()
+        empty_fig = {}
+        return empty_df, empty_fig

climateqa/{talk_to_data → engine/talk_to_data}/myVanna.py

@@ -1,5 +1,5 @@
 from dotenv import load_dotenv
-from climateqa.talk_to_data.vanna_class import MyCustomVectorDB
+from climateqa.engine.talk_to_data.vanna_class import MyCustomVectorDB
 from vanna.openai import OpenAI_Chat
 import os
 

climateqa/engine/talk_to_data/pinecone_vanna_training.ipynb

@@ -0,0 +1,592 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The autoreload extension is already loaded. To reload it, use:\n",
+      "  %reload_ext autoreload\n",
+      "{'temperature': 0.2, 'api_key': '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', 'model': 'gpt-4o-mini', 'pc_api_key': 'pcsk_5pEfJ8_GqSCikBaVhK3V6wehh4YHCegspQeshyWVesKeqmqzcmfLgkQRpWaUVJvSyTcdG', 'index_name': 'cqa-vanna'}\n",
+      "Loading embeddings model:  BAAI/bge-base-en-v1.5\n"
+     ]
+    }
+   ],
+   "source": [
+    "from vanna_class import MyCustomVectorDB\n",
+    "from vanna.openai import OpenAI_Chat\n",
+    "import os\n",
+    "from dotenv import load_dotenv\n",
+    "\n",
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
+    "\n",
+    "load_dotenv()\n",
+    "\n",
+    "OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')\n",
+    "OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')\n",
+    "PC_API_KEY = os.getenv('VANNA_PINECONE_API_KEY')\n",
+    "INDEX_NAME = os.getenv('VANNA_INDEX_NAME')\n",
+    "VANNA_MODEL = os.getenv('VANNA_MODEL')\n",
+    "\n",
+    "INDEX_NAME = \"cqa-vanna\"\n",
+    "PC_API_KEY = \"pcsk_5pEfJ8_GqSCikBaVhK3V6wehh4YHCegspQeshyWVesKeqmqzcmfLgkQRpWaUVJvSyTcdG\"\n",
+    "\n",
+    "class MyVanna(MyCustomVectorDB, OpenAI_Chat):\n",
+    "    def __init__(self, config=None):\n",
+    "        print(config)\n",
+    "        MyCustomVectorDB.__init__(self, config=config)\n",
+    "        OpenAI_Chat.__init__(self, config=config)\n",
+    "\n",
+    "vn = MyVanna(\n",
+    "    config={\n",
+    "        'temperature': 0.2,\n",
+    "        'api_key': OPENAI_API_KEY,\n",
+    "        'model': 'gpt-4o-mini',\n",
+    "        'pc_api_key': PC_API_KEY,\n",
+    "        'index_name': INDEX_NAME\n",
+    "    }\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 49,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "\n",
+    "with open('drias_metadata.json', 'r') as file:\n",
+    "    tables_info = json.load(file)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 50,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def convert(tables_info):\n",
+    "    text = \"\"\"- year: Year of the observation.\\n\n",
+    "              - month : Month of the observation.\\n\n",
+    "              - day: Day of the observation.\\n\"\"\"\n",
+    "    for table_name in tables_info:\n",
+    "        tables_info[table_name]['description'] = tables_info[table_name]['description'].replace(\"- 'time': Timestamp indicating the observation period.\", text)\n",
+    "        tables_info[table_name]['description'] = tables_info[table_name]['description'].replace(\"- `time`: Timestamp representing the date of the data.\", text)\n",
+    "        tables_info[table_name]['sql_query'] = tables_info[table_name]['sql_query'].replace(\"time TIMESTAMP\", \"year INT, \\n month INT, \\n  day INT \\n\")\n",
+    "        \n",
+    "    return tables_info"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 51,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'Frequency of rainy days index': {'description': 'The Frequency of rainy days index table contains the frequency index of rainy days for each latitude longitude couple for each date in the past and the future.\\nThe columns include:\\n- year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n- `x`: Coordinate in the Lambert II projection for the location.\\n- `y`: Coordinate in the Lambert II projection for the location.\\n- `IFM40D`: Frequency index of rainy days.\\n- `Lon`: Geographic longitude of the location.\\n- `lat`: Geographic latitude of the location.',\n",
+       "  'sql_query': 'CREATE TABLE Frequency_of_rainy_days_index (\\n    year INT, \\n month INT, \\n  day INT \\n,\\n    x FLOAT,\\n    y FLOAT,\\n    IFM40D FLOAT,\\n    Lon FLOAT,\\n    lat FLOAT\\n);'},\n",
+       " 'Remarkable daily precipitation total (Q99)': {'description': 'The Remarkable daily precipitation total table contains the daily cumulative exceptional rainfall (Q99) for each latitude longitude couple for each date in the past and the future.\\nThe columns include:\\n- year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n- `x`: Coordinate in the Lambert II projection for the location.\\n- `y`: Coordinate in the Lambert II projection for the location.\\n- `LambertParisII`: Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n- `RRq99`: Cumulative exceptional rainfall.\\n- `lat`: Geographic latitude of the location.\\n- `lon`: Geographic longitude of the location.',\n",
+       "  'sql_query': 'CREATE TABLE Remarkable_daily_precipitation_total_(Q99) (\\n    year INT, \\n month INT, \\n  day INT \\n,\\n    x FLOAT,\\n    y FLOAT,\\n    LambertParisII VARCHAR(255),\\n    RRq99 FLOAT,\\n    lat FLOAT,\\n    lon FLOAT\\n);'},\n",
+       " 'Frequency of remarkable daily precipitation': {'description': 'The Frequency of remarkable daily precipitation table contains the frequency of daily exceptional rainfall in the past and the future.\\nThe columns include:\\n- year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n- `x`: Coordinate in the Lambert II projection for the location.\\n- `y`: Coordinate in the Lambert II projection for the location.\\n- `LambertParisII`: Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n- `RRq99refD`: Frequency of exceptional rainfall.\\n- `lat`: Geographic latitude of the location.\\n- `lon`: Geographic longitude of the location.',\n",
+       "  'sql_query': 'CREATE TABLE Frequency_of_remarkable_daily_precipitation (\\n    year INT, \\n month INT, \\n  day INT \\n,\\n    x FLOAT,\\n    y FLOAT,\\n    LambertParisII VARCHAR(255),\\n    RRq99refD FLOAT,\\n    lat FLOAT,\\n    lon FLOAT\\n);'},\n",
+       " 'Winter precipitation total': {'description': 'The Winter precipitation total table contains the cumulative winter precipitation in the past and the future.\\nThe columns include:\\n- year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n- `x`: Coordinate in the Lambert II projection for the location.\\n- `y`: Coordinate in the Lambert II projection for the location.\\n- `LambertParisII`: Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n- `RR`: Cumulative winter precipitation.\\n- `lat`: Geographic latitude of the location.\\n- `lon`: Geographic longitude of the location.',\n",
+       "  'sql_query': 'CREATE TABLE Winter_precipitation_total (\\n    year INT, \\n month INT, \\n  day INT \\n,\\n    x FLOAT,\\n    y FLOAT,\\n    LambertParisII VARCHAR(255),\\n    RR FLOAT,\\n    lat FLOAT,\\n    lon FLOAT\\n);'},\n",
+       " 'Summer precipitation total': {'description': 'The Summer precipitation total table contains the cumulative summer precipitation in the past and the future.\\nThe columns include:\\n- year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n- `x`: Coordinate in the Lambert II projection for the location.\\n- `y`: Coordinate in the Lambert II projection for the location.\\n- `LambertParisII`: Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n- `RR`: Cumulative summer precipitation.\\n- `lat`: Geographic latitude of the location.\\n- `lon`: Geographic longitude of the location.',\n",
+       "  'sql_query': 'CREATE TABLE Summer_precipitation_total (\\n    year INT, \\n month INT, \\n  day INT \\n,\\n    x FLOAT,\\n    y FLOAT,\\n    LambertParisII VARCHAR(255),\\n    RR FLOAT,\\n    lat FLOAT,\\n    lon FLOAT\\n);'},\n",
+       " 'Annual precipitation total': {'description': \"The Annual precipitation total table contains information on the cumulative annual precipitation in the past and the future.\\nbased on Lambert Paris II projections.\\nThe variables are as follows:\\n- 'y' and 'x': Lambert Paris II coordinates for the location.\\n- year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n- 'lat' and 'lon': Latitude and longitude of the location.\\n- 'RR': Cumulative annual precipitation.\",\n",
+       "  'sql_query': 'CREATE TABLE Annual_precipitation_total (\\n    y FLOAT,\\n    x FLOAT,\\n    year INT, \\n month INT, \\n  day INT \\n,\\n    LambertParisII VARCHAR(255),\\n    lat FLOAT,\\n    lon FLOAT,\\n    RR FLOAT\\n);'},\n",
+       " 'Extreme precipitation intensity': {'description': \"The Extreme precipitation intensity table contains information on the intensity of extreme precipitation in the past and the future,\\nwhich represents the maximum value of total annual precipitation.\\nThe variables are as follows:\\n- 'y' and 'x': Lambert Paris II coordinates for the location.\\n- year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n- 'lat' and 'lon': Latitude and longitude of the location.\\n- 'RX1d': Intensity of extreme precipitation (maximum annual total precipitation).\",\n",
+       "  'sql_query': 'CREATE TABLE Extreme_precipitation_intensity (\\n    y FLOAT,\\n    x FLOAT,\\n    year INT, \\n month INT, \\n  day INT \\n,\\n    LambertParisII VARCHAR(255),\\n    lat FLOAT,\\n    lon FLOAT,\\n    RX1d FLOAT\\n);'},\n",
+       " 'Drought index': {'description': \"The Drought index table contains information on the drought index based on observations over the past and the future.\\nThe variables are as follows:\\n- year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n- 'y' and 'x': Lambert Paris II coordinates for the location.\\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n- 'lat' and 'lon': Latitude and longitude of the location.\\n- 'SWI04D': Drought index based on the analysis of precipitation and temperatures.\",\n",
+       "  'sql_query': 'CREATE TABLE Drought_index (\\n    year INT, \\n month INT, \\n  day INT \\n,\\n    y FLOAT,\\n    x FLOAT,\\n    lat FLOAT,\\n    lon FLOAT,\\n    LambertParisII VARCHAR(255),\\n    SWI04D FLOAT\\n);'},\n",
+       " 'Mean winter temperature': {'description': \"The Mean winter temperature table contains information on the average (mean) winter temperature in the past and the future.\\nThe variables are as follows:\\n- 'y' and 'x': Lambert Paris II coordinates for the location.\\n- year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n- 'lat' and 'lon': Latitude and longitude of the location.\\n- 'TMm': Average winter temperature.\",\n",
+       "  'sql_query': 'CREATE TABLE Mean_winter_temperature (\\n    y FLOAT,\\n    x FLOAT,\\n    year INT, \\n month INT, \\n  day INT \\n,\\n    LambertParisII VARCHAR(255),\\n    lat FLOAT,\\n    lon FLOAT,\\n    TMm FLOAT\\n);'},\n",
+       " 'Mean summer temperature': {'description': \"The Mean summer temperature table contains information on the average summer temperature in the past and the future.\\nThe variables are as follows:\\n- 'y' and 'x': Lambert Paris II coordinates for the location.\\n- year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n- 'LambertParisII': Indicates that the x and y coordinates are in Lambert Paris II projection.\\n- 'lat' and 'lon': Latitude and longitude of the location.\\n- 'TMm': Average summer temperature.\",\n",
+       "  'sql_query': 'CREATE TABLE Mean_summer_temperature (\\n    y FLOAT,\\n    x FLOAT,\\n    year INT, \\n month INT, \\n  day INT \\n,\\n    LambertParisII VARCHAR(255),\\n    lat FLOAT,\\n    lon FLOAT,\\n    TMm FLOAT\\n);'},\n",
+       " 'Number of tropical nights': {'description': \"The Number of tropical nights table contains information on the average summer temperature in the past and the future.\\nThe variables are as follows:\\n- 'y' and 'x': Lambert Paris II coordinates for the location.\\n- year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n- 'lat' and 'lon': Latitude and longitude of the location.\\n- 'TMm': Average summer temperature.\",\n",
+       "  'sql_query': 'CREATE TABLE Number_of_tropical_nights (\\n    y FLOAT,\\n    x FLOAT,\\n    year INT, \\n month INT, \\n  day INT \\n,\\n    LambertParisII VARCHAR(255),\\n    lat FLOAT,\\n    lon FLOAT,\\n    TR FLOAT\\n);'},\n",
+       " 'Number of days with Tx above 30C': {'description': \"The Number of days with Tx above 30C table contains information on the number of days when the maximum temperature in the past and the future\\nis greater than or equal to 30°C.\\nThe variables are as follows:\\n- 'y' and 'x': Lambert Paris II coordinates for the location.\\n- year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n- 'lat' and 'lon': Latitude and longitude of the location.\\n- 'TX30D': Number of days with Tx ≥ 30°C.\",\n",
+       "  'sql_query': 'CREATE TABLE Number_of_days_with_Tx_above_30C (\\n    y FLOAT,\\n    x FLOAT,\\n    year INT, \\n month INT, \\n  day INT \\n,\\n    LambertParisII VARCHAR(255),\\n    lat FLOAT,\\n    lon FLOAT,\\n    TX30D FLOAT\\n);'},\n",
+       " 'Number of days with Tx above 35C': {'description': \"The Number of days with Tx above 35C table contains information on the number of days when the maximum temperature in the past and the future\\nis greater than or equal to 35°C.\\nThe variables are as follows:\\n- 'y' and 'x': Lambert Paris II coordinates for the location.\\n- year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n- 'lat' and 'lon': Latitude and longitude of the location.\\n- 'TX35D': Number of days with Tx ≥ 35°C.\",\n",
+       "  'sql_query': 'CREATE TABLE Number_of_days_with_Tx_above_35C (\\n    y FLOAT,\\n    x FLOAT,\\n    year INT, \\n month INT, \\n  day INT \\n,\\n    LambertParisII VARCHAR(255),\\n    lat FLOAT,\\n    lon FLOAT,\\n    TX35D FLOAT\\n);'},\n",
+       " 'Maximum summer temperature': {'description': \"The Maximum summer temperature table contains information on the maximum temperature in summer in the past and the future,\\nwhich is the highest temperature recorded during the summer period.\\nThe variables are as follows:\\n- 'y' and 'x': Lambert Paris II coordinates for the location.\\n- year: Year of the observation.\\n\\n              - month : Month of the observation.\\n\\n              - day: Day of the observation.\\n\\n- 'LambertParisII': Indicates that the x, y coordinates are in the Lambert Paris II projection.\\n- 'lat' and 'lon': Latitude and longitude of the location.\\n- 'TXm': Maximum temperature recorded in summer.\",\n",
+       "  'sql_query': 'CREATE TABLE Maximum_summer_temperature (\\n    y FLOAT,\\n    x FLOAT,\\n    year INT, \\n month INT, \\n  day INT \\n,\\n    LambertParisII VARCHAR(255),\\n    lat FLOAT,\\n    lon FLOAT,\\n    TXm FLOAT\\n);'}}"
+      ]
+     },
+     "execution_count": 51,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "new_tables_info = convert(tables_info)\n",
+    "new_tables_info\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 52,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Adding ddl: CREATE TABLE Frequency_of_rainy_days_index (\n",
+      "    year INT, \n",
+      " month INT, \n",
+      "  day INT \n",
+      ",\n",
+      "    x FLOAT,\n",
+      "    y FLOAT,\n",
+      "    IFM40D FLOAT,\n",
+      "    Lon FLOAT,\n",
+      "    lat FLOAT\n",
+      ");\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Adding documentation....\n",
+      "Adding ddl: CREATE TABLE Remarkable_daily_precipitation_total_(Q99) (\n",
+      "    year INT, \n",
+      " month INT, \n",
+      "  day INT \n",
+      ",\n",
+      "    x FLOAT,\n",
+      "    y FLOAT,\n",
+      "    LambertParisII VARCHAR(255),\n",
+      "    RRq99 FLOAT,\n",
+      "    lat FLOAT,\n",
+      "    lon FLOAT\n",
+      ");\n",
+      "Adding documentation....\n",
+      "Adding ddl: CREATE TABLE Frequency_of_remarkable_daily_precipitation (\n",
+      "    year INT, \n",
+      " month INT, \n",
+      "  day INT \n",
+      ",\n",
+      "    x FLOAT,\n",
+      "    y FLOAT,\n",
+      "    LambertParisII VARCHAR(255),\n",
+      "    RRq99refD FLOAT,\n",
+      "    lat FLOAT,\n",
+      "    lon FLOAT\n",
+      ");\n",
+      "Adding documentation....\n",
+      "Adding ddl: CREATE TABLE Winter_precipitation_total (\n",
+      "    year INT, \n",
+      " month INT, \n",
+      "  day INT \n",
+      ",\n",
+      "    x FLOAT,\n",
+      "    y FLOAT,\n",
+      "    LambertParisII VARCHAR(255),\n",
+      "    RR FLOAT,\n",
+      "    lat FLOAT,\n",
+      "    lon FLOAT\n",
+      ");\n",
+      "Adding documentation....\n",
+      "Adding ddl: CREATE TABLE Summer_precipitation_total (\n",
+      "    year INT, \n",
+      " month INT, \n",
+      "  day INT \n",
+      ",\n",
+      "    x FLOAT,\n",
+      "    y FLOAT,\n",
+      "    LambertParisII VARCHAR(255),\n",
+      "    RR FLOAT,\n",
+      "    lat FLOAT,\n",
+      "    lon FLOAT\n",
+      ");\n",
+      "Adding documentation....\n",
+      "Adding ddl: CREATE TABLE Annual_precipitation_total (\n",
+      "    y FLOAT,\n",
+      "    x FLOAT,\n",
+      "    year INT, \n",
+      " month INT, \n",
+      "  day INT \n",
+      ",\n",
+      "    LambertParisII VARCHAR(255),\n",
+      "    lat FLOAT,\n",
+      "    lon FLOAT,\n",
+      "    RR FLOAT\n",
+      ");\n",
+      "Adding documentation....\n",
+      "Adding ddl: CREATE TABLE Extreme_precipitation_intensity (\n",
+      "    y FLOAT,\n",
+      "    x FLOAT,\n",
+      "    year INT, \n",
+      " month INT, \n",
+      "  day INT \n",
+      ",\n",
+      "    LambertParisII VARCHAR(255),\n",
+      "    lat FLOAT,\n",
+      "    lon FLOAT,\n",
+      "    RX1d FLOAT\n",
+      ");\n",
+      "Adding documentation....\n",
+      "Adding ddl: CREATE TABLE Drought_index (\n",
+      "    year INT, \n",
+      " month INT, \n",
+      "  day INT \n",
+      ",\n",
+      "    y FLOAT,\n",
+      "    x FLOAT,\n",
+      "    lat FLOAT,\n",
+      "    lon FLOAT,\n",
+      "    LambertParisII VARCHAR(255),\n",
+      "    SWI04D FLOAT\n",
+      ");\n",
+      "Adding documentation....\n",
+      "Adding ddl: CREATE TABLE Mean_winter_temperature (\n",
+      "    y FLOAT,\n",
+      "    x FLOAT,\n",
+      "    year INT, \n",
+      " month INT, \n",
+      "  day INT \n",
+      ",\n",
+      "    LambertParisII VARCHAR(255),\n",
+      "    lat FLOAT,\n",
+      "    lon FLOAT,\n",
+      "    TMm FLOAT\n",
+      ");\n",
+      "Adding documentation....\n",
+      "Adding ddl: CREATE TABLE Mean_summer_temperature (\n",
+      "    y FLOAT,\n",
+      "    x FLOAT,\n",
+      "    year INT, \n",
+      " month INT, \n",
+      "  day INT \n",
+      ",\n",
+      "    LambertParisII VARCHAR(255),\n",
+      "    lat FLOAT,\n",
+      "    lon FLOAT,\n",
+      "    TMm FLOAT\n",
+      ");\n",
+      "Adding documentation....\n",
+      "Adding ddl: CREATE TABLE Number_of_tropical_nights (\n",
+      "    y FLOAT,\n",
+      "    x FLOAT,\n",
+      "    year INT, \n",
+      " month INT, \n",
+      "  day INT \n",
+      ",\n",
+      "    LambertParisII VARCHAR(255),\n",
+      "    lat FLOAT,\n",
+      "    lon FLOAT,\n",
+      "    TR FLOAT\n",
+      ");\n",
+      "Adding documentation....\n",
+      "Adding ddl: CREATE TABLE Number_of_days_with_Tx_above_30C (\n",
+      "    y FLOAT,\n",
+      "    x FLOAT,\n",
+      "    year INT, \n",
+      " month INT, \n",
+      "  day INT \n",
+      ",\n",
+      "    LambertParisII VARCHAR(255),\n",
+      "    lat FLOAT,\n",
+      "    lon FLOAT,\n",
+      "    TX30D FLOAT\n",
+      ");\n",
+      "Adding documentation....\n",
+      "Adding ddl: CREATE TABLE Number_of_days_with_Tx_above_35C (\n",
+      "    y FLOAT,\n",
+      "    x FLOAT,\n",
+      "    year INT, \n",
+      " month INT, \n",
+      "  day INT \n",
+      ",\n",
+      "    LambertParisII VARCHAR(255),\n",
+      "    lat FLOAT,\n",
+      "    lon FLOAT,\n",
+      "    TX35D FLOAT\n",
+      ");\n",
+      "Adding documentation....\n",
+      "Adding ddl: CREATE TABLE Maximum_summer_temperature (\n",
+      "    y FLOAT,\n",
+      "    x FLOAT,\n",
+      "    year INT, \n",
+      " month INT, \n",
+      "  day INT \n",
+      ",\n",
+      "    LambertParisII VARCHAR(255),\n",
+      "    lat FLOAT,\n",
+      "    lon FLOAT,\n",
+      "    TXm FLOAT\n",
+      ");\n",
+      "Adding documentation....\n"
+     ]
+    }
+   ],
+   "source": [
+    "for table in new_tables_info:\n",
+    "    vn.train(ddl = new_tables_info[table]['sql_query'])\n",
+    "    vn.train(documentation = new_tables_info[table]['description'])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Requests"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# examples"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 56,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Loading embeddings model:  BAAI/bge-base-en-v1.5\n"
+     ]
+    }
+   ],
+   "source": [
+    "question_sql = [\n",
+    "    # ['How will the precipitations change in the coming years', \n",
+    "    #  'SELECT * FROM Annual_precipitation_total WHERE year > 2024;'],\n",
+    "    # ['What is the number of days where the temperature above 35 in year 2050 in (lat, lon) = (48.82337188720703, 2.390951633453369)', \n",
+    "    #  'SELECT TX35D FROM Number_of_days_with_Tx_above_35C WHERE year = 2050 AND lat = 48.82337188720703 AND lon = 2.390951633453369;'],\n",
+    "    # ['How will change the mean winter temperature in the coming years in lat = 43.2961743 lon = 5.3699525', \n",
+    "    #  'SELECT * FROM Mean_winter_temperature WHERE year > 2023 AND lat = 43.2961743 AND lon = 5.3699525'],\n",
+    "    # ['How will change the mean summer temperature in the coming years in lat = 43.2961743, lon = 5.3699525', \n",
+    "    #  'SELECT * FROM Mean_summer_temperature WHERE year > 2023 AND lat = 43.2961743 AND lon = 5.3699525'],\n",
+    "    # ['How will change the temperature in the coming years', \n",
+    "    #  'SELECT * FROM Mean_summer_temperature JOIN Mean_winter_temperature WHERE year > 2024']\n",
+    "    [\"Quelle sera la température à lat, long : (43.2961743, 5.3699525) sur les prochaines années ?\",\n",
+    "     'SELECT Mean_winter_temperature AS table_name, lat, lon, year, TMm \\nFROM Mean_winter_temperature \\nWHERE lat = 43.2961743 AND lon = 5.3699525\\nUNION ALL\\nSELECT \"Mean_summer_temperature\" AS table_name, lat, lon, year, TMm \\nFROM Mean_summer_temperature \\nWHERE lat = 43.2961743 AND lon = 5.3699525;']\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 58,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "de6738cb3a67eaec29a04119ad160b27acd64f2d87cf83782c5b2bdc84be445b_sql\n"
+     ]
+    }
+   ],
+   "source": [
+    "for question in question_sql:\n",
+    "    print(vn.train(question = question[0], sql = question[1]))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Delete"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'cqa-vanna'"
+      ]
+     },
+     "execution_count": 31,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "INDEX_NAME"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": []
+     },
+     "execution_count": 36,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from pinecone.grpc import PineconeGRPC as Pinecone\n",
+    "\n",
+    "pc = Pinecone(api_key=PC_API_KEY)\n",
+    "\n",
+    "# To get the unique host for an index, \n",
+    "# see https://docs.pinecone.io/guides/data/target-an-index\n",
+    "index = pc.Index(INDEX_NAME)\n",
+    "# index\n",
+    "index.delete(delete_all=True, namespace='ddl')\n",
+    "index.delete(delete_all=True, namespace='documentation')\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# OLD"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import vanna\n",
+    "from vanna.remote import VannaDefault"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# PC_API_KEY = os.getenv('VANNA_PINECONE_API_KEY')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[\n",
+       "    {\n",
+       "        \"name\": \"cqa-vanna\",\n",
+       "        \"dimension\": 768,\n",
+       "        \"metric\": \"cosine\",\n",
+       "        \"host\": \"cqa-vanna-9xlylwt.svc.aped-4627-b74a.pinecone.io\",\n",
+       "        \"spec\": {\n",
+       "            \"serverless\": {\n",
+       "                \"cloud\": \"aws\",\n",
+       "                \"region\": \"us-east-1\"\n",
+       "            }\n",
+       "        },\n",
+       "        \"status\": {\n",
+       "            \"ready\": true,\n",
+       "            \"state\": \"Ready\"\n",
+       "        },\n",
+       "        \"deletion_protection\": \"disabled\"\n",
+       "    },\n",
+       "    {\n",
+       "        \"name\": \"unepqa\",\n",
+       "        \"dimension\": 768,\n",
+       "        \"metric\": \"cosine\",\n",
+       "        \"host\": \"unepqa-9xlylwt.svc.aped-4627-b74a.pinecone.io\",\n",
+       "        \"spec\": {\n",
+       "            \"serverless\": {\n",
+       "                \"cloud\": \"aws\",\n",
+       "                \"region\": \"us-east-1\"\n",
+       "            }\n",
+       "        },\n",
+       "        \"status\": {\n",
+       "            \"ready\": true,\n",
+       "            \"state\": \"Ready\"\n",
+       "        },\n",
+       "        \"deletion_protection\": \"disabled\"\n",
+       "    }\n",
+       "]"
+      ]
+     },
+     "execution_count": 34,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from pinecone.grpc import PineconeGRPC as Pinecone\n",
+    "\n",
+    "pc = Pinecone(api_key=PC_API_KEY)\n",
+    "pc.list_indexes()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'dimension': 768,\n",
+       " 'index_fullness': 0.0,\n",
+       " 'namespaces': {'ddl': {'vector_count': 14},\n",
+       "                'documentation': {'vector_count': 14}},\n",
+       " 'total_vector_count': 28}"
+      ]
+     },
+     "execution_count": 35,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "pc.Index(\"cqa-vanna\").describe_index_stats()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "climateqa",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

climateqa/{talk_to_data → engine/talk_to_data}/vanna_class.py

@@ -167,80 +167,156 @@ class MyCustomVectorDB(VannaBase):
         pass
 
 
-def get_sql_prompt(
-        self,
-        initial_prompt : str,
-        question: str,
-        question_sql_list: list,
-        ddl_list: list,
-        doc_list: list,
-        **kwargs,
-    ):
-        """
-        Example:
-        ```python
-        vn.get_sql_prompt(
-            question="What are the top 10 customers by sales?",
-            question_sql_list=[{"question": "What are the top 10 customers by sales?", "sql": "SELECT * FROM customers ORDER BY sales DESC LIMIT 10"}],
-            ddl_list=["CREATE TABLE customers (id INT, name TEXT, sales DECIMAL)"],
-            doc_list=["The customers table contains information about customers and their sales."],
-        )
+    def get_sql_prompt(
+            self,
+            initial_prompt : str,
+            question: str,
+            question_sql_list: list,
+            ddl_list: list,
+            doc_list: list,
+            **kwargs,
+        ):
+            """
+            Example:
+            ```python
+            vn.get_sql_prompt(
+                question="What are the top 10 customers by sales?",
+                question_sql_list=[{"question": "What are the top 10 customers by sales?", "sql": "SELECT * FROM customers ORDER BY sales DESC LIMIT 10"}],
+                ddl_list=["CREATE TABLE customers (id INT, name TEXT, sales DECIMAL)"],
+                doc_list=["The customers table contains information about customers and their sales."],
+            )
 
-        ```
+            ```
 
-        This method is used to generate a prompt for the LLM to generate SQL.
+            This method is used to generate a prompt for the LLM to generate SQL.
 
-        Args:
-            question (str): The question to generate SQL for.
-            question_sql_list (list): A list of questions and their corresponding SQL statements.
-            ddl_list (list): A list of DDL statements.
-            doc_list (list): A list of documentation.
+            Args:
+                question (str): The question to generate SQL for.
+                question_sql_list (list): A list of questions and their corresponding SQL statements.
+                ddl_list (list): A list of DDL statements.
+                doc_list (list): A list of documentation.
 
-        Returns:
-            any: The prompt for the LLM to generate SQL.
-        """
+            Returns:
+                any: The prompt for the LLM to generate SQL.
+            """
 
-        if initial_prompt is None:
-            initial_prompt = f"You are a {self.dialect} expert. " + \
-            "Please help to generate a SQL query to answer the question. Your response should ONLY be based on the given context and follow the response guidelines and format instructions. "
+            if initial_prompt is None:
+                initial_prompt = f"You are a {self.dialect} expert. " + \
+                "Please help to generate a SQL query to answer the question. Your response should ONLY be based on the given context and follow the response guidelines and format instructions. "
 
-        initial_prompt = self.add_ddl_to_prompt(
-            initial_prompt, ddl_list, max_tokens=self.max_tokens
-        )
+            initial_prompt = self.add_ddl_to_prompt(
+                initial_prompt, ddl_list, max_tokens=self.max_tokens
+            )
 
-        if self.static_documentation != "":
-            doc_list.append(self.static_documentation)
+            if self.static_documentation != "":
+                doc_list.append(self.static_documentation)
 
-        initial_prompt = self.add_documentation_to_prompt(
-            initial_prompt, doc_list, max_tokens=self.max_tokens
-        )
+            initial_prompt = self.add_documentation_to_prompt(
+                initial_prompt, doc_list, max_tokens=self.max_tokens
+            )
 
-        initial_prompt = self.add_sql_to_prompt(
-            initial_prompt, question_sql_list, max_tokens=self.max_tokens
-        )
+            # initial_prompt = self.add_sql_to_prompt(
+            #     initial_prompt, question_sql_list, max_tokens=self.max_tokens
+            # )
 
 
-        initial_prompt += (
-            "===Response Guidelines \n"
-            "1. If the provided context is sufficient, please generate a valid SQL query without any explanations for the question. \n"
-            "2. If the provided context is almost sufficient but requires knowledge of a specific string in a particular column, please generate an intermediate SQL query to find the distinct strings in that column. Prepend the query with a comment saying intermediate_sql \n"
-            "3. If the provided context is insufficient, please give a sql query based on your knowledge and the context provided. \n"
-            "4. Please use the most relevant table(s). \n"
-            "5. If the question has been asked and answered before, please repeat the answer exactly as it was given before. \n"
-            f"6. Ensure that the output SQL is {self.dialect}-compliant and executable, and free of syntax errors. \n"
-        )
+            initial_prompt += (
+                "===Response Guidelines \n"
+                "1. If the provided context is sufficient, please generate a valid SQL query without any explanations for the question. \n"
+                "2. If the provided context is almost sufficient but requires knowledge of a specific string in a particular column, please generate an intermediate SQL query to find the distinct strings in that column. Prepend the query with a comment saying intermediate_sql \n"
+                "3. If the provided context is insufficient, please give a sql query based on your knowledge and the context provided. \n"
+                "4. Please use the most relevant table(s). \n"
+                "5. If the question has been asked and answered before, please repeat the answer exactly as it was given before. \n"
+                f"6. Ensure that the output SQL is {self.dialect}-compliant and executable, and free of syntax errors. \n"
+                # "7. Add a description of the table in the result of the sql query."
+                # "7. If the question is about a specific latitude, longitude, query an interval of 0.3 and keep only the first set of coordinate. \n"
+                # "7. Table names should be included in the result of the sql query. Use for example Mean_winter_temperature AS table_name in the query \n"
+            )
+
+
+            message_log = [self.system_message(initial_prompt)]
+
+            for example in question_sql_list:
+                if example is None:
+                    print("example is None")
+                else:
+                    if example is not None and "question" in example and "sql" in example:
+                        message_log.append(self.user_message(example["question"]))
+                        message_log.append(self.assistant_message(example["sql"]))
+
+            message_log.append(self.user_message(question))
+
+            return message_log
+        
+
+# def get_sql_prompt(
+#         self,
+#         initial_prompt : str,
+#         question: str,
+#         question_sql_list: list,
+#         ddl_list: list,
+#         doc_list: list,
+#         **kwargs,
+#     ):
+#         """
+#         Example:
+#         ```python
+#         vn.get_sql_prompt(
+#             question="What are the top 10 customers by sales?",
+#             question_sql_list=[{"question": "What are the top 10 customers by sales?", "sql": "SELECT * FROM customers ORDER BY sales DESC LIMIT 10"}],
+#             ddl_list=["CREATE TABLE customers (id INT, name TEXT, sales DECIMAL)"],
+#             doc_list=["The customers table contains information about customers and their sales."],
+#         )
+
+#         ```
+
+#         This method is used to generate a prompt for the LLM to generate SQL.
+
+#         Args:
+#             question (str): The question to generate SQL for.
+#             question_sql_list (list): A list of questions and their corresponding SQL statements.
+#             ddl_list (list): A list of DDL statements.
+#             doc_list (list): A list of documentation.
+
+#         Returns:
+#             any: The prompt for the LLM to generate SQL.
+#         """
+
+#         if initial_prompt is None:
+#             initial_prompt = f"You are a {self.dialect} expert. " + \
+#             "Please help to generate a SQL query to answer the question. Your response should ONLY be based on the given context and follow the response guidelines and format instructions. "
+
+#         initial_prompt = self.add_ddl_to_prompt(
+#             initial_prompt, ddl_list, max_tokens=self.max_tokens
+#         )
+
+#         if self.static_documentation != "":
+#             doc_list.append(self.static_documentation)
+
+#         initial_prompt = self.add_documentation_to_prompt(
+#             initial_prompt, doc_list, max_tokens=self.max_tokens
+#         )
 
+#         initial_prompt += (
+#             "===Response Guidelines \n"
+#             "1. If the provided context is sufficient, please generate a valid SQL query without any explanations for the question. \n"
+#             "2. If the provided context is almost sufficient but requires knowledge of a specific string in a particular column, please generate an intermediate SQL query to find the distinct strings in that column. Prepend the query with a comment saying intermediate_sql \n"
+#             "3. If the provided context is insufficient, please explain why it can't be generated. \n"
+#             "4. Please use the most relevant table(s). \n"
+#             "5. If the question has been asked and answered before, please repeat the answer exactly as it was given before. \n"
+#             f"6. Ensure that the output SQL is {self.dialect}-compliant and executable, and free of syntax errors. \n"
+#         )
 
-        message_log = [self.system_message(initial_prompt)]
+#         message_log = [self.system_message(initial_prompt)]
 
-        for example in question_sql_list:
-            if example is None:
-                print("example is None")
-            else:
-                if example is not None and "question" in example and "sql" in example:
-                    message_log.append(self.user_message(example["question"]))
-                    message_log.append(self.assistant_message(example["sql"]))
+#         for example in question_sql_list:
+#             if example is None:
+#                 print("example is None")
+#             else:
+#                 if example is not None and "question" in example and "sql" in example:
+#                     message_log.append(self.user_message(example["question"]))
+#                     message_log.append(self.assistant_message(example["sql"]))
 
-        message_log.append(self.user_message(question))
+#         message_log.append(self.user_message(question))
 
-        return message_log
+#         return message_log

climateqa/talk_to_data/main.py

@@ -1,43 +0,0 @@
-from climateqa.talk_to_data.myVanna import MyVanna
-from climateqa.talk_to_data.utils import loc2coords, detect_location_with_openai, detectTable, nearestNeighbourSQL
-import sqlite3
-import os
-import pandas as pd
-
-
-
-OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
-PC_API_KEY = os.getenv('VANNA_PINECONE_API_KEY')
-INDEX_NAME = os.getenv('VANNA_INDEX_NAME')
-VANNA_MODEL = os.getenv('VANNA_MODEL')
-
-
-#Vanna object
-vn = MyVanna(config = {"temperature": 0, "api_key": OPENAI_API_KEY, 'model': VANNA_MODEL, 'pc_api_key': PC_API_KEY, 'index_name': INDEX_NAME})
-db_vanna_path = os.path.join(os.path.dirname(__file__), "database/drias.db")
-vn.connect_to_sqlite(db_vanna_path)
-
-
-def ask_vanna(query):
-    location = detect_location_with_openai(OPENAI_API_KEY, query)
-    if location:
-        coords = loc2coords(location)
-        user_input = query.replace(location, f"lat, long : {coords}")
-        answer = vn.ask(user_input, print_results=False, allow_llm_to_see_data=True)
-        table = detectTable(answer[0])
-        coords2 = nearestNeighbourSQL(db_vanna_path, coords, table[0])
-
-        query  = answer[0].replace(f"{coords[0]}", f"{coords2[0]}")
-        sql_query = query.replace(f"{coords[1]}", f"{coords2[1]}")
-
-        db = sqlite3.connect(db_vanna_path)
-        result = db.cursor().execute(sql_query).fetchall()
-        print(result)
-        df = pd.DataFrame(result, columns=answer[1].columns)
-        
-    else: 
-        answer = vn.ask(query, visualize=True, print_results=False, allow_llm_to_see_data=True)
-        sql_query = answer[0]
-        df = answer[1]
-        
-    return (sql_query, df)

sandbox/20241104 - CQA - StepByStep CQA.ipynb

@@ -163,43 +163,15 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "INFO:sentence_transformers.SentenceTransformer:Load pretrained SentenceTransformer: BAAI/bge-base-en-v1.5\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Loading embeddings model:  BAAI/bge-base-en-v1.5\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "INFO:sentence_transformers.SentenceTransformer:Use pytorch device_name: cpu\n",
-      "INFO:pinecone_plugin_interface.logging:Discovering subpackages in _NamespacePath(['/home/tim/anaconda3/envs/climateqa/lib/python3.11/site-packages/pinecone_plugins'])\n",
-      "INFO:pinecone_plugin_interface.logging:Looking for plugins in pinecone_plugins.inference\n",
-      "INFO:pinecone_plugin_interface.logging:Installing plugin inference into Pinecone\n",
-      "INFO:pinecone_plugin_interface.logging:Discovering subpackages in _NamespacePath(['/home/tim/anaconda3/envs/climateqa/lib/python3.11/site-packages/pinecone_plugins'])\n",
-      "INFO:pinecone_plugin_interface.logging:Looking for plugins in pinecone_plugins.inference\n",
-      "INFO:pinecone_plugin_interface.logging:Installing plugin inference into Pinecone\n",
-      "INFO:pinecone_plugin_interface.logging:Discovering subpackages in _NamespacePath(['/home/tim/anaconda3/envs/climateqa/lib/python3.11/site-packages/pinecone_plugins'])\n",
-      "INFO:pinecone_plugin_interface.logging:Looking for plugins in pinecone_plugins.inference\n",
-      "INFO:pinecone_plugin_interface.logging:Installing plugin inference into Pinecone\n"
-     ]
-    },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
+      "Loading embeddings model:  BAAI/bge-base-en-v1.5\n",
+      "Loading embeddings model:  BAAI/bge-base-en-v1.5\n",
       "Loading FlashRankRanker model ms-marco-TinyBERT-L-2-v2\n",
       "Loading model FlashRank model ms-marco-TinyBERT-L-2-v2...\n"
      ]
@@ -238,7 +210,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -261,7 +233,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -279,9 +251,51 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 11,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "---- Categorize_message ----\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "Output intent categorization: {'language': 'French', 'intent': 'search'}\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "{'user_input': \"Quel est l'impact du changement climatique sur Bordeaux ?\",\n",
+       " 'audience': 'the general public who know the basics in science and climate change and want to learn more about it without technical terms. Still use references to passages.',\n",
+       " 'relevant_content_sources_selection': ['Figures (IPCC/IPBES)', 'POC region'],\n",
+       " 'search_only': False,\n",
+       " 'reports': [],\n",
+       " 'language': 'French',\n",
+       " 'intent': 'search',\n",
+       " 'query': \"Quel est l'impact du changement climatique sur Bordeaux ?\"}"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "cat_node = make_intent_categorization_node(llm)\n",
     "state.update(cat_node(inial_state))\n",
@@ -290,7 +304,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 95,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -300,7 +314,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 96,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -312,9 +326,60 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 14,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "---- Transform query ----\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "{'user_input': \"Quel est l'impact du changement climatique sur Bordeaux ?\",\n",
+       " 'audience': 'the general public who know the basics in science and climate change and want to learn more about it without technical terms. Still use references to passages.',\n",
+       " 'relevant_content_sources_selection': ['Figures (IPCC/IPBES)', 'POC region'],\n",
+       " 'search_only': False,\n",
+       " 'reports': [],\n",
+       " 'language': 'French',\n",
+       " 'intent': 'search',\n",
+       " 'query': \"Quel est l'impact du changement climatique sur Bordeaux ?\",\n",
+       " 'questions_list': [{'question': \"Quel est l'impact du changement climatique sur la région de Bordeaux ?\",\n",
+       "   'sources': ['AcclimaTerra'],\n",
+       "   'index': 'POC',\n",
+       "   'source_type': 'POC'},\n",
+       "  {'question': 'How does climate change affect the economy and environment of Bordeaux?',\n",
+       "   'sources': ['IPCC'],\n",
+       "   'index': 'IPx',\n",
+       "   'source_type': 'IPx'},\n",
+       "  {'question': \"Comment le changement climatique affecte-t-il l'économie et l'environnement de Bordeaux ?\",\n",
+       "   'sources': ['AcclimaTerra'],\n",
+       "   'index': 'POC',\n",
+       "   'source_type': 'POC'}],\n",
+       " 'n_questions': {'total': 3, 'IPx': 1, 'POC': 2},\n",
+       " 'handled_questions_index': []}"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "state.update(make_query_transform_node(llm)(state))\n",
     "state"