wip

app.py

@@ -1,87 +1,193 @@
-import streamlit as st
-from pathlib import Path
-from langchain.llms.openai import OpenAI
-from langchain.agents import create_sql_agent
-from langchain.sql_database import SQLDatabase
-from langchain.agents.agent_types import AgentType
-from langchain_community.callbacks import StreamlitCallbackHandler
-from langchain.agents.agent_toolkits import SQLDatabaseToolkit
-from sqlalchemy import create_engine
-import sqlite3
+# This example does not use a langchain agent, 
+# The langchain sql chain has knowledge of the database, but doesn't interact with it becond intialization.
+# The output of the sql chain is parsed seperately and passed to `duckdb.sql()` by streamlit
+
 import os
-from langchain_openai import ChatOpenAI
-os.environ["OPENAI_API_KEY"] = st.secrets["OPENAI_API_KEY"]
-st.set_page_config(page_title="Protected Areas Database Chat", page_icon="🦜", layout="wide")
-st.title("🦜 Protected Areas Database Chat")
+os.environ["WEBSOCKET_TIMEOUT_MS"] = "300000" # no effect
 
-#db_uri = "duckdb:///:memory:"
+import streamlit as st
+import geopandas as gpd
+from shapely import wkb
+import leafmap.foliumap as leafmap
+
+# Helper plotting functions
+import pydeck as pdk
+def deck_map(gdf):
+    st.write(
+        pdk.Deck(
+            map_style="mapbox://styles/mapbox/light-v9",
+            initial_view_state={
+                "latitude": 35,
+                "longitude": -100,
+                "zoom": 3,
+                "pitch": 50,
+            },
+            layers=[
+                pdk.Layer(
+                    "GeoJsonLayer",
+                    gdf,
+                    pickable=True,
+                    stroked=True,
+                    filled=True,
+                    extruded=True,
+                    elevation_scale=10,
+                    get_fill_color=[2, 200, 100],
+                    get_line_color=[0,0,0],
+                    line_width_min_pixels=0,
+                ),
+            ],
+        )
+    )
+
+def leaf_map(gdf):
+    m = leafmap.Map(center=[35, -100], zoom=4, layers_control=True)
+    m.add_gdf(gdf)
+    return m.to_streamlit()
+
+
+@st.cache_data
+def query_database(response):
+    return con.sql(response).to_pandas().head(25)
+
+@st.cache_data
+def get_geom(tbl):
+    tbl['geometry'] = tbl['geometry'].apply(wkb.loads)
+    gdf = gpd.GeoDataFrame(tbl, geometry='geometry')
+    return gdf
+
+
+## Database connection
+from sqlalchemy import create_engine
+from langchain.sql_database import SQLDatabase
 db_uri = "duckdb:///pad.duckdb"
-engine = create_engine(db_uri)
-from sqlalchemy import text
-con = engine.connect()
-#con.execute(text("create or replace view agency_name as select * from 'https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-agency-name.parquet'"))
-#con.execute(text("create or replace view agency_name as select * from 'https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-agency-name.parquet'"))
-#con.execute(text("create or replace view agency_type as  select * from 'https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-agency-type.parquet'"))
-#con.execute(text("create or replace view category as  select * from 'https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-category.parquet'"))
-#con.execute(text("create or replace view designation_type as  select * from 'https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-desgination-type.parquet'"))
-#con.execute(text("create or replace view easement as  select * from 'https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-easement.parquet'"))
-#con.execute(text("create or replace view fee as  select * from 'https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-fee.parquet'"))
-#con.execute(text("create or replace view marine as  select * from 'https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-marine.parquet'"))
-#con.execute(text("create or replace view iucn as  select * from 'https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-iucn.parquet'"))
-#con.execute(text("create or replace view public_access as  select * from 'https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-public-access.parquet'"))
-#con.execute(text("create or replace view state_name as  select * from 'https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-state-name.parquet'"))
-#con.execute(text("create or replace view combined as  select * from 'https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-combined.parquet'"))
-
+engine = create_engine(db_uri, connect_args={'read_only': True})
 db = SQLDatabase(engine, view_support=True)
-db.get_usable_table_names()
 
+import ibis
+con = ibis.connect("duckdb://pad.duckdb", read_only=True)
+con.load_extension("spatial")
 
+## ChatGPT Connection
+from langchain_openai import ChatOpenAI 
 
+# Requires ollama server running locally
+from langchain_community.llms import Ollama
 
-# User inputs
-radio_opt = ["US Protected Areas v3"]
-selected_opt = st.sidebar.radio(label="Choose suitable option", options=radio_opt)
+## should we use ChatOllama instead?
+# from langchain_community.llms import ChatOllama
 
-llm = ChatOpenAI(model="gpt-3.5-turbo", temperature=0)
-agent = create_sql_agent(llm, db=db, agent_type="openai-tools", verbose=True)
+models = {"chatgpt3.5": ChatOpenAI(model="gpt-3.5-turbo", temperature=0, api_key=st.secrets["OPENAI_API_KEY"])}
 
-def handle_user_input(user_query):
-    with history:
-        st.session_state.messages.append({"role": "user", "content": user_query})
-        #st.chat_message("user").write(user_query)
+other_models = {
+          "chatgpt4": ChatOpenAI(model="gpt-4", temperature=0, api_key=st.secrets["OPENAI_API_KEY"]),
+          "duckdb-nsql": Ollama(model="duckdb-nsql", temperature=0),
+          "command-r-plus": Ollama(model="command-r-plus", temperature=0),
+          "mixtral:8x22b":  Ollama(model="mixtral:8x22b", temperature=0),
+          "wizardlm2:8x22b":  Ollama(model="wizardlm2:8x22b", temperature=0),
+          "sqlcoder": Ollama(model="sqlcoder", temperature=0),
+          "zephyr": Ollama(model="zephyr", temperature=0),
+          "gemma:7b": Ollama(model="gemma:7b", temperature=0),
+          "codegemma": Ollama(model="codegemma", temperature=0),
+          "llama2": Ollama(model="llama2", temperature=0),
+         }
 
-        with st.chat_message("assistant"):
-            st_cb = StreamlitCallbackHandler(st.container())
-            response = agent.run(user_query, callbacks=[st_cb])
-            st.session_state.messages.append({"role": "assistant", "content": response})
-        #    st.write(response) # thinking is only shown transiently this way
 
+st.set_page_config(page_title="Protected Areas Database Chat", page_icon="🦜", layout="wide")
+st.title("Protected Areas Database Chat")
+
+map_tool = {"leafmap": leaf_map,
+            "deckgl": deck_map
+           }
+
+with st.sidebar:
+    choice = st.radio("Select an LLM:", models)
+    llm = models[choice]
+    map_choice = st.radio("Select mapping tool", map_tool)
+    mapper = map_tool[map_choice]
+## A SQL Chain
+from langchain.chains import create_sql_query_chain
+chain = create_sql_query_chain(llm, db)
 
-if "messages" not in st.session_state:
-    st.session_state["messages"] = []
 
 main = st.container()
+
+## Does not preserve history
 with main:
-    history = st.container(height=400)
-    # stores all questions and responses, but not the 'thinking'
-    with history:
-        for msg in st.session_state.messages:
-            st.chat_message(msg["role"]).write(msg["content"])  
-    if user_query := st.chat_input(placeholder="Ask me about US Protected areas!"):
-        handle_user_input(user_query)
-    
-    st.markdown("\n") #add some space for iphone users
 
+    '''
+    The Protected Areas Database of the United States (PAD-US) is the official national inventory of
+    America’s parks and other protected lands, and is published by the USGS Gap Analysis Project,
+    [https://doi.org/10.5066/P9Q9LQ4B.](https://doi.org/10.5066/P9Q9LQ4B).  
 
+    This interactive tool allows users to explore the dataset, as well as a range of biodiversity
+    and climate indicators associated with each protected area. These indicators are integrated into 
+    a single table format shown below.  The chatbot assistant can turn natural language queries into
+    SQL queries based on the table schema.
 
+    See our [Protected Areas Explorer](https://huggingface.co/spaces/boettiger-lab/pad-us) for a companion non-chat-based tool.
 
-EXAMPLE_PROMPTS = ["What is the total area in each GAP_Sts category in the fee table?",
-                   "List the name of each table in the database",
-                   "How much BLM land (BLM is a Mang_Name in the fee table) is in each GAP_Sts category?",
-                   "Federal agencies are identified as 'FED' in the Mang_Type column in the 'combined' data table. The Mang_Name column indicates the different agencies. The full name of each agency is given in the agency_name table. Which federal agencies, by full name, manage the greatest area of GAP_Sts 1 or 2 land?"]
+    ##### Example Queries returning summary tables
 
-with st.sidebar:
-    with st.container():
-        st.title("Examples")
-        for prompt in EXAMPLE_PROMPTS:
-            st.button(prompt, args=(prompt,), on_click=handle_user_input)
+    - What is the percent area in each gap code as a fraction of the total protected area?
+    - The manager_type column indicates whether a manager is federal, state, local, private, or NGO.  
+      the manager_name column indicates the responsible agency (National Park Service, Bureau of Land Management,
+      etc) in the case of federal manager types.  Which of the federal managers manage the most land in
+      gap_code 1 or 2, as a fraction of the total area?
+
+    When queries refer to specific managed areas, the chatbot can show those areas on an interactive map.
+    Do to software limitations, these maps will show no more than 25 polygons, even if more areas match the
+    requested search. The chatbot sometimes requires help identifying the right columns.  In order to create
+    a map, the SQL query must also return the geometry column.  Conisder the following examples:
+
+    ##### Example queries returning maps + tables
+
+    - Show me all the national monuments (designation_type) in Utah. Include the geometry column
+    - Show examples of Bureau of Land Management (manager_name) with the highest species richness? Include the geometry column
+    - Which site has the overall highest range-size-rarity? Include the geometry column, manager_name, and IUCN category.
+
+    '''
+
+    st.markdown("## 🦜 Chatbot:")
+    chatbox = st.container()
+    with chatbox:           
+        if prompt := st.chat_input(key="chain"):
+            st.chat_message("user").write(prompt)
+            with st.chat_message("assistant"):
+                response = chain.invoke({"question": prompt})
+                st.write(response)
+                tbl = query_database(response)
+                if 'geometry' in tbl:
+                    gdf = get_geom(tbl)
+                    mapper(gdf)
+                    n = len(gdf)
+                    st.write(f"matching features: {n}")
+                st.dataframe(tbl)
+
+
+st.divider()
+
+with st.container():
+    st.text("Database schema (top 3 rows)")
+    tbl = tbl = query_database("select * from pad limit 3")
+    st.dataframe(tbl)
+
+
+st.divider()
+
+'''
+Experimental prototype.  
+
+- Author: [Carl Boettiger](https://carlboettiger.info)
+- For data sources and processing, see: https://beta.source.coop/repositories/cboettig/pad-us-3/description/
+
+
+'''
+
+# duckdb_sql fails but chatgpt3.5 succeeds with a query like:
+# use the st_area function and st_GeomFromWKB functions to compute the area of the Shape column in the fee table, and then use that to compute the total area under each GAP_Sts category
+
+# For most queries, duckdb_sql does much better than alternative open models though
+
+# Federal agencies are identified as 'FED' in the Mang_Type column in the 'combined' data table. The Mang_Name column indicates the different agencies. Which federal agencies manage the greatest area of GAP_Sts 1 or 2 land?
+
+# Federal agencies are identified as 'FED' in the Mang_Type column in the table named "fee". The Mang_Name column indicates the different agencies. List which managers manage the largest total areas that identified as GAP_Sts '1' or '2' ?

minimal-example.py

@@ -29,7 +29,10 @@ from langchain_community.llms import Ollama
 
 models = {"duckdb-nsql": Ollama(model="duckdb-nsql", temperature=0),
           "sqlcoder": Ollama(model="sqlcoder", temperature=0),
-          "gemma": Ollama(model="gemma", temperature=0),
+          "zephyr": Ollama(model="zephyr", temperature=0),
+          "gemma:7b": Ollama(model="gemma:7b", temperature=0),
+          "codegemma": Ollama(model="codegemma", temperature=0),
+          "llama2:70b": Ollama(model="llama2:70b", temperature=0),
           "chatgpt3.5": chatgpt_llm, 
           "chatgpt4": chatgpt4_llm}
 with st.sidebar:
@@ -57,5 +60,8 @@ if prompt := st.chat_input():
 # use the st_area function and st_GeomFromWKB functions to compute the area of the Shape column in the fee table, and then use that to compute the total area under each GAP_Sts category
 
 
+# For most queries, duckdb_sql does much better than alternative open models though
+
 # Federal agencies are identified as 'FED' in the Mang_Type column in the 'combined' data table. The Mang_Name column indicates the different agencies. Which federal agencies manage the greatest area of GAP_Sts 1 or 2 land?
 
+# Federal agencies are identified as 'FED' in the Mang_Type column in the table named "fee". The Mang_Name column indicates the different agencies. List which managers manage the largest total areas that identified as GAP_Sts '1' or '2' ?

minimal-requirements.txt

@@ -4,4 +4,4 @@ langchain
 langchain-community
 langchain-openai
 SQLAlchemy==1.4.52
-streamlit
+streamlit