pages

app.py

@@ -1,193 +1,22 @@
-# This example does not use a langchain agent, 
-# The langchain sql chain has knowledge of the database, but doesn't interact with it becond intialization.
-# The output of the sql chain is parsed seperately and passed to `duckdb.sql()` by streamlit
-
-import os
-os.environ["WEBSOCKET_TIMEOUT_MS"] = "300000" # no effect
-
 import streamlit as st
-import geopandas as gpd
-from shapely import wkb
-import leafmap.foliumap as leafmap
-
-# Helper plotting functions
-import pydeck as pdk
-def deck_map(gdf):
-    st.write(
-        pdk.Deck(
-            map_style="mapbox://styles/mapbox/light-v9",
-            initial_view_state={
-                "latitude": 35,
-                "longitude": -100,
-                "zoom": 3,
-                "pitch": 50,
-            },
-            layers=[
-                pdk.Layer(
-                    "GeoJsonLayer",
-                    gdf,
-                    pickable=True,
-                    stroked=True,
-                    filled=True,
-                    extruded=True,
-                    elevation_scale=10,
-                    get_fill_color=[2, 200, 100],
-                    get_line_color=[0,0,0],
-                    line_width_min_pixels=0,
-                ),
-            ],
-        )
-    )
-
-def leaf_map(gdf):
-    m = leafmap.Map(center=[35, -100], zoom=4, layers_control=True)
-    m.add_gdf(gdf)
-    return m.to_streamlit()
-
-
-@st.cache_data
-def query_database(response):
-    return con.sql(response).to_pandas().head(25)
-
-@st.cache_data
-def get_geom(tbl):
-    tbl['geometry'] = tbl['geometry'].apply(wkb.loads)
-    gdf = gpd.GeoDataFrame(tbl, geometry='geometry')
-    return gdf
-
-
-## Database connection
-from sqlalchemy import create_engine
-from langchain.sql_database import SQLDatabase
-db_uri = "duckdb:///pad.duckdb"
-engine = create_engine(db_uri, connect_args={'read_only': True})
-db = SQLDatabase(engine, view_support=True)
-
-import ibis
-con = ibis.connect("duckdb://pad.duckdb", read_only=True)
-con.load_extension("spatial")
-
-## ChatGPT Connection
-from langchain_openai import ChatOpenAI 
-
-# Requires ollama server running locally
-from langchain_community.llms import Ollama
-
-## should we use ChatOllama instead?
-# from langchain_community.llms import ChatOllama
-
-models = {"chatgpt3.5": ChatOpenAI(model="gpt-3.5-turbo", temperature=0, api_key=st.secrets["OPENAI_API_KEY"])}
-
-other_models = {
-          "chatgpt4": ChatOpenAI(model="gpt-4", temperature=0, api_key=st.secrets["OPENAI_API_KEY"]),
-          "duckdb-nsql": Ollama(model="duckdb-nsql", temperature=0),
-          "command-r-plus": Ollama(model="command-r-plus", temperature=0),
-          "mixtral:8x22b":  Ollama(model="mixtral:8x22b", temperature=0),
-          "wizardlm2:8x22b":  Ollama(model="wizardlm2:8x22b", temperature=0),
-          "sqlcoder": Ollama(model="sqlcoder", temperature=0),
-          "zephyr": Ollama(model="zephyr", temperature=0),
-          "gemma:7b": Ollama(model="gemma:7b", temperature=0),
-          "codegemma": Ollama(model="codegemma", temperature=0),
-          "llama2": Ollama(model="llama2", temperature=0),
-         }
-
-
-st.set_page_config(page_title="Protected Areas Database Chat", page_icon="🦜", layout="wide")
-st.title("Protected Areas Database Chat")
-
-map_tool = {"leafmap": leaf_map,
-            "deckgl": deck_map
-           }
-
-with st.sidebar:
-    choice = st.radio("Select an LLM:", models)
-    llm = models[choice]
-    map_choice = st.radio("Select mapping tool", map_tool)
-    mapper = map_tool[map_choice]
-## A SQL Chain
-from langchain.chains import create_sql_query_chain
-chain = create_sql_query_chain(llm, db)
 
+st.set_page_config(
+    page_title="Protected Area Database Chat tool",
+    page_icon="🦜",
+)
 
-main = st.container()
+st.sidebar.success("Select a demo above.")
 
-## Does not preserve history
-with main:
+st.markdown('''
 
-    '''
     The Protected Areas Database of the United States (PAD-US) is the official national inventory of
     America’s parks and other protected lands, and is published by the USGS Gap Analysis Project,
     [https://doi.org/10.5066/P9Q9LQ4B.](https://doi.org/10.5066/P9Q9LQ4B).  
 
-    This interactive tool allows users to explore the dataset, as well as a range of biodiversity
-    and climate indicators associated with each protected area. These indicators are integrated into 
-    a single table format shown below.  The chatbot assistant can turn natural language queries into
-    SQL queries based on the table schema.
+    These interactive tools allows users to explore the dataset, as well as a range of biodiversity
+    and climate indicators associated with each protected area. Select a tool from the menu on the left.
 
     See our [Protected Areas Explorer](https://huggingface.co/spaces/boettiger-lab/pad-us) for a companion non-chat-based tool.
 
-    ##### Example Queries returning summary tables
-
-    - What is the percent area in each gap code as a fraction of the total protected area?
-    - The manager_type column indicates whether a manager is federal, state, local, private, or NGO.  
-      the manager_name column indicates the responsible agency (National Park Service, Bureau of Land Management,
-      etc) in the case of federal manager types.  Which of the federal managers manage the most land in
-      gap_code 1 or 2, as a fraction of the total area?
-
-    When queries refer to specific managed areas, the chatbot can show those areas on an interactive map.
-    Do to software limitations, these maps will show no more than 25 polygons, even if more areas match the
-    requested search. The chatbot sometimes requires help identifying the right columns.  In order to create
-    a map, the SQL query must also return the geometry column.  Conisder the following examples:
-
-    ##### Example queries returning maps + tables
-
-    - Show me all the national monuments (designation_type) in Utah. Include the geometry column
-    - Show examples of Bureau of Land Management (manager_name) with the highest species richness? Include the geometry column
-    - Which site has the overall highest range-size-rarity? Include the geometry column, manager_name, and IUCN category.
-
-    '''
-
-    st.markdown("## 🦜 Chatbot:")
-    chatbox = st.container()
-    with chatbox:           
-        if prompt := st.chat_input(key="chain"):
-            st.chat_message("user").write(prompt)
-            with st.chat_message("assistant"):
-                response = chain.invoke({"question": prompt})
-                st.write(response)
-                tbl = query_database(response)
-                if 'geometry' in tbl:
-                    gdf = get_geom(tbl)
-                    mapper(gdf)
-                    n = len(gdf)
-                    st.write(f"matching features: {n}")
-                st.dataframe(tbl)
-
-
-st.divider()
-
-with st.container():
-    st.text("Database schema (top 3 rows)")
-    tbl = tbl = query_database("select * from pad limit 3")
-    st.dataframe(tbl)
-
-
-st.divider()
-
-'''
-Experimental prototype.  
-
-- Author: [Carl Boettiger](https://carlboettiger.info)
-- For data sources and processing, see: https://beta.source.coop/repositories/cboettig/pad-us-3/description/
-
-
-'''
-
-# duckdb_sql fails but chatgpt3.5 succeeds with a query like:
-# use the st_area function and st_GeomFromWKB functions to compute the area of the Shape column in the fee table, and then use that to compute the total area under each GAP_Sts category
-
-# For most queries, duckdb_sql does much better than alternative open models though
-
-# Federal agencies are identified as 'FED' in the Mang_Type column in the 'combined' data table. The Mang_Name column indicates the different agencies. Which federal agencies manage the greatest area of GAP_Sts 1 or 2 land?
 
-# Federal agencies are identified as 'FED' in the Mang_Type column in the table named "fee". The Mang_Name column indicates the different agencies. List which managers manage the largest total areas that identified as GAP_Sts '1' or '2' ?
+''')

graphs.py → pages/1_🌍_Mapping_Demo.py

@@ -7,44 +7,8 @@ os.environ["WEBSOCKET_TIMEOUT_MS"] = "300000" # no effect
 
 import streamlit as st
 import geopandas as gpd
-import pandas as pd
 from shapely import wkb
-
-
-st.set_page_config(page_title="Protected Areas Database Chat", page_icon="🦜", layout="wide")
-st.title("Protected Areas Database Chat")
-
-## Database connection, reading directly from remote parquet file
-from sqlalchemy import create_engine
-from langchain.sql_database import SQLDatabase
-db_uri = "duckdb:///:memory:"
-parquet = "https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/pad-stats.parquet"
-engine = create_engine(db_uri) #connect_args={'read_only': True})
-con = engine.connect()
-con.execute("install spatial; load spatial;")
-h = con.execute(f"create or replace view pad as select * from read_parquet('{parquet}');")
-h.fetchall()
-db = SQLDatabase(engine, view_support=True)
-
-
-@st.cache_data
-def query_database(response):
-    # con.sql(response).to_pandas().head(25) # uses ibis connection
-    # instead, use direct sqlAlchemy connection
-    z = con.execute(response).fetchall()
-    return pd.DataFrame(z).head(25)
-     
-
-query_database("select * from pad limit 1")
-
-
-@st.cache_data
-def get_geom(tbl):
-    tbl['geometry'] = tbl['geometry'].apply(wkb.loads)
-    gdf = gpd.GeoDataFrame(tbl, geometry='geometry')
-    return gdf
-
-
+import leafmap.foliumap as leafmap
 
 # Helper plotting functions
 import pydeck as pdk
@@ -75,20 +39,41 @@ def deck_map(gdf):
         )
     )
 
-import leafmap.foliumap as leafmap
 def leaf_map(gdf):
     m = leafmap.Map(center=[35, -100], zoom=4, layers_control=True)
     m.add_gdf(gdf)
     return m.to_streamlit()
 
 
+@st.cache_data
+def query_database(response):
+    return con.sql(response).to_pandas().head(25)
+
+@st.cache_data
+def get_geom(tbl):
+    tbl['geometry'] = tbl['geometry'].apply(wkb.loads)
+    gdf = gpd.GeoDataFrame(tbl, geometry='geometry')
+    return gdf
+
+
+## Database connection
+from sqlalchemy import create_engine
+from langchain.sql_database import SQLDatabase
+db_uri = "duckdb:///pad.duckdb"
+engine = create_engine(db_uri, connect_args={'read_only': True})
+db = SQLDatabase(engine, view_support=True)
+
+import ibis
+con = ibis.connect("duckdb://pad.duckdb", read_only=True)
+con.load_extension("spatial")
 
 ## ChatGPT Connection
 from langchain_openai import ChatOpenAI 
+
 # Requires ollama server running locally
 from langchain_community.llms import Ollama
 
-# # should we use ChatOllama instead?
+## should we use ChatOllama instead?
 # from langchain_community.llms import ChatOllama
 
 models = {"chatgpt3.5": ChatOpenAI(model="gpt-3.5-turbo", temperature=0, api_key=st.secrets["OPENAI_API_KEY"])}
@@ -107,7 +92,8 @@ other_models = {
          }
 
 
-
+st.set_page_config(page_title="Protected Areas Database Chat", page_icon="🦜", layout="wide")
+st.title("Protected Areas Database Chat")
 
 map_tool = {"leafmap": leaf_map,
             "deckgl": deck_map
@@ -129,16 +115,7 @@ main = st.container()
 with main:
 
     '''
-    The Protected Areas Database of the United States (PAD-US) is the official national inventory of
-    America’s parks and other protected lands, and is published by the USGS Gap Analysis Project,
-    [https://doi.org/10.5066/P9Q9LQ4B.](https://doi.org/10.5066/P9Q9LQ4B).  
 
-    This interactive tool allows users to explore the dataset, as well as a range of biodiversity
-    and climate indicators associated with each protected area. These indicators are integrated into 
-    a single table format shown below.  The chatbot assistant can turn natural language queries into
-    SQL queries based on the table schema.
-
-    See our [Protected Areas Explorer](https://huggingface.co/spaces/boettiger-lab/pad-us) for a companion non-chat-based tool.
 
     ##### Example Queries returning summary tables
 
@@ -196,3 +173,12 @@ Experimental prototype.
 
 
 '''
+
+# duckdb_sql fails but chatgpt3.5 succeeds with a query like:
+# use the st_area function and st_GeomFromWKB functions to compute the area of the Shape column in the fee table, and then use that to compute the total area under each GAP_Sts category
+
+# For most queries, duckdb_sql does much better than alternative open models though
+
+# Federal agencies are identified as 'FED' in the Mang_Type column in the 'combined' data table. The Mang_Name column indicates the different agencies. Which federal agencies manage the greatest area of GAP_Sts 1 or 2 land?
+
+# Federal agencies are identified as 'FED' in the Mang_Type column in the table named "fee". The Mang_Name column indicates the different agencies. List which managers manage the largest total areas that identified as GAP_Sts '1' or '2' ?

pages/2_📊_Chart_Demo.py

@@ -0,0 +1,207 @@
+# This example does not use a langchain agent, 
+# The langchain sql chain has knowledge of the database, but doesn't interact with it becond intialization.
+# The output of the sql chain is parsed seperately and passed to `duckdb.sql()` by streamlit
+
+import os
+os.environ["WEBSOCKET_TIMEOUT_MS"] = "300000" # no effect
+
+import streamlit as st
+import geopandas as gpd
+import pandas as pd
+from shapely import wkb
+
+st.set_page_config(page_title="Protected Areas Database Chat", page_icon="🦜", layout="wide")
+st.title("Protected Areas Database Chat")
+
+## Database connection, reading directly from remote parquet file
+from sqlalchemy import create_engine
+from langchain.sql_database import SQLDatabase
+db_uri = "duckdb:///my.duckdb"
+parquet = "https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/pad-stats.parquet"
+stats = "../pad-us/pad-stats.parquet"
+groups = "../pad-us/pad-groupings.parquet"
+engine = create_engine(db_uri) #connect_args={'read_only': True})
+con = engine.connect()
+con.execute("install spatial; load spatial;")
+# con.execute(f"create or replace view stats as select * from read_parquet('{stats}');").fetchall()
+con.execute(f"create or replace table groups as select * from read_parquet('{groups}');").fetchall()
+
+db = SQLDatabase(engine, view_support=True)
+
+
+@st.cache_data
+def query_database(response):
+    z = con.execute(response).fetchall()
+    return pd.DataFrame(z).head(25)
+     
+import ibis
+from ibis import _
+import ibis.selectors as s
+import altair as alt
+ibis_con = ibis.duckdb.connect("my.duckdb")
+stats = ibis_con.read_parquet(stats)
+us_lower_48_area_m2 = 7.8e+12
+
+
+
+
+def summary_table(stats, query, column):
+    #z = con.execute(query).fetchall()
+    groups = ibis_con.table("groups").sql(query.replace(";", ""))
+
+    df = (stats
+        .inner_join(groups, "row_n")
+        .select(~s.contains("_right"))
+        .rename(area = "area_square_meters")
+        .group_by(_[column])
+        .aggregate(percent_protected =  100 * _.area.sum() / us_lower_48_area_m2,
+                hectares =  _.area.sum() / 10000,
+                n = _.area.count(),
+                richness = (_.richness * _.area).sum() / _.area.sum(),
+                rsr = (_.rsr * _.area).sum() / _.area.sum(),
+                carbon_lost = (_.deforest_carbon * _.area).sum() / _.area.sum(),
+                crop_expansion = (_.crop_expansion * _.area).sum() / _.area.sum(),
+                human_impact =  (_.human_impact * _.area).sum() / _.area.sum(),
+                )
+        .mutate(percent_protected = _.percent_protected.round(1))
+        )
+    return df.to_pandas()
+
+def area_plot(df, column):
+    base = alt.Chart(df).encode(
+           alt.Theta("percent_protected:Q").stack(True),
+           alt.Color(column+":N").legend(None)
+    )
+
+    pie = base.mark_arc(innerRadius= 40, outerRadius=80)
+    text = base.mark_text(radius=120, size=20).encode(
+    text="percent_protected:Q"
+)
+    plot = pie + text
+    return st.altair_chart(plot)
+
+def bar_chart(df, x, y):
+    chart = alt.Chart(df).mark_bar().encode(
+        x=x,
+        y=y,
+        color=alt.Color(x).legend(None)
+    ).properties(width="container", height=300)
+    return chart
+
+
+## ChatGPT Connection
+from langchain_openai import ChatOpenAI 
+from langchain_community.llms import Ollama
+# from langchain_community.llms import ChatOllama
+
+models = {"chatgpt3.5": ChatOpenAI(model="gpt-3.5-turbo", temperature=0, api_key=st.secrets["OPENAI_API_KEY"])}
+
+other_models = {
+          "chatgpt4": ChatOpenAI(model="gpt-4", temperature=0, api_key=st.secrets["OPENAI_API_KEY"]),
+          "duckdb-nsql": Ollama(model="duckdb-nsql", temperature=0),
+          "command-r-plus": Ollama(model="command-r-plus", temperature=0),
+          "mixtral:8x22b":  Ollama(model="mixtral:8x22b", temperature=0),
+          "wizardlm2:8x22b":  Ollama(model="wizardlm2:8x22b", temperature=0),
+          "sqlcoder": Ollama(model="sqlcoder", temperature=0),
+          "zephyr": Ollama(model="zephyr", temperature=0),
+          "gemma:7b": Ollama(model="gemma:7b", temperature=0),
+          "codegemma": Ollama(model="codegemma", temperature=0),
+          "llama2": Ollama(model="llama2", temperature=0),
+         }
+
+with st.sidebar:
+    choice = st.radio("Select an LLM:", models)
+    llm = models[choice]
+    column = st.text_input("grouping column", "labels")
+
+
+## A SQL Chain
+from langchain.chains import create_sql_query_chain
+chain = create_sql_query_chain(llm, db)
+
+
+main = st.container()
+
+## Does not preserve history
+with main:
+
+    '''
+    ##### Example Queries returning summary tables
+
+    - gap 1, 2, 3 are labelled 'conserved lands' and gap 4 is labeled 'other'
+    - exclude gap 4, include only Federal manager types, labelled by manager_name
+    - label gap 1, 2 as "permanently protected", label gap 3 as "additional conserved area", and gap 4 as other
+
+
+
+    '''
+    
+    prefix = "construct a select query that creates a column called 'labels' that only contains rows that meet the following criteria:"
+    suffix = ". Do not use LIMIT. Always return all columns. Do not try to select specific columns."
+
+    st.markdown("Specify how data should be labelled, as in the examples above:")
+    chatbox = st.container()
+    with chatbox:           
+        if prompt := st.chat_input(key="chain"):
+            st.chat_message("user").write(prompt)
+            with st.chat_message("assistant"):
+                response = chain.invoke({"question": prefix + prompt + suffix})
+                st.write(response)
+                df = summary_table(stats, response, column)
+                
+                with st.container():
+                    col1, col2, col3 = st.columns(3)
+                    with col1:
+                        total_percent = df.percent_protected.sum().round(1)
+                        f"{total_percent}% Continental US Covered"
+                        area_plot(df, column)
+
+                    with col2:
+                        "Species Richness"
+                        st.altair_chart(bar_chart(df, column, "richness"), use_container_width=True)
+
+                    with col3:
+                        "Range-Size Rarity"
+                        st.altair_chart(bar_chart(df, column, "rsr"), use_container_width=True)
+
+                with st.container():
+                    col1b, col2b, col3b = st.columns(3)
+                    with col1b:
+                        "Carbon Lost ('02-'22)"
+                        st.altair_chart(bar_chart(df, column, "carbon_lost"), use_container_width=True)
+
+                    with col2b:
+                        "Crop expansion"
+                        st.altair_chart(bar_chart(df, column, "crop_expansion"), use_container_width=True)
+
+                    with col3b:
+                        "Human Impact"
+                        st.altair_chart(bar_chart(df, column, "human_impact"), use_container_width=True)
+                        
+
+                    st.divider()
+                    st.dataframe(df)
+
+
+
+
+
+                
+                
+#st.divider()
+#with st.container():
+#    st.text("Database schema (top 3 rows)")
+#    tbl = tbl = query_database("select * from groups limit 3")
+#    st.dataframe(tbl)
+
+
+st.divider()
+
+'''
+Experimental prototype.  
+
+- Author: [Carl Boettiger](https://carlboettiger.info)
+- For data sources and processing, see: https://beta.source.coop/repositories/cboettig/pad-us-3/description/
+
+
+'''