WIP

app.py

@@ -11,7 +11,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# +
 import leafmap.foliumap as leafmap
 import streamlit as st
 import altair as alt
@@ -19,73 +18,62 @@ import ibis
 from ibis import _
 import ibis.selectors as s
 
-# defaults
+# defaults, consider user palette via st.color_picker()
 private_color = "#DE881E" # orange #"#850101" # red
 tribal_color = "#BF40BF" # purple
 mixed_color = "#005a00" # green
 public_color = "#3388ff" # blue
 
+# default color breaks, consider tool via st.slider()
 low = 2
 high = 3
 alpha = .5
 style_choice = "Manager Type"
+us_lower_48_area_m2 = 7.8e+12
 
 
-# +
 st.set_page_config(layout="wide", page_title="Protected Areas Explorer", page_icon=":globe:")
 
 '''
 # US Protected Area Database Explorer
 
 '''
-# -
-
-pad_pmtiles = "https://data.source.coop/cboettig/pad-us-3/pad-mobi.pmtiles"
-
-# +
-#parquet = "https://data.source.coop/cboettig/pad-us-3/pad-mobi.parquet"
-parquet = "https://minio.carlboettiger.info/public-biodiversity/pad-us-3/pad-mobi.parquet"
-
-@st.cache_resource
-def ibis_connection(parquet):
-    return ibis.read_parquet(parquet)
-
-pad_data = ibis_connection(parquet)
 
+#pad_pmtiles = "https://data.source.coop/cboettig/pad-us-3/pad-stats.pmtiles"
+#parquet = "https://data.source.coop/cboettig/pad-us-3/pad-stats.parquet"
+pad_pmtiles = "https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/pad-stats.pmtiles"
+parquet = "https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/pad-stats.parquet"
 
-# +
-@st.cache_data
-def pad_stats(_pad_data):
-    return (_pad_data
-             .aggregate(min_richness = _.richness.min(),
-                        max_richness = _.richness.max(),
-                        mean_richness = _.richness.mean(),
-                        sd_richness = _.richness.std(),
-                        min_rsr = _.rsr.min(),
-                        max_rsr = _.rsr.max(),
-                        mean_rsr = _.rsr.mean(),
-                        sd_rsr = _.rsr.std())
-             .to_pandas()
-            )
-    
-stats = pad_stats(pad_data)
-upper_rsr = stats["mean_rsr"][0] + stats["sd_rsr"][0]
-upper_richness = stats["mean_richness"][0] + stats["sd_richness"][0]
-# -
-
-m = leafmap.Map(center=[35, -100], zoom=4, layers_control=True, fullscreen_control=True)
 
-metadata = leafmap.pmtiles_metadata(pad_pmtiles)
-#print(f"layer names: {metadata['layer_names']}")
+m = leafmap.Map(center=[35, -100], zoom=4, layers_control=True)
 
-# +
 custom_style = '''
 "blue"
 '''
 
 
+sample_q = '''(
+ibis.read_parquet('https://minio.carlboettiger.info/public-biodiversity/pad-us-3/pad-mobi.parquet').
+group_by(_.bucket).
+aggregate(percent_protected =  100 * _.area.sum() / us_lower_48_area_m2,
+        mean_richness = (_.richness * _.area).sum() / _.area.sum(),
+        mean_rsr = (_.rsr * _.area).sum() / _.area.sum()
+        ).
+mutate(percent_protected = _.percent_protected.round())
+)
+'''
+
+def bar_chart(df, x, y):
+    chart = alt.Chart(df).mark_bar().encode(
+        x=x,
+        y=y,
+        color=alt.Color('color').scale(None)
+    ).properties(width="container", height=200)
+    return chart
+
+
 manager = {
-            'property': 'bucket',
+            'property': 'manager_group',
             'type': 'categorical',
             'stops': [
                 ['public', public_color],
@@ -95,7 +83,7 @@ manager = {
             ]
             }
 easement = {
-            'property': 'FeatClass',
+            'property': 'category',
             'type': 'categorical',
             'stops': [
                 ['Fee', public_color],
@@ -104,35 +92,43 @@ easement = {
             ]
             }
 
+access = {
+    'property': 'public_access',
+    'type': 'categorical',
+    'stops': [
+        ['Open Access', public_color],
+        ['Closed', private_color],
+        ['Unknown', "grey"],
+        ['Restricted Access', tribal_color]
+    ]
+}
+
 gap = {
-        'property': 'GAP_Sts',
+        'property': 'gap_code',
         'type': 'categorical',
         'stops': [
-            ['1', "#26633d"],
-            ['2', "#879647"],
-            ['3', "#BBBBBB"],
-            ['4', "#F8F8F8"]
+            [1, "#26633d"],
+            [2, "#879647"],
+            [3, "#BBBBBB"],
+            [4, "#F8F8F8"]
         ]
         }
 
 iucn = {
-            'property': 'IUCN_Cat',
+            'property': 'iucn_category',
             'type': 'categorical',
             'stops': [
-                       ["Ia", "#4B0082"],
-                       ["Ib", "#663399"],
-                       ["II", "#7B68EE"],
-                       ["III", "#9370DB"],
-                       ["IV", "#8A2BE2"],
-                       ["V", "#9932CC"],
-                       ["VI", "#9400D3"],
-                       ["Other Conservation Area", "#DDA0DD"],
-                       ["Unassigned", "#F8F8F8"]
+                ["Ia: Strict nature reserves", "#4B0082"],
+                ["Ib: Wilderness areas", "#663399"],
+                ["II: National park", "#7B68EE"],
+                ["III: Natural monument or feature", "#9370DB"],
+                ["IV: Habitat / species management", "#8A2BE2"],
+                ["V: Protected landscape / seascape", "#9932CC"],
+                ["VI: Protected area with sustainable use of natural resources", "#9400D3"],
+                ["Other Conservation Area", "#DDA0DD"],
+                ["Unassigned", "#F8F8F8"],
             ]
-            }
-
-
-
+        }
 
 thresholds = ['case',
               ['<', ['get', 'richness'], low],
@@ -156,7 +152,6 @@ rsr =  ["interpolate",
        0.006, "#850101"
        ]
 
-
 def pad_style(paint, alpha):
     return {
     "version": 8,
@@ -168,7 +163,7 @@ def pad_style(paint, alpha):
     "layers": [{
             "id": "public",
             "source": "pad",
-            "source-layer": "pad-mobi",
+            "source-layer": "pad-stats",
             "type": "fill",
             "paint": {
                 "fill-color": paint,
@@ -201,6 +196,7 @@ with st.sidebar:
                          "IUCN Status Code": iucn,
                          "Manager Type": manager,
                          "Fee/Easement": easement,
+                         "Public Access": access,
                          "Mean Richness": richness,
                          "Mean RSR": rsr,
                          "custom": eval(custom)}
@@ -244,6 +240,20 @@ with st.sidebar:
         hi="https://data.source.coop/vizzuality/hfp-100/hfp_2021_100m_v1-2_cog.tif"
         m.add_cog_layer(hi, palette="purples", name="Human Impact", transparent_bg=True, opacity = 0.8, zoom_to_layer=False)
 
+    if st.toggle("cropland expansion"):
+        m.add_cog_layer("https://data.source.coop/vizzuality/lg-land-carbon-data/natcrop_expansion_100m_cog.tif",
+                        palette="greens", name="cropland expansion", transparent_bg=True, opacity = 0.8, zoom_to_layer=False)
+
+    if st.toggle("Biodiversity Intactness Loss"):
+        m.add_cog_layer("https://data.source.coop/vizzuality/lg-land-carbon-data/natcrop_bii_100m_cog.tif",
+                        palette="reds", name="biodiversity intactness loss", transparent_bg=True, opacity = 0.8, zoom_to_layer=False)
+ 
+    if st.toggle("Forest Integrity Loss"):
+        m.add_cog_layer("https://data.source.coop/vizzuality/lg-land-carbon-data/natcrop_fii_100m_cog.tif",
+                        palette="reds", name="forest integrity loss", transparent_bg=True, opacity = 0.8, zoom_to_layer=False)
+
+
+
     if st.toggle("Custom map layers"):
         
         code = st.text_area(label = "leafmap code:",
@@ -280,162 +290,158 @@ with st.sidebar:
                     "source": "source1",
                     "source-layer": "mtbs_perims_DD",
                     "type": "fill",
-                    "paint": {"fill-color": "#FFA500", "fill-opacity": 0.2}}]}
+                    "paint": {"fill-color": "#FFA500", "fill-opacity": 0.4}}]}
         m.add_pmtiles(usgs, name="Fire", style=combined_style, overlay=True, show=True, zoom_to_layer=False)
 
-# +
-# And here we go!
-m.to_streamlit(height=700)
-# -
-
-
+# Map radio buttons to corresponding column:
 select_column = {
-                "GAP Status Code": "GAP_Sts",
-                "IUCN Status Code": "IUCN_Cat",
-                "Manager Type": "bucket",
-                "Fee/Easement": "FeatClass",
-                "Mean Richness": "bucket",
-                "Mean RSR": "bucket",
-                 "custom": "bucket"}
+                "GAP Status Code": "gap_code",
+                "IUCN Status Code": "iucn_category",
+                "Manager Type": "manager_group",
+                "Fee/Easement": "category",
+                "Public Access": "public_access",
+                "Mean Richness": "manager_group",
+                "Mean RSR": "manager_group",
+                 "custom": "gap_code"}
 column = select_column[style_choice]
 
+# Map radio buttons to corresponding color-scheme:
 select_colors = {
                 "GAP Status Code": gap["stops"],
                 "IUCN Status Code": iucn["stops"],
                 "Manager Type": manager["stops"],
                 "Fee/Easement": easement["stops"],
+                "Public Access": access["stops"],
                 "Mean Richness": manager["stops"],
                 "Mean RSR": manager["stops"],
-                 "custom": manager["stops"]}
-colors = ibis.memtable(select_colors[style_choice], columns = [column, "color"]).to_pandas()
-
-st.divider()
+                "custom": manager["stops"]}
+colors = (ibis
+          .memtable(select_colors[style_choice], columns = [column, "color"])
+          .to_pandas()
+         )
+
+
+main = st.container()
+
+with main:
+    map_col, stats_col = st.columns([2,1])
+
+    with map_col:
+        m.to_streamlit(height=700)
+
+
+    @st.cache_resource
+    def ibis_connection(parquet):
+        return ibis.read_parquet(parquet)
+    pad_data = ibis_connection(parquet)
+
+    @st.cache_data()
+    def summary_table(column = column, colors = colors):
+        df = (pad_data
+            .rename(area = "area_square_meters")
+            .group_by(_[column])
+            .aggregate(percent_protected =  100 * _.area.sum() / us_lower_48_area_m2,
+                        mean_richness = (_.richness * _.area).sum() / _.area.sum(),
+                        mean_rsr = (_.rsr * _.area).sum() / _.area.sum(),
+                        carbon_lost = (_.deforest_carbon * _.area).sum() / _.area.sum(),
+                        crop_expansion = (_.crop_expansion * _.area).sum() / _.area.sum(),
+                        human_impact =  (_.human_impact * _.area).sum() / _.area.sum(),
+                        )
+            .mutate(percent_protected = _.percent_protected.round())
+            .inner_join(colors, column)
+            )
+        df = df.to_pandas()
+        df[column] = df[column].astype(str)
+        return df
 
+    df = summary_table(column, colors)
+    total_percent = df.percent_protected.sum()
 
-# +
-us_lower_48_area_m2 = 7.8e+12
 
-@st.cache_data()
-def summary_table(column = column, colors = colors):
-    df = (pad_data.
-        group_by(_[column]).
-        aggregate(percent_protected =  100 * _.area.sum() / us_lower_48_area_m2,
-                mean_richness = (_.richness * _.area).sum() / _.area.sum(),
-                mean_rsr = (_.rsr * _.area).sum() / _.area.sum()
-               ).
-        mutate(percent_protected = _.percent_protected.round())
-        ).inner_join(colors, column)
-    return df.to_pandas()
-
-df = summary_table(column, colors)
-total_percent = df.percent_protected.sum()
-
-
-base = alt.Chart(df).encode(
-    alt.Theta("percent_protected:Q").stack(True),
-    alt.Color("color:N").scale(None).legend(None)
-)
+    base = alt.Chart(df).encode(
+        alt.Theta("percent_protected:Q").stack(True),
+        alt.Color("color:N").scale(None).legend(None)
+    )
 
-area_chart = (
-  base.mark_arc(innerRadius=50, outerRadius=120) +
-  base.mark_text(radius=165, size=20).encode(text=column) +
-  base.mark_text(radius=135, size=20).encode(text="percent_protected:N")
-)
+    area_chart = (
+    base.mark_arc(innerRadius=40, outerRadius=70)
+    ).properties(width=180, height=180)
 
-# area_chart
 
-# +
-richness_chart = alt.Chart(df).mark_bar().encode(
-    x=column,
-    y='mean_richness',
-    color=alt.Color('color').scale(None)
-)
+    richness_chart = bar_chart(df, column, 'mean_richness')
+    rsr_chart = bar_chart(df, column, 'mean_rsr')
+    carbon_lost = bar_chart(df, column, 'carbon_lost')
+    crop_expansion = bar_chart(df, column, 'crop_expansion')
+    human_impact = bar_chart(df, column, 'human_impact')
 
+    with stats_col:
+        col1, col2, col3 = st.columns(3)
+        with col1:
+            f"{total_percent}% Continental US Covered"
+            st.altair_chart(area_chart, use_container_width=False)
+            "Carbon Lost ('02-'22)"
+            st.altair_chart(carbon_lost, use_container_width=True)
 
-# +
-rsr_chart = alt.Chart(df).mark_bar().encode(
-    x=column,
-    y='mean_rsr',
-    color=alt.Color('color').scale(None)
-)
-# -
+        with col2:
+            "Species Richness"
+            st.altair_chart(richness_chart, use_container_width=True)
+            "Crop expansion"
+            st.altair_chart(crop_expansion, use_container_width=True)
 
-'''
-## Summary Statistics
+        with col3:
+            "Range-Size Rarity"
+            st.altair_chart(rsr_chart, use_container_width=True)
+            "Human Impact"
+            st.altair_chart(human_impact, use_container_width=True)
 
-'''
 
-# +
-col1, col2, col3 = st.columns(3)
+st.divider()
 
-with col1:
-    f"#### {total_percent} Percent of Continental US Area Covered"
-    st.altair_chart(area_chart, use_container_width=True)
-    
-# -
+footer = st.container()
 
-with col2: 
-    "#### Mean Species Richness"
-    st.altair_chart(richness_chart, use_container_width=True)
 
+with footer:
+    '''
+    ## Custom queries
 
-with col3: 
-    "#### Mean Range-Size Rarity"
-    st.altair_chart(rsr_chart, use_container_width=True)
+    Input custom python code below to interactively explore the data. 
 
-# +
-'''
-## Custom queries
+    '''
 
-Input custom python code below to interactively explore the data. 
+    col2_1, col2_2 = st.columns(2)
 
-'''
 
-col2_1, col2_2 = st.columns(2)
 
+    with col2_1:
+        query = st.text_area(
+            label = "Python code:",
+            value = sample_q, 
+            height = 300)
 
-sample_q = '''(
-    ibis.read_parquet('https://minio.carlboettiger.info/public-biodiversity/pad-us-3/pad-mobi.parquet').
-    group_by(_.bucket).
-    aggregate(percent_protected =  100 * _.area.sum() / us_lower_48_area_m2,
-              mean_richness = (_.richness * _.area).sum() / _.area.sum(),
-              mean_rsr = (_.rsr * _.area).sum() / _.area.sum()
-              ).
-    mutate(percent_protected = _.percent_protected.round())
-)
-'''
-
-with col2_1:
-    query = st.text_area(
-        label = "Python code:",
-        value = sample_q, 
-        height = 300)
+    with col2_2:
+        "Output table:"
+        df = eval(query)
+        st.write(df.to_pandas())
 
-with col2_2:
-    "Output table:"
-    df = eval(query)
-    st.write(df.to_pandas())
+    st.divider()
 
+    '''
+    ## Credits
 
-# +
+    Author: Carl Boettiger, UC Berkeley
+    License: BSD-2-clause
 
-'''
-## Credits
+    ### Data sources
 
-Author: Carl Boettiger, UC Berkeley
-License: BSD-2-clause
+    - US Protected Areas Database v3 by USGS, data hosted on https://beta.source.coop/cboettig/us-pad-3. Citation: https://doi.org/10.5066/P9Q9LQ4B, License: Public Domain
+    - Carbon-loss by Vizzuality, on https://beta.source.coop/repositories/vizzuality/lg-land-carbon-data. Citation: https://doi.org/10.1101/2023.11.01.565036, License: CC-BY
+    - Human Footprint by Vizzuality, on https://beta.source.coop/repositories/vizzuality/hfp-100.  Citation: https://doi.org/10.3389/frsen.2023.1130896, License: Public Domain
+    - Fire polygons by USGS, reprocessed to PMTiles on https://beta.source.coop/cboettig/fire/. License: Public Domain.
+    - Irrecoverable Carbon from Conservation International, reprocessed to COG on https://beta.source.coop/cboettig/carbon, citation: https://doi.org/10.1038/s41893-021-00803-6, License: CC-BY-NC
 
-### Data sources
+    ### Software
 
-- US Protected Areas Database v3 by USGS, data hosted on https://beta.source.coop/cboettig/us-pad-3. Citation: https://doi.org/10.5066/P9Q9LQ4B, License: Public Domain
-- Carbon-loss by Vizzuality, on https://beta.source.coop/repositories/vizzuality/lg-land-carbon-data. Citation: https://doi.org/10.1101/2023.11.01.565036, License: CC-BY
-- Human Footprint by Vizzuality, on https://beta.source.coop/repositories/vizzuality/hfp-100.  Citation: https://doi.org/10.3389/frsen.2023.1130896, License: Public Domain
-- Fire polygons by USGS, reprocessed to PMTiles on https://beta.source.coop/cboettig/fire/. License: Public Domain.
-- Irrecoverable Carbon from Conservation International, reprocessed to COG on https://beta.source.coop/cboettig/carbon, citation: https://doi.org/10.1038/s41893-021-00803-6, License: CC-BY-NC
-
-### Software
-
-Proudly built with a free and Open Source software stack: Streamlit (reactive application), HuggingFace (application hosting), Source.Coop (data hosting),
-using cloud-native data serializations in COG, PMTiles, and GeoParquet. Coded in pure python using leafmap and duckdb. Map styling with [MapLibre](https://maplibre.org/).
-'''
+    Proudly built with a free and Open Source software stack: Streamlit (reactive application), HuggingFace (application hosting), Source.Coop (data hosting),
+    using cloud-native data serializations in COG, PMTiles, and GeoParquet. Coded in pure python using leafmap and duckdb. Map styling with [MapLibre](https://maplibre.org/).
+    '''
 

pad-stats.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9585c3c27d05039ff30faae0c9f5244a1a061e722b47ba30b2e45c4f51df8dc
+size 882042199

pad.duckdb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06f01cd8a18be80c44cea7b5cb7f731a50cbc67c6285ddb80c8ae195adf7a770
+size 1079783424

preprocess.py

@@ -1,21 +1,29 @@
-# +
 import ibis
+import ibis.selectors as s
 from ibis import _
-import xarray
-from shapely.geometry import box
-from geocube.api.core import make_geocube
-import geopandas
-import fiona
 
-import multiprocessing.popen_spawn_posix
-from dask.distributed import Client, LocalCluster, Lock
-import rioxarray
+# +
+fgb = "https://data.source.coop/cboettig/pad-us-3/pad-us3-combined.fgb"
+parquet = "https://data.source.coop/cboettig/pad-us-3/pad-us3-combined.parquet"
 
+#pad = ibis.read_parquet(parquet)
+# Currently ibis doesn't detect that this is GeoParquet.  We need a SQL escape-hatch to cast the geometry
+con = ibis.duckdb.connect()
+con.load_extension("spatial")
+con.raw_sql(f"CREATE OR REPLACE VIEW pad AS SELECT *, st_geomfromwkb(geometry) as geom from read_parquet('{parquet}')")
+pad = con.table("pad")
 
 
 # +
-fgb = "https://data.source.coop/cboettig/pad-us-3/pad-us3-combined.fgb"
-parquet = "https://data.source.coop/cboettig/pad-us-3/pad-us3-combined.parquet"
+#pad.filter(_.Category == "Easement").select("EHoldTyp", "Mang_Type", "Unit_Nm").distinct().head(100).to_pandas()
+# pad.filter(_.Category == "Easement").select("EsmtHldr", "Mang_Name", "Unit_Nm").distinct().sample(.1).to_pandas()
+#pad.select("Comments").distinct().head(100).to_pandas()
+
+# +
+import fiona
+import rioxarray
+from shapely.geometry import box
+
 cog = "https://data.source.coop/cboettig/mobi/species-richness-all/SpeciesRichness_All.tif"
 
 # fiona not built with parquet support.  ideally duckdb's st_read_meta would do this.
@@ -27,24 +35,13 @@ nrow = len(meta)
 r = rioxarray.open_rasterio(cog)
 bounds = box(*r.rio.transform_bounds(crs))
 
-# +
-#import leafmap
-#leafmap.cog_validate(cog)
-# -
-
-con = ibis.duckdb.connect()
-# We could just read the flatgeobuf with ibis.read_geo() but it is not as fast as working with the (Geo)Parquet
-# pad = con.read_geo(fgb)
-
-# Currently ibis doesn't detect that this is GeoParquet.  We need a SQL escape-hatch to cast the geometry
-con.load_extension("spatial")
-con.raw_sql(f"CREATE OR REPLACE VIEW pad AS SELECT *, st_geomfromwkb(geometry) as geom from read_parquet('{parquet}')")
-pad = con.table("pad")
-
 # +
 # Now we can do all the usual SQL queries to subset the data.  Note the `geom.within()` spatial filter!
-focal_columns = ["bucket", "FeatClass", "Mang_Name",  "Mang_Type",  "Des_Tp",
-                 "Pub_Access",   "GAP_Sts",  "IUCN_Cat",   "Unit_Nm",  "geom"]
+focal_columns = ["bucket", "FeatClass", "Mang_Name", 
+                 "Mang_Type",  "Des_Tp", "Pub_Access",
+                 "GAP_Sts",  "IUCN_Cat",   "Unit_Nm",  
+                 "State_Nm", "EsmtHldr", "Date_Est",
+                 "SHAPE_Area", "geom"]
 public = ["DIST", "LOC", "FED", "STAT", "JNT"]
 
 case = (
@@ -56,171 +53,187 @@ case = (
     .end()
 )
 
-pad_labeled = (
-    pad.
-    filter((_.FeatClass.isin(["Easement", "Fee"])) | (
+pad_parquet = (
+    pad
+    .filter((_.FeatClass.isin(["Easement", "Fee"])) | (
            (_.FeatClass == "Proclamation") & (_.Mang_Name == "TRIB"))
-          ).
-    filter(_.Mang_Type.notin(["UNK", "TERR"])).
-    filter(_.geom.within(bounds)).
-    mutate(GAP_Sts = _.GAP_Sts.cast("int")).
-    mutate(bucket = case).
-    select(focal_columns).
-    mutate(row_n=ibis.row_number())
+           )
+    .filter(_.Mang_Type.notin(["UNK", "TERR"]))
+    .filter(_.geom.within(bounds))
+    .mutate(GAP_Sts = _.GAP_Sts.cast("int"))
+    .mutate(bucket = case)
+    .mutate(row_n=ibis.row_number())
+    .select(focal_columns)
+    .rename(geometry="geom")
 )
 
+#pad_parquet.to_parquet("pad-processed.parquet")
+# -
+
+agency_name = con.read_parquet("/home/rstudio/huggingface/datasets/pad-us-3/parquet/pad-agency-name.parquet").select(manager_name_id = "Code", manager_name = "Dom")
+agency_type = con.read_parquet("/home/rstudio/huggingface/datasets/pad-us-3/parquet/pad-agency-type.parquet").select(manager_type_id = "Code", manager_type = "Dom")
+desig_type = con.read_parquet("/home/rstudio/huggingface/datasets/pad-us-3/parquet/pad-desgination-type.parquet").select(designation_type_id = "Code", designation_type = "Dom")
+public_access = con.read_parquet("/home/rstudio/huggingface/datasets/pad-us-3/parquet/pad-public-access.parquet").select(public_access_id = "Code", public_access = "Dom")
+state_name = con.read_parquet("/home/rstudio/huggingface/datasets/pad-us-3/parquet/pad-state-name.parquet").select(state_id = "Code", state = "Dom")
+iucn = con.read_parquet("/home/rstudio/huggingface/datasets/pad-us-3/parquet/pad-iucn.parquet").select(iucn_id = "CODE", iucn_category = "DOM")
+
+pad_processed = (pad_parquet
+    .rename(manager_name_id = "Mang_Name", 
+            manager_type_id = "Mang_Type", 
+            manager_group="bucket",
+            designation_type_id = "Des_Tp",
+            public_access_id = "Pub_Access",
+            category = "FeatClass",
+            iucn_id = "IUCN_Cat",
+            gap_code = "GAP_Sts",
+            state_id = "State_Nm",
+            easement_holder = "EsmtHldr",
+            date_established = "Date_Est",
+            area_square_meters = "SHAPE_Area",
+            name = "Unit_Nm")
+    .left_join(agency_name, "manager_name_id")
+    .left_join(agency_type, "manager_type_id")
+    .left_join(desig_type, "designation_type_id")
+    .left_join(public_access, "public_access_id")
+    .left_join(state_name, "state_id")
+    .left_join(iucn, "iucn_id")
+    .select(~s.contains("_right"))
+    .select(~s.contains("_id"))
+)
+# pad_processed.to_parquet("pad-processed.parquet")
 
 # +
-# # %%time
-# smoke test -- only the lower 48 states!
-# (pad.filter(_.geom.within(bounds)).group_by([_.State_Nm]).aggregate(n = _.count()).to_pandas()) 
-# -
+# if we keep the original geoparquet WKB 'geometry' column, to_pandas() (or execute) gives us only a normal pandas data.frame, and geopandas doesn't see the metadata.
+# if we replace the geometry with duckdb-native 'geometry' type, to_pandas() gives us a geopanadas!  But requires reading into RAM.  
+import geopandas as gpd
+
+gdf = pad_processed.to_pandas()
+gdf = gdf.set_crs(crs)
+gdf.to_parquet("pad-processed.parquet")
 
-# Or be bold!
-df = pad_labeled.to_pandas()
-geo = geopandas.GeoDataFrame(df, geometry=df.geometry, crs=crs)
-geo.to_parquet("pad-filtered.parquet")
-
-
-def zonal_stats(cog, geo, band_name = "mean", row_n = "row_n"):
-# https://corteva.github.io/geocube/html/examples/zonal_statistics.html
-    raster = (rioxarray.
-         open_rasterio('/vsicurl/'+cog, masked=True, chunks=True, lock=False).
-         rio.clip_box(*geo.total_bounds, crs=geo.crs).
-         rio.clip(geo.geometry.values, crs=geo.crs, from_disk=True).
-         sel(band=1).drop_vars("band")
-    )
-    out_grid = make_geocube(
-        vector_data=geo,
-        measurements=[row_n],
-        like=raster, # ensure the data are on the same grid
-    )
-    # merge the two together
-    out_grid["values"] = (raster.dims, raster.values, raster.attrs, raster.encoding)
-    grouped_raster = out_grid.drop_vars("spatial_ref").groupby(out_grid.row_n)
-    # can add other stats
-    grid_mean = grouped_raster.mean().rename({"values": band_name})
-    zonal_stats = xarray.merge([grid_mean]).to_dataframe()
-    geo = geo.merge(zonal_stats, how="left", on=row_n)
-    return geo
-
-
-import numpy as np
-# consider doing multiple cogs per slice
-def piecewise_zonal2(cog, geo, band_name = "mean", dirname = "pad_parquet",  n = 10000, row_n = "row_n"):
-    total = len(geo)
-    for i in range(0,total,n):
-        k = i // n
-        path = f"{dirname}/part_{k}.parquet"
-        print(f"processing {path}")
-        end = np.min([i + n,total])
-        geo_slice = geo.iloc[i:end]
-        geo_slice = zonal_stats(cog, geo_slice, band_name, row_n)
-        geo_slice.to_parquet(path)
+# +
+import rasterio
+from rasterstats import zonal_stats
+import geopandas as gpd
+import pandas as pd
+from joblib import Parallel, delayed
 
+def big_zonal_stats(vec_file, tif_file, stats, col_name, n_jobs, verbose = 10, timeout=1000):
 
-# %%time
-piecewise_zonal2(cog, geo, "richness", dirname = "pad_mobi", n = 50000) # 6 min
+    # read in vector as geopandas, match CRS to raster
+    with rasterio.open(tif_file) as src:
+        raster_profile = src.profile
+    gdf = gpd.read_parquet(vec_file).to_crs(raster_profile['crs'])
 
+    gdf["row_n"] = gdf.index + 1
 
-# # Manual approach
+    # lamba fn to zonal_stats a slice:
+    def get_stats(geom_slice, tif_file, stats):
+        stats = zonal_stats(geom_slice.geometry, tif_file, stats = stats)
+        stats[0]['row_n'] = geom_slice.row_n
+        return stats[0]
+    
+    # iteratation (could be a list comprehension?)
+    jobs = []
+    for r in gdf.itertuples():
+        jobs.append(delayed(get_stats)(r, tif_file, stats))
+
+    # And here we go
+    output = Parallel(n_jobs=n_jobs, timeout=timeout, verbose=verbose)(jobs)
+
+    # reshape output
+    df_zonal_stats = (
+        pd.DataFrame(output)
+        .rename(columns={'mean': col_name})
+        .merge(gdf, how='right', on = 'row_n')
+        )
+    return df_zonal_stats
 
-# +
-import geopandas
-import multiprocessing.popen_spawn_posix
-from dask.distributed import Client, LocalCluster, Lock
-import rioxarray
 
-geo = geopandas.read_parquet("pad_mobi") # ~ 4.8 GB RAM
 
 # +
 # %%time
-band_name = "human_impact"
-row_n = "row_n"
-cog = "https://data.source.coop/vizzuality/hfp-100/hfp_2021_100m_v1-2_cog.tif"
 
-raster = (rioxarray.
-          open_rasterio('/vsicurl/'+cog, masked=True, chunks=True, lock=False).
-          rio.clip_box(*geo.total_bounds, crs=geo.crs).
-          rio.clip(geo.geometry.values, geo.crs, from_disk=True).
-          sel(band=1).drop_vars("band")
-         )
+tif_file = './hfp_2021_100m_v1-2_cog.tif'
+vec_file = './pad-processed.parquet'
+
+df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],
+                     col_name = "human_impact", n_jobs=-1, verbose=0)
+gpd.GeoDataFrame(df, geometry="geometry").to_parquet("pad-stats.parquet")
 
 # +
 # %%time
 
-band_name = "human_impact"
-row_n = "row_n"
-cog = "https://data.source.coop/vizzuality/hfp-100/hfp_2021_100m_v1-2_cog.tif"
+tif_file = '/home/rstudio/source.coop/cboettig/mobi/species-richness-all/SpeciesRichness_All.tif'
+vec_file = './pad-stats.parquet'
 
-
-with LocalCluster() as cluster, Client(cluster) as client:
-    raster = (rioxarray.
-              open_rasterio('/vsicurl/'+cog, masked=True, chunks=True, lock=False).
-              rio.clip(geo.geometry.values, geo.crs, from_disk=True).
-              sel(band=1).drop_vars("band")
-             )
+df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],
+                     col_name = "richness", n_jobs=-1, verbose=0)
+gpd.GeoDataFrame(df, geometry="geometry").to_parquet("pad-stats.parquet")
 
 # +
 # %%time
 
-out_grid = make_geocube(
-    vector_data=geo,
-    measurements=['row_n'],
-    like=raster, # ensure the data are on the same grid
-)
-# ~ +1 Gb, 1.2s
+tif_file = '/home/rstudio/source.coop/cboettig/mobi/range-size-rarity-all/RSR_All.tif'
+vec_file = './pad-stats.parquet'
+
+df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],
+                      col_name = "rsr", n_jobs=-1, verbose=0)
+gpd.GeoDataFrame(df, geometry="geometry").to_parquet("pad-stats.parquet")
 
 # +
 # %%time
-# 100 ~ 30s, 1000 ~ 30s
 
-out_grid["values"] = (raster.dims, raster.values, raster.attrs, raster.encoding)
-grouped_raster = out_grid.drop_vars("spatial_ref").groupby(out_grid.row_n)  # ~ +3 Gb
+tif_file = '/home/rstudio/source.coop/vizzuality/lg-land-carbon-data/deforest_carbon_100m_cog.tif'
+vec_file = './pad-stats.parquet'
+
+df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],  col_name = "deforest_carbon", n_jobs=-1, verbose=0)
+gpd.GeoDataFrame(df, geometry="geometry").to_parquet("pad-stats.parquet")
 
 # +
 # %%time
-grid_mean = grouped_raster.mean().rename({"values": band_name})
-zonal_stats = xarray.merge([grid_mean]).to_dataframe()
-geo = geo.merge(zonal_stats, how="left", on=row_n)
-geo.to_parquet("test.parquet")
-len(geo)
 
-# 1.2 s
-# -
+tif_file = '/home/rstudio/source.coop/vizzuality/lg-land-carbon-data/natcrop_bii_100m_cog.tif'
+vec_file = './pad-stats.parquet'
 
+df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],  col_name = "biodiversity_intactness", n_jobs=-1, verbose=0)
+gpd.GeoDataFrame(df, geometry="geometry").to_parquet("pad-stats.parquet")
 
+# +
+# %%time
 
-# Lastly we need to convert to PMTiles:
-#
-# ```
-# ogr2ogr  -dsco MAX_SIZE=90000000 -dsco MAX_FEATURES=50000000 -dsco MAXZOOM=10 pad-mobi.pmtiles pad-mobi.parquet
-# ```
+tif_file = '/home/rstudio/source.coop/vizzuality/lg-land-carbon-data/natcrop_fii_100m_cog.tif'
+vec_file = './pad-stats.parquet'
 
-geo.plot(column="richness", legend=True)
+df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],  col_name = "forest_integrity", n_jobs=-1, verbose=0)
+gpd.GeoDataFrame(df, geometry="geometry").to_parquet("pad-stats.parquet")
 
-import geopandas
-gdf = geopandas.read_parquet("pad-mobi.parquet")
-cog = "https://data.source.coop/cboettig/mobi/range-size-rarity-all/RSR_All.tif"
+# +
+# %%time
+
+tif_file = '/home/rstudio/source.coop/vizzuality/lg-land-carbon-data/natcrop_expansion_100m_cog.tif'
+vec_file = './pad-stats.parquet'
 
+df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],  col_name = "crop_expansion", n_jobs=-1, verbose=0)
+gpd.GeoDataFrame(df, geometry="geometry").to_parquet("pad-stats.parquet")
 
-human_impacts_2021 = "https://data.source.coop/vizzuality/hfp-100/hfp_2021_100m_v1-2_cog.tif"
-geo = zonal_stats(human_impacts_2021, geo, "human_impacts_2021")
+# +
+# %%time
 
+tif_file = '/home/rstudio/source.coop/vizzuality/lg-land-carbon-data/natcrop_reduction_100m_cog.tif'
+vec_file = './pad-stats.parquet'
 
-geo.to_parquet("pad-extended.parquet")
+df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],  col_name = "crop_reduction", n_jobs=-1, verbose=0)
+gpd.GeoDataFrame(df, geometry="geometry").to_parquet("pad-stats.parquet")
 
 # +
-# %%time
-geo = zonal_stats("https://data.source.coop/vizzuality/lg-land-carbon-data/deforest_carbon_100m_cog.tif", geo, "deforest_carbon")
-geo = zonal_stats("https://data.source.coop/vizzuality/lg-land-carbon-data/natcrop_fii_100m_cog.tif", geo, "fii")
-geo = zonal_stats("https://data.source.coop/vizzuality/lg-land-carbon-data/natcrop_bii_100m_cog.tif", geo, "bii")
-geo = zonal_stats("https://data.source.coop/vizzuality/lg-land-carbon-data/natcrop_expansion_100m_cog.tif", geo, "crop_expansion")
-geo = zonal_stats("https://data.source.coop/vizzuality/lg-land-carbon-data/natcrop_reduction_100m_cog.tif", geo, "crop_reduction")
-	
-geo = zonal_stats("https://data.source.coop/cboettig/carbon/cogs/irrecoverable_c_total_2018.tif", geo, "irrecoverable_c_total_2018")
-geo = zonal_stats("https://data.source.coop/cboettig/carbon/cogs/manageable_c_total_2018.tif", geo, "manageable_c_total_2018")
-
-cog = "https://data.source.coop/cboettig/mobi/range-size-rarity-all/RSR_All.tif"
-
-geo.to_parquet("pad-extended.parquet")
+from sqlalchemy import create_engine
+from sqlalchemy import text
+db_uri = "duckdb:///pad.duckdb"
+engine = create_engine(db_uri)
+con = engine.connect()
+con.execute("create or replace table pad as select * from 'pad-stats.parquet'")
+con.close()
+
+# pad_stats = ibis.read_parquet("pad-stats.parquet")
+# pad_stats.head(20).to_pandas()