dynamic charts

app.py

@@ -109,11 +109,11 @@ gap = {
             ['1', "#26633d"],
             ['2', "#879647"],
             ['3', "#BBBBBB"],
-            ['4', "#FFFFFF"]
+            ['4', "#F8F8F8"]
         ]
         }
 
-IUCN = {
+iucn = {
             'property': 'IUCN_Cat',
             'type': 'categorical',
             'stops': [
@@ -125,7 +125,7 @@ IUCN = {
                        ["V", "#9932CC"],
                        ["VI", "#9400D3"],
                        ["Other Conservation Area", "#DDA0DD"],
-                       ["Unassigned", "#FFFFFF"]
+                       ["Unassigned", "#F8F8F8"]
             ]
             }
 
@@ -174,7 +174,8 @@ def pad_style(paint, alpha):
             }
         }]}
 
-
+manager_colors = {"bucket": ["public", "tribal", "mixed", "private"], 
+                   "color": [public_color, tribal_color, mixed_color, private_color]}
 
 # +
 ## Map controls sidebar
@@ -194,7 +195,7 @@ with st.sidebar:
 
         style_options = {
                          "GAP Status Code": gap,
-                         "IUCN Status Code": IUCN,
+                         "IUCN Status Code": iucn,
                          "Manager Type": manager,
                          "Fee/Easement": easement,
                          "Mean Richness": richness,
@@ -240,7 +241,6 @@ with st.sidebar:
         hi="https://data.source.coop/vizzuality/hfp-100/hfp_2021_100m_v1-2_cog.tif"
         m.add_cog_layer(hi, palette="purples", name="Human Impact", transparent_bg=True, opacity = 0.8, zoom_to_layer=False)
 
-    
 #    "## Boundaries"
 #    boundaries = st.radio("Boundaries:",
 #                          ["None",
@@ -253,8 +253,6 @@ with st.sidebar:
     "## Basemaps"
     if st.toggle("Shaded Relief Topo"):
         m.add_basemap("Esri.WorldShadedRelief")
-
-    
    
     "## Additional elements"
     # Fire Polygons, USGS
@@ -275,15 +273,32 @@ with st.sidebar:
                     "paint": {"fill-color": "#FFA500", "fill-opacity": 0.2}}]}
         m.add_pmtiles(usgs, name="Fire", style=combined_style, overlay=True, show=True, zoom_to_layer=False)
 
-
-
-
 # +
-
 # And here we go!
 m.to_streamlit(height=700)
 # -
 
+
+select_column = {
+                "GAP Status Code": "GAP_Sts",
+                "IUCN Status Code": "IUCN_Cat",
+                "Manager Type": "bucket",
+                "Fee/Easement": "FeatClass",
+                "Mean Richness": "bucket",
+                "Mean RSR": "bucket",
+                 "custom": "bucket"}
+column = select_column[style_choice]
+
+select_colors = {
+                "GAP Status Code": gap["stops"],
+                "IUCN Status Code": iucn["stops"],
+                "Manager Type": manager["stops"],
+                "Fee/Easement": easement["stops"],
+                "Mean Richness": manager["stops"],
+                "Mean RSR": manager["stops"],
+                 "custom": manager["stops"]}
+colors = ibis.memtable(select_colors[style_choice], columns = [column, "color"]).to_pandas()
+
 st.divider()
 
 
@@ -291,56 +306,44 @@ st.divider()
 us_lower_48_area_m2 = 7.8e+12
 
 @st.cache_data()
-def summary_table():
-    x = ibis.memtable({"bucket": ["public", "tribal", "mixed", "private"], 
-                       "color": [public_color, tribal_color, mixed_color, private_color]})
+def summary_table(column = column, colors = colors):
     df = (pad_data.
-        group_by(_.bucket).
+        group_by(_[column]).
         aggregate(percent_protected =  100 * _.area.sum() / us_lower_48_area_m2,
                 mean_richness = (_.richness * _.area).sum() / _.area.sum(),
                 mean_rsr = (_.rsr * _.area).sum() / _.area.sum()
                ).
         mutate(percent_protected = _.percent_protected.round())
-        ).inner_join(x, "bucket")
+        ).inner_join(colors, column)
     return df.to_pandas()
 
-df = summary_table()
-# st.table(richness_table)
-
-
-# +
-#summary_table.to_pandas()
+df = summary_table(column, colors)
 
-# +
 
 base = alt.Chart(df).encode(
     alt.Theta("percent_protected:Q").stack(True),
     alt.Color("color:N").scale(None).legend(None)
 )
 
-
 area_chart = (
   base.mark_arc(innerRadius=50, outerRadius=120) +
-  base.mark_text(radius=165, size=20).encode(text="bucket") +
+  base.mark_text(radius=165, size=20).encode(text=column) +
   base.mark_text(radius=135, size=20).encode(text="percent_protected:N")
 )
 
 # area_chart
 
 # +
-
 richness_chart = alt.Chart(df).mark_bar().encode(
-    x='bucket',
+    x=column,
     y='mean_richness',
     color=alt.Color('color').scale(None)
 )
-#richness_chart
 
 
 # +
-
 rsr_chart = alt.Chart(df).mark_bar().encode(
-    x='bucket',
+    x=column,
     y='mean_rsr',
     color=alt.Color('color').scale(None)
 )
@@ -355,7 +358,7 @@ rsr_chart = alt.Chart(df).mark_bar().encode(
 col1, col2, col3 = st.columns(3)
 
 with col1:
-    "#### Percent of Continental US Area"
+    f"#### Percent of Continental US Area"
     st.altair_chart(area_chart, use_container_width=True)
     
 # -

preprocess.py

@@ -1,12 +1,17 @@
+# +
 import ibis
 from ibis import _
-import rioxarray
 import xarray
 from shapely.geometry import box
 from geocube.api.core import make_geocube
 import geopandas
 import fiona
 
+import multiprocessing.popen_spawn_posix
+from dask.distributed import Client, LocalCluster, Lock
+import rioxarray
+
+
 
 # +
 fgb = "https://data.source.coop/cboettig/pad-us-3/pad-us3-combined.fgb"
@@ -65,14 +70,24 @@ pad_labeled = (
 )
 
 
+# +
+# # %%time
+# smoke test -- only the lower 48 states!
+# (pad.filter(_.geom.within(bounds)).group_by([_.State_Nm]).aggregate(n = _.count()).to_pandas()) 
 # -
 
+# Or be bold!
+df = pad_labeled.to_pandas()
+geo = geopandas.GeoDataFrame(df, geometry=df.geometry, crs=crs)
+geo.to_parquet("pad-filtered.parquet")
+
 
 def zonal_stats(cog, geo, band_name = "mean", row_n = "row_n"):
 # https://corteva.github.io/geocube/html/examples/zonal_statistics.html
     raster = (rioxarray.
-         open_rasterio('/vsicurl/'+cog, masked=True).
-         rio.clip(geo.geometry.values, geo.crs, from_disk=True).
+         open_rasterio('/vsicurl/'+cog, masked=True, chunks=True, lock=False).
+         rio.clip_box(*geo.total_bounds, crs=geo.crs).
+         rio.clip(geo.geometry.values, crs=geo.crs, from_disk=True).
          sel(band=1).drop_vars("band")
     )
     out_grid = make_geocube(
@@ -90,71 +105,89 @@ def zonal_stats(cog, geo, band_name = "mean", row_n = "row_n"):
     return geo
 
 
-total_features = pad_labeled.count().to_pandas()
+import numpy as np
+# consider doing multiple cogs per slice
+def piecewise_zonal2(cog, geo, band_name = "mean", dirname = "pad_parquet",  n = 10000, row_n = "row_n"):
+    total = len(geo)
+    for i in range(0,total,n):
+        k = i // n
+        path = f"{dirname}/part_{k}.parquet"
+        print(f"processing {path}")
+        end = np.min([i + n,total])
+        geo_slice = geo.iloc[i:end]
+        geo_slice = zonal_stats(cog, geo_slice, band_name, row_n)
+        geo_slice.to_parquet(path)
 
 
-# +
-# # %%time
-# testing -- only the lower 48 states!
-# (pad.filter(_.geom.within(bounds)).group_by([_.State_Nm]).aggregate(n = _.count()).to_pandas()) 
+# %%time
+piecewise_zonal2(cog, geo, "richness", dirname = "pad_mobi", n = 50000) # 6 min
 
-# +
 
-def piecewise_zonal(cog, tbl, crs, band_name = "mean", row_n = "row_n", dirname = "pad_parquet"):
-    total_features = tbl.count().to_pandas()
-    n = 10000
-    steps = range(0, total_features, 10000)
-    parts = [*[i for i in steps], total_features]
-    for i in range(0,len(steps)):
-        begin = parts[i]
-        end = parts[i+1] - 1
-        df = tbl.filter([_[row_n] > begin, _[row_n] <= end]).to_pandas()
-        geo = geopandas.GeoDataFrame(df, geometry=df.geometry, crs=crs)
-        geo = zonal_stats(cog, geo, band_name, row_n)
-        geo.to_parquet(f"{dirname}/part_{i}.parquet")
-    
+# # Manual approach
 
+# +
+import geopandas
+import multiprocessing.popen_spawn_posix
+from dask.distributed import Client, LocalCluster, Lock
+import rioxarray
 
-# -
+geo = geopandas.read_parquet("pad_mobi") # ~ 4.8 GB RAM
 
+# +
 # %%time
-cog = "https://data.source.coop/cboettig/mobi/species-richness-all/SpeciesRichness_All.tif"
-piecewise_zonal(cog, pad_labeled, crs, "richness")
+band_name = "human_impact"
+row_n = "row_n"
+cog = "https://data.source.coop/vizzuality/hfp-100/hfp_2021_100m_v1-2_cog.tif"
 
-# Or be bold!
-df = pad_labeled.to_pandas()
-geo = geopandas.GeoDataFrame(df, geometry=df.geometry, crs=crs)
+raster = (rioxarray.
+          open_rasterio('/vsicurl/'+cog, masked=True, chunks=True, lock=False).
+          rio.clip_box(*geo.total_bounds, crs=geo.crs).
+          rio.clip(geo.geometry.values, geo.crs, from_disk=True).
+          sel(band=1).drop_vars("band")
+         )
 
 # +
-import numpy as np
+# %%time
 
-def piecewise_zonal2(cog, geo, band_name = "mean", n = 10000, row_n = "row_n", dirname = "pad_parquet2"):
-    total = len(geo)
-    for i in range(0,total,n):
-        end = np.min([i + n,total])
-        geo_slice = geo.iloc[i:end]
-        geo_slice = zonal_stats(cog, geo_slice, band_name, row_n)
-        geo_slice.to_parquet(f"{dirname}/part_{i}.parquet")
-    
+band_name = "human_impact"
+row_n = "row_n"
+cog = "https://data.source.coop/vizzuality/hfp-100/hfp_2021_100m_v1-2_cog.tif"
 
 
-# -
+with LocalCluster() as cluster, Client(cluster) as client:
+    raster = (rioxarray.
+              open_rasterio('/vsicurl/'+cog, masked=True, chunks=True, lock=False).
+              rio.clip(geo.geometry.values, geo.crs, from_disk=True).
+              sel(band=1).drop_vars("band")
+             )
 
+# +
 # %%time
-piecewise_zonal2(cog, geo, "richness") # 6 min
-
-
-import geopandas
-gdf = geopandas.read_parquet("pad_parquet2")
 
-gdf.columns
+out_grid = make_geocube(
+    vector_data=geo,
+    measurements=['row_n'],
+    like=raster, # ensure the data are on the same grid
+)
+# ~ +1 Gb, 1.2s
 
+# +
 # %%time
-human_impacts_2021 = "https://data.source.coop/vizzuality/hfp-100/hfp_2021_100m_v1-2_cog.tif"
-gdf = piecewise_zonal2(human_impacts_2021, gdf, "human_impacts_2021")
+# 100 ~ 30s, 1000 ~ 30s
 
+out_grid["values"] = (raster.dims, raster.values, raster.attrs, raster.encoding)
+grouped_raster = out_grid.drop_vars("spatial_ref").groupby(out_grid.row_n)  # ~ +3 Gb
 
+# +
+# %%time
+grid_mean = grouped_raster.mean().rename({"values": band_name})
+zonal_stats = xarray.merge([grid_mean]).to_dataframe()
+geo = geo.merge(zonal_stats, how="left", on=row_n)
+geo.to_parquet("test.parquet")
+len(geo)
 
+# 1.2 s
+# -