working on pre-process

preprocess.py

@@ -1,34 +1,44 @@
 import ibis
 import ibis.selectors as s
 from ibis import _
+con = ibis.duckdb.connect()
+con.load_extension("spatial")
 
 # +
 fgb = "https://data.source.coop/cboettig/pad-us-3/pad-us3-combined.fgb"
 parquet = "https://data.source.coop/cboettig/pad-us-3/pad-us3-combined.parquet"
+# "/home/rstudio/source.coop/cboettig/pad-us-3/PADUS3/PAD_US3_0.gdb"
 
-#pad = ibis.read_parquet(parquet)
+# pad = ibis.read_parquet(parquet)
 # Currently ibis doesn't detect that this is GeoParquet.  We need a SQL escape-hatch to cast the geometry
-con = ibis.duckdb.connect()
-con.load_extension("spatial")
-con.raw_sql(f"CREATE OR REPLACE VIEW pad AS SELECT *, st_geomfromwkb(geometry) as geom from read_parquet('{parquet}')")
-pad = con.table("pad")
+# con.raw_sql(f"CREATE OR REPLACE VIEW pad AS SELECT *, st_geomfromwkb(geometry) as geom from read_parquet('{parquet}')")
+# pad = con.table("pad")
 
+# or read the fgb version
+pad = con.read_geo(fgb)
 
-# +
-#pad.filter(_.Category == "Easement").select("EHoldTyp", "Mang_Type", "Unit_Nm").distinct().head(100).to_pandas()
+
+# -
+
+
+pad.filter(_.Category == "Easement").select("EHoldTyp", "Mang_Type", "Unit_Nm").distinct().head(100).to_pandas()
 # pad.filter(_.Category == "Easement").select("EsmtHldr", "Mang_Name", "Unit_Nm").distinct().sample(.1).to_pandas()
 #pad.select("Comments").distinct().head(100).to_pandas()
 
-# +
 import fiona
+meta = fiona.open(fgb)
+crs = meta.crs
+
+# +
+## optional getting bounds
+
 import rioxarray
 from shapely.geometry import box
 
 cog = "https://data.source.coop/cboettig/mobi/species-richness-all/SpeciesRichness_All.tif"
 
-# fiona not built with parquet support.  ideally duckdb's st_read_meta would do this.
-meta = fiona.open(fgb)
-crs = meta.crs
+# fiona is not built with parquet support.  ideally duckdb's st_read_meta would do this.
+
 nrow = len(meta)
 
 # extract bounds. (in this case these are already in the same projection actually so r.rio.bounds() would work)
@@ -44,13 +54,15 @@ focal_columns = ["bucket", "FeatClass", "Mang_Name",
                  "SHAPE_Area", "geom"]
 public = ["DIST", "LOC", "FED", "STAT", "JNT"]
 
+# Add our custom bucket categories:
+# really could be done seperately.
 case = (
     ibis.case()
-    .when( (_.Mang_Type.isin(public) & _.GAP_Sts.isin(["1","2"])), "public protected")
-    .when( (_.Mang_Type.isin(public) & _.GAP_Sts.isin(["3"])), "mixed")
-    .when( (_.Mang_Type.isin(public) & _.GAP_Sts.isin(["4"])), "public other")
-    .when( (_.Mang_Type.isin(["PVT", "NGO"])  & (_.GAP_Sts.isin(["1","2", "3"]))), "private protected")
-    .when( (_.Mang_Type.isin(["PVT", "NGO"])  & (_.GAP_Sts.isin(["4"]))), "private other")
+    .when( (_.Mang_Type.isin(public) & _.GAP_Sts.isin(["1","2"])), "public conservation")
+    .when( (_.Mang_Type.isin(public) & _.GAP_Sts.isin(["3"])), "mixed use")
+    .when( (_.Mang_Type.isin(public) & _.GAP_Sts.isin(["4"])), "public unprotected")
+    .when( (_.Mang_Type.isin(["PVT", "NGO"])  & (_.GAP_Sts.isin(["1","2", "3"]))), "private conservation")
+    .when( (_.Mang_Type.isin(["PVT", "NGO"])  & (_.GAP_Sts.isin(["4"]))), "private unprotected")
     .when( (_.Mang_Type == "TRIB"), "tribal")
     .end()
 )
@@ -60,8 +72,8 @@ pad_parquet = (
     .filter((_.FeatClass.isin(["Easement", "Fee"])) | (
            (_.FeatClass == "Proclamation") & (_.Mang_Name == "TRIB"))
            )
-    .filter(_.Mang_Type.notin(["UNK", "TERR"]))
-    .filter(_.geom.within(bounds))
+#    .filter(_.Mang_Type.notin(["UNK", "TERR"]))
+#    .filter(_.geom.within(bounds))  
     .mutate(GAP_Sts = _.GAP_Sts)   # do not cast to integer!
     .mutate(bucket = case)
     .mutate(row_n=ibis.row_number())
@@ -257,8 +269,8 @@ tif_file = '/home/rstudio/minio/shared-biodiversity/redlist/cog/combined_sr_2022
 vec_file = './pad-stats.parquet'
 
 df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],  col_name = "all_species_richness", n_jobs=-1, verbose=0).to_parquet("pad-stats.parquet")
-# -
 
+# +
 columns = '''
 area_name,
 manager_name, 
@@ -289,13 +301,18 @@ forest_integrity_loss,
 biodiversity_intactness_loss
 '''
 
+items = columns.split(',')
+# Remove empty strings and whitespace
+items = [item.strip() for item in items if item.strip()]
+items
+# -
+
 import ibis
-df = ibis.read_parquet("pad-stats.parquet")
-df.columns
+from ibis import _
+df = ibis.read_parquet("pad-stats.parquet").select(items)
+df.group_by(_.manager_group).aggregate(n = _.manager_group.count()).to_pandas()
 
 # +
-
-
 ## create pad.duckdb
 from sqlalchemy import create_engine
 from sqlalchemy import text
@@ -307,3 +324,10 @@ con.close()
 
 # pad_stats = ibis.read_parquet("pad-stats.parquet")
 # pad_stats.head(20).to_pandas()
+# -
+
+import pandas as pd
+db_uri = "duckdb:///pad.duckdb"
+engine = create_engine(db_uri)
+con = engine.connect()
+pd.DataFrame(con.execute("select * from pad limit 1").fetchall())