refactor

preprocess.py

@@ -1,63 +1,72 @@
-# +
 import ibis
 import ibis.selectors as s
 from ibis import _
-con = ibis.duckdb.connect()
-con.load_extension("spatial")
-
-threads = 2
+import fiona
+import geopandas as gpd
+import rioxarray
+from shapely.geometry import box
 
 # +
+
 fgb = "https://data.source.coop/cboettig/pad-us-3/pad-us3-combined.fgb"
 parquet = "https://data.source.coop/cboettig/pad-us-3/pad-us3-combined.parquet"
-# "/home/rstudio/source.coop/cboettig/pad-us-3/PADUS3/PAD_US3_0.gdb"
+# gdb = "https://data.source.coop/cboettig/pad-us-3/PADUS3/PAD_US3_0.gdb" # original, all tables
 
-pad = ibis.read_parquet(parquet)
-# Currently ibis doesn't detect that this is GeoParquet.  We need a SQL escape-hatch to cast the geometry
-con.raw_sql(f"CREATE OR REPLACE VIEW pad AS SELECT *, st_geomfromwkb(geometry) as geom from read_parquet('{parquet}')")
-pad = con.table("pad")
+con = ibis.duckdb.connect()
+con.load_extension("spatial")
+threads = 24
 
 # or read the fgb version, much slower
 # pad = con.read_geo(fgb)
-
-
-# +
-# pad.filter(_.Category == "Easement").select("EHoldTyp", "Mang_Type", "Unit_Nm").distinct().head(100).to_pandas()
-# pad.filter(_.Category == "Easement").select("EsmtHldr", "Mang_Name", "Unit_Nm").distinct().sample(.1).to_pandas()
-#pad.select("Comments").distinct().head(100).to_pandas()
+# pad = con.read_parquet(parquet)
+# Currently ibis doesn't detect that this is GeoParquet.  We need a SQL escape-hatch to cast the geometry
+con.raw_sql(f"CREATE OR REPLACE VIEW pad AS SELECT *, st_geomfromwkb(geometry) as geom from read_parquet('{parquet}')")
+pad = con.table("pad")
 # -
 
-import fiona
+
+# Get the CRS
+# fiona is not built with parquet support, must read this from fgb.  ideally duckdb's st_read_meta would do this from the parquet
 meta = fiona.open(fgb)
 crs = meta.crs
 
 # +
 ## optional getting bounds
-
-import rioxarray
-from shapely.geometry import box
-
 cog = "https://data.source.coop/cboettig/mobi/species-richness-all/SpeciesRichness_All.tif"
 
-# fiona is not built with parquet support.  ideally duckdb's st_read_meta would do this.
-
-nrow = len(meta)
-
 # extract bounds. (in this case these are already in the same projection actually so r.rio.bounds() would work)
 r = rioxarray.open_rasterio(cog)
 bounds = box(*r.rio.transform_bounds(crs))
 
 # +
 # Now we can do all the usual SQL queries to subset the data.  Note the `geom.within()` spatial filter!
-focal_columns = ["bucket", "FeatClass", "Mang_Name", 
+focal_columns = ["row_n", "FeatClass", "Mang_Name", 
                  "Mang_Type",  "Des_Tp", "Pub_Access",
                  "GAP_Sts",  "IUCN_Cat",   "Unit_Nm",  
                  "State_Nm", "EsmtHldr", "Date_Est",
                  "SHAPE_Area", "geom"]
-public = ["DIST", "LOC", "FED", "STAT", "JNT"]
+pad_parquet = (
+    pad
+    .mutate(row_n=ibis.row_number())
+    .filter((_.FeatClass.isin(["Easement", "Fee"])) | (
+           (_.FeatClass == "Proclamation") & (_.Mang_Name == "TRIB"))
+           )
+    .filter(_.geom.within(bounds))  
+    .select(focal_columns)
+    .rename(geometry="geom")
+)
+
+pad_parquet.to_parquet("pad-processed.parquet")
 
+# +
 # Add our custom bucket categories:
 # really could be done seperately.
+categorical_columns = ["bucket", "FeatClass", "Mang_Name", 
+                 "Mang_Type",  "Des_Tp", "Pub_Access",
+                 "GAP_Sts",  "IUCN_Cat",   "Unit_Nm",  
+                 "State_Nm", "EsmtHldr", "Date_Est",
+                 "row_n"]
+public = ["DIST", "LOC", "FED", "STAT", "JNT"]
 case = (
     ibis.case()
     .when( (_.Mang_Type.isin(public) & _.GAP_Sts.isin(["1","2"])), "public conservation")
@@ -68,35 +77,29 @@ case = (
     .when( (_.Mang_Type == "TRIB"), "tribal")
     .end()
 )
-
-pad_parquet = (
+pad_grouping = (
     pad
+    .mutate(row_n=ibis.row_number())
     .filter((_.FeatClass.isin(["Easement", "Fee"])) | (
            (_.FeatClass == "Proclamation") & (_.Mang_Name == "TRIB"))
            )
-#    .filter(_.Mang_Type.notin(["UNK", "TERR"]))
-#    .filter(_.geom.within(bounds))  
-    .mutate(GAP_Sts = _.GAP_Sts)   # do not cast to integer!
     .mutate(bucket = case)
-    .mutate(row_n=ibis.row_number())
-    .select(focal_columns)
-    .rename(geometry="geom")
+    .select(categorical_columns)
 )
 
-#pad_parquet.to_parquet("pad-processed.parquet")
+pad_grouping.to_parquet("pad-groupings.parquet")
 # -
 
-agency_name = con.read_parquet("/home/rstudio/huggingface/datasets/pad-us-3/parquet/pad-agency-name.parquet").select(manager_name_id = "Code", manager_name = "Dom")
-agency_type = con.read_parquet("/home/rstudio/huggingface/datasets/pad-us-3/parquet/pad-agency-type.parquet").select(manager_type_id = "Code", manager_type = "Dom")
-desig_type = con.read_parquet("/home/rstudio/huggingface/datasets/pad-us-3/parquet/pad-desgination-type.parquet").select(designation_type_id = "Code", designation_type = "Dom")
-public_access = con.read_parquet("/home/rstudio/huggingface/datasets/pad-us-3/parquet/pad-public-access.parquet").select(public_access_id = "Code", public_access = "Dom")
-state_name = con.read_parquet("/home/rstudio/huggingface/datasets/pad-us-3/parquet/pad-state-name.parquet").select(state = "Code", state_name = "Dom")
-iucn = con.read_parquet("/home/rstudio/huggingface/datasets/pad-us-3/parquet/pad-iucn.parquet").select(iucn_code = "CODE", iucn_category = "DOM")
+agency_name = con.read_parquet("https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-agency-name.parquet").select(manager_name_id = "Code", manager_name = "Dom")
+agency_type = con.read_parquet("https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-agency-type.parquet").select(manager_type_id = "Code", manager_type = "Dom")
+desig_type = con.read_parquet("https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-desgination-type.parquet").select(designation_type_id = "Code", designation_type = "Dom")
+public_access = con.read_parquet("https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-public-access.parquet").select(public_access_id = "Code", public_access = "Dom")
+state_name = con.read_parquet("https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-state-name.parquet").select(state = "Code", state_name = "Dom")
+iucn = con.read_parquet("https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-iucn.parquet").select(iucn_code = "CODE", iucn_category = "DOM")
 
-pad_processed = (pad_parquet
+(pad_parquet
     .rename(manager_name_id = "Mang_Name", 
             manager_type_id = "Mang_Type", 
-            manager_group="bucket",
             designation_type_id = "Des_Tp",
             public_access_id = "Pub_Access",
             category = "FeatClass",
@@ -114,18 +117,14 @@ pad_processed = (pad_parquet
     .left_join(state_name, "state")
     .left_join(iucn, "iucn_code")
     .select(~s.contains("_right"))
- #   .select(~s.contains("_id"))
-)
-# pad_processed.to_parquet("pad-processed.parquet")
-
-# +
+#   .select(~s.contains("_id"))
 # if we keep the original geoparquet WKB 'geometry' column, to_pandas() (or execute) gives us only a normal pandas data.frame, and geopandas doesn't see the metadata.
 # if we replace the geometry with duckdb-native 'geometry' type, to_pandas() gives us a geopanadas!  But requires reading into RAM.  
-import geopandas as gpd
+    .to_pandas()
+    .set_crs(crs)
+    .to_parquet("pad-processed.parquet")
+)
 
-gdf = pad_processed.to_pandas()
-gdf = gdf.set_crs(crs)
-gdf.to_parquet("pad-processed.parquet")
 
 # +
 import rasterio
@@ -141,7 +140,8 @@ def big_zonal_stats(vec_file, tif_file, stats, col_name, n_jobs, verbose = 10, t
         raster_profile = src.profile
     gdf = gpd.read_parquet(vec_file).to_crs(raster_profile['crs'])
 
-    gdf["row_n"] = gdf.index + 1
+    # row_n is a global id, may refer to excluded polygons
+    # gdf["row_id"] = gdf.index + 1
 
     # lamba fn to zonal_stats a slice:
     def get_stats(geom_slice, tif_file, stats):
@@ -275,10 +275,13 @@ df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],  col_name = "all_spec
 # +
 columns = '''
 area_name,
-manager_name, 
-manager_type, 
+manager_name,
+manager_name_id,
+manager_type,
+manager_type_id,
 manager_group,
 designation_type,
+designation_type_id,
 public_access,
 category,
 iucn_code,