update preprocess

preprocess.py

@@ -46,9 +46,11 @@ public = ["DIST", "LOC", "FED", "STAT", "JNT"]
 
 case = (
     ibis.case()
-    .when( (_.Mang_Type.isin(public) & (_.GAP_Sts <= 2)), "public")
-    .when( (_.Mang_Type.isin(public) & (_.GAP_Sts > 2)), "mixed")
-    .when( (_.Mang_Type.isin(["PVT", "NGO"])), "private")
+    .when( (_.Mang_Type.isin(public) & _.GAP_Sts.isin(["1","2"])), "public protected")
+    .when( (_.Mang_Type.isin(public) & _.GAP_Sts.isin(["3"])), "mixed")
+    .when( (_.Mang_Type.isin(public) & _.GAP_Sts.isin(["4"])), "public other")
+    .when( (_.Mang_Type.isin(["PVT", "NGO"])  & (_.GAP_Sts.isin(["1","2", "3"]))), "private protected")
+    .when( (_.Mang_Type.isin(["PVT", "NGO"])  & (_.GAP_Sts.isin(["4"]))), "private other")
     .when( (_.Mang_Type == "TRIB"), "tribal")
     .end()
 )
@@ -60,7 +62,7 @@ pad_parquet = (
            )
     .filter(_.Mang_Type.notin(["UNK", "TERR"]))
     .filter(_.geom.within(bounds))
-    .mutate(GAP_Sts = _.GAP_Sts.cast("int"))
+    .mutate(GAP_Sts = _.GAP_Sts)   # do not cast to integer!
     .mutate(bucket = case)
     .mutate(row_n=ibis.row_number())
     .select(focal_columns)
@@ -74,8 +76,8 @@ agency_name = con.read_parquet("/home/rstudio/huggingface/datasets/pad-us-3/parq
 agency_type = con.read_parquet("/home/rstudio/huggingface/datasets/pad-us-3/parquet/pad-agency-type.parquet").select(manager_type_id = "Code", manager_type = "Dom")
 desig_type = con.read_parquet("/home/rstudio/huggingface/datasets/pad-us-3/parquet/pad-desgination-type.parquet").select(designation_type_id = "Code", designation_type = "Dom")
 public_access = con.read_parquet("/home/rstudio/huggingface/datasets/pad-us-3/parquet/pad-public-access.parquet").select(public_access_id = "Code", public_access = "Dom")
-state_name = con.read_parquet("/home/rstudio/huggingface/datasets/pad-us-3/parquet/pad-state-name.parquet").select(state_id = "Code", state = "Dom")
-iucn = con.read_parquet("/home/rstudio/huggingface/datasets/pad-us-3/parquet/pad-iucn.parquet").select(iucn_id = "CODE", iucn_category = "DOM")
+state_name = con.read_parquet("/home/rstudio/huggingface/datasets/pad-us-3/parquet/pad-state-name.parquet").select(state = "Code", state_name = "Dom")
+iucn = con.read_parquet("/home/rstudio/huggingface/datasets/pad-us-3/parquet/pad-iucn.parquet").select(iucn_code = "CODE", iucn_category = "DOM")
 
 pad_processed = (pad_parquet
     .rename(manager_name_id = "Mang_Name", 
@@ -84,21 +86,21 @@ pad_processed = (pad_parquet
             designation_type_id = "Des_Tp",
             public_access_id = "Pub_Access",
             category = "FeatClass",
-            iucn_id = "IUCN_Cat",
+            iucn_code = "IUCN_Cat",
             gap_code = "GAP_Sts",
-            state_id = "State_Nm",
+            state = "State_Nm",
             easement_holder = "EsmtHldr",
             date_established = "Date_Est",
             area_square_meters = "SHAPE_Area",
-            name = "Unit_Nm")
+            area_name = "Unit_Nm")
     .left_join(agency_name, "manager_name_id")
     .left_join(agency_type, "manager_type_id")
     .left_join(desig_type, "designation_type_id")
     .left_join(public_access, "public_access_id")
-    .left_join(state_name, "state_id")
-    .left_join(iucn, "iucn_id")
+    .left_join(state_name, "state")
+    .left_join(iucn, "iucn_code")
     .select(~s.contains("_right"))
-    .select(~s.contains("_id"))
+ #   .select(~s.contains("_id"))
 )
 # pad_processed.to_parquet("pad-processed.parquet")
 
@@ -142,19 +144,25 @@ def big_zonal_stats(vec_file, tif_file, stats, col_name, n_jobs, verbose = 10, t
     output = Parallel(n_jobs=n_jobs, timeout=timeout, verbose=verbose)(jobs)
 
     # reshape output
-    df_zonal_stats = (
+    df = (
         pd.DataFrame(output)
         .rename(columns={'mean': col_name})
         .merge(gdf, how='right', on = 'row_n')
         )
-    return df_zonal_stats
+    gdf = gpd.GeoDataFrame(df, geometry="geometry")
+    return gdf
 
 
 
+# -
+
+import geopandas as gpd
+test = gpd.read_parquet("pad-processed.parquet")
+test.columns
+
 # +
 # %%time
-
-tif_file = './hfp_2021_100m_v1-2_cog.tif'
+tif_file = "/home/rstudio/boettiger-lab/us-pa-policy/hfp_2021_100m_v1-2_cog.tif"
 vec_file = './pad-processed.parquet'
 
 df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],
@@ -167,9 +175,8 @@ gpd.GeoDataFrame(df, geometry="geometry").to_parquet("pad-stats.parquet")
 tif_file = '/home/rstudio/source.coop/cboettig/mobi/species-richness-all/SpeciesRichness_All.tif'
 vec_file = './pad-stats.parquet'
 
-df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],
-                     col_name = "richness", n_jobs=-1, verbose=0)
-gpd.GeoDataFrame(df, geometry="geometry").to_parquet("pad-stats.parquet")
+big_zonal_stats(vec_file, tif_file, stats = ['mean'], col_name = "richness", n_jobs=-1, verbose=0).to_parquet("pad-stats.parquet")
+
 
 # +
 # %%time
@@ -178,8 +185,7 @@ tif_file = '/home/rstudio/source.coop/cboettig/mobi/range-size-rarity-all/RSR_Al
 vec_file = './pad-stats.parquet'
 
 df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],
-                      col_name = "rsr", n_jobs=-1, verbose=0)
-gpd.GeoDataFrame(df, geometry="geometry").to_parquet("pad-stats.parquet")
+                      col_name = "rsr", n_jobs=-1, verbose=0).to_parquet("pad-stats.parquet")
 
 # +
 # %%time
@@ -187,8 +193,8 @@ gpd.GeoDataFrame(df, geometry="geometry").to_parquet("pad-stats.parquet")
 tif_file = '/home/rstudio/source.coop/vizzuality/lg-land-carbon-data/deforest_carbon_100m_cog.tif'
 vec_file = './pad-stats.parquet'
 
-df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],  col_name = "deforest_carbon", n_jobs=-1, verbose=0)
-gpd.GeoDataFrame(df, geometry="geometry").to_parquet("pad-stats.parquet")
+df = big_zonal_stats(vec_file, tif_file, stats = ['mean'], 
+                     col_name = "deforest_carbon", n_jobs=-1, verbose=0).to_parquet("pad-stats.parquet")
 
 # +
 # %%time
@@ -196,8 +202,8 @@ gpd.GeoDataFrame(df, geometry="geometry").to_parquet("pad-stats.parquet")
 tif_file = '/home/rstudio/source.coop/vizzuality/lg-land-carbon-data/natcrop_bii_100m_cog.tif'
 vec_file = './pad-stats.parquet'
 
-df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],  col_name = "biodiversity_intactness", n_jobs=-1, verbose=0)
-gpd.GeoDataFrame(df, geometry="geometry").to_parquet("pad-stats.parquet")
+df = big_zonal_stats(vec_file, tif_file, stats = ['mean'], 
+                     col_name = "biodiversity_intactness_loss", n_jobs=-1, verbose=0).to_parquet("pad-stats.parquet")
 
 # +
 # %%time
@@ -205,8 +211,8 @@ gpd.GeoDataFrame(df, geometry="geometry").to_parquet("pad-stats.parquet")
 tif_file = '/home/rstudio/source.coop/vizzuality/lg-land-carbon-data/natcrop_fii_100m_cog.tif'
 vec_file = './pad-stats.parquet'
 
-df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],  col_name = "forest_integrity", n_jobs=-1, verbose=0)
-gpd.GeoDataFrame(df, geometry="geometry").to_parquet("pad-stats.parquet")
+df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],
+                     col_name = "forest_integrity_loss", n_jobs=-1, verbose=0).to_parquet("pad-stats.parquet")
 
 # +
 # %%time
@@ -219,20 +225,84 @@ gpd.GeoDataFrame(df, geometry="geometry").to_parquet("pad-stats.parquet")
 
 # +
 # %%time
-
 tif_file = '/home/rstudio/source.coop/vizzuality/lg-land-carbon-data/natcrop_reduction_100m_cog.tif'
 vec_file = './pad-stats.parquet'
 
-df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],  col_name = "crop_reduction", n_jobs=-1, verbose=0)
-gpd.GeoDataFrame(df, geometry="geometry").to_parquet("pad-stats.parquet")
+df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],  col_name = "crop_reduction", n_jobs=-1, verbose=0).to_parquet("pad-stats.parquet")
+
+# +
+# %%time
+tif_file = '/home/rstudio/source.coop/cboettig/carbon/cogs/irrecoverable_c_total_2018.tif'
+vec_file = './pad-stats.parquet'
+
+df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],  col_name = "irrecoverable_carbon", n_jobs=-1, verbose=0).to_parquet("pad-stats.parquet")
 
 # +
+# %%time
+tif_file = '/home/rstudio/source.coop/cboettig/carbon/cogs/manageable_c_total_2018.tif'
+vec_file = './pad-stats.parquet'
+
+df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],  col_name = "manageable_carbon", n_jobs=-1, verbose=0).to_parquet("pad-stats.parquet")
+
+# +
+# %%time
+tif_file = '/home/rstudio/minio/shared-biodiversity/redlist/cog/combined_rwr_2022.tif'
+vec_file = './pad-stats.parquet'
+
+df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],  col_name = "all_species_rwr", n_jobs=-1, verbose=0).to_parquet("pad-stats.parquet")
+
+# +
+# %%time
+tif_file = '/home/rstudio/minio/shared-biodiversity/redlist/cog/combined_sr_2022.tif'
+vec_file = './pad-stats.parquet'
+
+df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],  col_name = "all_species_richness", n_jobs=-1, verbose=0).to_parquet("pad-stats.parquet")
+# -
+
+columns = '''
+area_name,
+manager_name, 
+manager_type, 
+manager_group,
+designation_type,
+public_access,
+category,
+iucn_code,
+iucn_category,
+gap_code,
+state,
+state_name,
+easement_holder,
+date_established,
+area_square_meters,
+geometry,
+all_species_richness,
+all_species_rwr,
+manageable_carbon,
+irrecoverable_carbon,
+crop_reduction,
+crop_expansion,
+deforest_carbon,
+richness,
+rsr,
+forest_integrity_loss,
+biodiversity_intactness_loss
+'''
+
+import ibis
+df = ibis.read_parquet("pad-stats.parquet")
+df.columns
+
+# +
+
+
+## create pad.duckdb
 from sqlalchemy import create_engine
 from sqlalchemy import text
 db_uri = "duckdb:///pad.duckdb"
 engine = create_engine(db_uri)
 con = engine.connect()
-con.execute("create or replace table pad as select * from 'pad-stats.parquet'")
+con.execute(f"create or replace table pad as select {columns} from 'pad-stats.parquet'")
 con.close()
 
 # pad_stats = ibis.read_parquet("pad-stats.parquet")