...

preprocess.py

@@ -1,29 +1,31 @@
+# +
 import ibis
 import ibis.selectors as s
 from ibis import _
 con = ibis.duckdb.connect()
 con.load_extension("spatial")
 
+threads = 2
+
 # +
 fgb = "https://data.source.coop/cboettig/pad-us-3/pad-us3-combined.fgb"
 parquet = "https://data.source.coop/cboettig/pad-us-3/pad-us3-combined.parquet"
 # "/home/rstudio/source.coop/cboettig/pad-us-3/PADUS3/PAD_US3_0.gdb"
 
-# pad = ibis.read_parquet(parquet)
+pad = ibis.read_parquet(parquet)
 # Currently ibis doesn't detect that this is GeoParquet.  We need a SQL escape-hatch to cast the geometry
-# con.raw_sql(f"CREATE OR REPLACE VIEW pad AS SELECT *, st_geomfromwkb(geometry) as geom from read_parquet('{parquet}')")
-# pad = con.table("pad")
+con.raw_sql(f"CREATE OR REPLACE VIEW pad AS SELECT *, st_geomfromwkb(geometry) as geom from read_parquet('{parquet}')")
+pad = con.table("pad")
 
-# or read the fgb version
-pad = con.read_geo(fgb)
+# or read the fgb version, much slower
+# pad = con.read_geo(fgb)
 
 
-# -
-
-
-pad.filter(_.Category == "Easement").select("EHoldTyp", "Mang_Type", "Unit_Nm").distinct().head(100).to_pandas()
+# +
+# pad.filter(_.Category == "Easement").select("EHoldTyp", "Mang_Type", "Unit_Nm").distinct().head(100).to_pandas()
 # pad.filter(_.Category == "Easement").select("EsmtHldr", "Mang_Name", "Unit_Nm").distinct().sample(.1).to_pandas()
 #pad.select("Comments").distinct().head(100).to_pandas()
+# -
 
 import fiona
 meta = fiona.open(fgb)
@@ -132,7 +134,7 @@ import geopandas as gpd
 import pandas as pd
 from joblib import Parallel, delayed
 
-def big_zonal_stats(vec_file, tif_file, stats, col_name, n_jobs, verbose = 10, timeout=1000):
+def big_zonal_stats(vec_file, tif_file, stats, col_name, n_jobs, verbose = 10, timeout=10000):
 
     # read in vector as geopandas, match CRS to raster
     with rasterio.open(tif_file) as src:
@@ -178,7 +180,7 @@ tif_file = "/home/rstudio/boettiger-lab/us-pa-policy/hfp_2021_100m_v1-2_cog.tif"
 vec_file = './pad-processed.parquet'
 
 df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],
-                     col_name = "human_impact", n_jobs=-1, verbose=0)
+                     col_name = "human_impact", n_jobs=threads, verbose=0)
 gpd.GeoDataFrame(df, geometry="geometry").to_parquet("pad-stats.parquet")
 
 # +
@@ -187,7 +189,7 @@ gpd.GeoDataFrame(df, geometry="geometry").to_parquet("pad-stats.parquet")
 tif_file = '/home/rstudio/source.coop/cboettig/mobi/species-richness-all/SpeciesRichness_All.tif'
 vec_file = './pad-stats.parquet'
 
-big_zonal_stats(vec_file, tif_file, stats = ['mean'], col_name = "richness", n_jobs=-1, verbose=0).to_parquet("pad-stats.parquet")
+big_zonal_stats(vec_file, tif_file, stats = ['mean'], col_name = "richness", n_jobs=threads, verbose=0).to_parquet("pad-stats.parquet")
 
 
 # +
@@ -197,7 +199,7 @@ tif_file = '/home/rstudio/source.coop/cboettig/mobi/range-size-rarity-all/RSR_Al
 vec_file = './pad-stats.parquet'
 
 df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],
-                      col_name = "rsr", n_jobs=-1, verbose=0).to_parquet("pad-stats.parquet")
+                      col_name = "rsr", n_jobs=threads, verbose=0).to_parquet("pad-stats.parquet")
 
 # +
 # %%time
@@ -206,7 +208,7 @@ tif_file = '/home/rstudio/source.coop/vizzuality/lg-land-carbon-data/deforest_ca
 vec_file = './pad-stats.parquet'
 
 df = big_zonal_stats(vec_file, tif_file, stats = ['mean'], 
-                     col_name = "deforest_carbon", n_jobs=-1, verbose=0).to_parquet("pad-stats.parquet")
+                     col_name = "deforest_carbon", n_jobs=threads, verbose=0).to_parquet("pad-stats.parquet")
 
 # +
 # %%time
@@ -215,7 +217,7 @@ tif_file = '/home/rstudio/source.coop/vizzuality/lg-land-carbon-data/natcrop_bii
 vec_file = './pad-stats.parquet'
 
 df = big_zonal_stats(vec_file, tif_file, stats = ['mean'], 
-                     col_name = "biodiversity_intactness_loss", n_jobs=-1, verbose=0).to_parquet("pad-stats.parquet")
+                     col_name = "biodiversity_intactness_loss", n_jobs=threads, verbose=0).to_parquet("pad-stats.parquet")
 
 # +
 # %%time
@@ -224,7 +226,7 @@ tif_file = '/home/rstudio/source.coop/vizzuality/lg-land-carbon-data/natcrop_fii
 vec_file = './pad-stats.parquet'
 
 df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],
-                     col_name = "forest_integrity_loss", n_jobs=-1, verbose=0).to_parquet("pad-stats.parquet")
+                     col_name = "forest_integrity_loss", n_jobs=threads, verbose=0).to_parquet("pad-stats.parquet")
 
 # +
 # %%time
@@ -232,7 +234,7 @@ df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],
 tif_file = '/home/rstudio/source.coop/vizzuality/lg-land-carbon-data/natcrop_expansion_100m_cog.tif'
 vec_file = './pad-stats.parquet'
 
-df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],  col_name = "crop_expansion", n_jobs=-1, verbose=0)
+df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],  col_name = "crop_expansion", n_jobs=threads, verbose=0)
 gpd.GeoDataFrame(df, geometry="geometry").to_parquet("pad-stats.parquet")
 
 # +
@@ -240,35 +242,35 @@ gpd.GeoDataFrame(df, geometry="geometry").to_parquet("pad-stats.parquet")
 tif_file = '/home/rstudio/source.coop/vizzuality/lg-land-carbon-data/natcrop_reduction_100m_cog.tif'
 vec_file = './pad-stats.parquet'
 
-df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],  col_name = "crop_reduction", n_jobs=-1, verbose=0).to_parquet("pad-stats.parquet")
+df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],  col_name = "crop_reduction", n_jobs=threads, verbose=0).to_parquet("pad-stats.parquet")
 
 # +
 # %%time
 tif_file = '/home/rstudio/source.coop/cboettig/carbon/cogs/irrecoverable_c_total_2018.tif'
 vec_file = './pad-stats.parquet'
 
-df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],  col_name = "irrecoverable_carbon", n_jobs=-1, verbose=0).to_parquet("pad-stats.parquet")
+df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],  col_name = "irrecoverable_carbon", n_jobs=threads, verbose=0).to_parquet("pad-stats.parquet")
 
 # +
 # %%time
 tif_file = '/home/rstudio/source.coop/cboettig/carbon/cogs/manageable_c_total_2018.tif'
 vec_file = './pad-stats.parquet'
 
-df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],  col_name = "manageable_carbon", n_jobs=-1, verbose=0).to_parquet("pad-stats.parquet")
+df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],  col_name = "manageable_carbon", n_jobs=threads, verbose=0).to_parquet("pad-stats.parquet")
 
 # +
 # %%time
 tif_file = '/home/rstudio/minio/shared-biodiversity/redlist/cog/combined_rwr_2022.tif'
 vec_file = './pad-stats.parquet'
 
-df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],  col_name = "all_species_rwr", n_jobs=-1, verbose=0).to_parquet("pad-stats.parquet")
+df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],  col_name = "all_species_rwr", n_jobs=threads, verbose=0).to_parquet("pad-stats.parquet")
 
 # +
 # %%time
 tif_file = '/home/rstudio/minio/shared-biodiversity/redlist/cog/combined_sr_2022.tif'
 vec_file = './pad-stats.parquet'
 
-df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],  col_name = "all_species_richness", n_jobs=-1, verbose=0).to_parquet("pad-stats.parquet")
+df = big_zonal_stats(vec_file, tif_file, stats = ['mean'],  col_name = "all_species_richness", n_jobs=threads, verbose=0).to_parquet("pad-stats.parquet")
 
 # +
 columns = '''