parts

preprocess.py

@@ -1,16 +1,22 @@
 import ibis
 from ibis import _
 import rioxarray
+import xarray
 from shapely.geometry import box
+from geocube.api.core import make_geocube
+import geopandas
 import fiona
 
+
 # +
 fgb = "https://data.source.coop/cboettig/pad-us-3/pad-us3-combined.fgb"
 parquet = "https://data.source.coop/cboettig/pad-us-3/pad-us3-combined.parquet"
 cog = "https://data.source.coop/cboettig/mobi/species-richness-all/SpeciesRichness_All.tif"
 
 # fiona not built with parquet support.  ideally duckdb's st_read_meta would do this.
-crs = fiona.open(fgb).crs
+meta = fiona.open(fgb)
+crs = meta.crs
+nrow = len(meta)
 
 # extract bounds. (in this case these are already in the same projection actually so r.rio.bounds() would work)
 r = rioxarray.open_rasterio(cog)
@@ -27,7 +33,7 @@ con = ibis.duckdb.connect()
 
 # Currently ibis doesn't detect that this is GeoParquet.  We need a SQL escape-hatch to cast the geometry
 con.load_extension("spatial")
-con.raw_sql(f"CREATE VIEW pad AS SELECT *, st_geomfromwkb(geometry) as geom from read_parquet('{parquet}')")
+con.raw_sql(f"CREATE OR REPLACE VIEW pad AS SELECT *, st_geomfromwkb(geometry) as geom from read_parquet('{parquet}')")
 pad = con.table("pad")
 
 # +
@@ -59,34 +65,15 @@ pad_labeled = (
 )
 
 
-# +
-# # %%time
-# testing -- only the lower 48 states!
-# (pad.filter(_.geom.within(bounds)).group_by([_.State_Nm]).aggregate(n = _.count()).to_pandas()) 
 # -
 
-# We could work in chunks, possibly parallelize this....
-start = 0
-end = 10000
-df = pad_labeled.filter([_.row_n > start, _.row_n <= end]).to_pandas()
-
-# Or be bold!
-df = pad_labeled.to_pandas()
-
-from geocube.api.core import make_geocube
-import xarray
-import geopandas
-geo = geopandas.GeoDataFrame(df, geometry=df.geometry, crs=crs)
-
-geo.shape
 
-
-def zonal_stats(cog, geo, crs, row_n = "row_n"):
+def zonal_stats(cog, geo, band_name = "mean", row_n = "row_n"):
 # https://corteva.github.io/geocube/html/examples/zonal_statistics.html
     raster = (rioxarray.
          open_rasterio('/vsicurl/'+cog, masked=True).
-         rio.clip(geo.geometry.values, crs, from_disk=True).
-         sel(band=1).drop("band")
+         rio.clip(geo.geometry.values, geo.crs, from_disk=True).
+         sel(band=1).drop_vars("band")
     )
     out_grid = make_geocube(
         vector_data=geo,
@@ -95,22 +82,112 @@ def zonal_stats(cog, geo, crs, row_n = "row_n"):
     )
     # merge the two together
     out_grid["values"] = (raster.dims, raster.values, raster.attrs, raster.encoding)
-    grouped_raster = out_grid.drop("spatial_ref").groupby(out_grid.row_n)
+    grouped_raster = out_grid.drop_vars("spatial_ref").groupby(out_grid.row_n)
     # can add other stats
-    grid_mean = grouped_raster.mean().rename({"values": "mean"})
+    grid_mean = grouped_raster.mean().rename({"values": band_name})
     zonal_stats = xarray.merge([grid_mean]).to_dataframe()
     geo = geo.merge(zonal_stats, how="left", on=row_n)
     return geo
 
 
-geo = zonal_stats(cog, geo, crs)
+total_features = pad_labeled.count().to_pandas()
+
+
+# +
+# # %%time
+# testing -- only the lower 48 states!
+# (pad.filter(_.geom.within(bounds)).group_by([_.State_Nm]).aggregate(n = _.count()).to_pandas()) 
+
+# +
+
+def piecewise_zonal(cog, tbl, crs, band_name = "mean", row_n = "row_n", dirname = "pad_parquet"):
+    total_features = tbl.count().to_pandas()
+    n = 10000
+    steps = range(0, total_features, 10000)
+    parts = [*[i for i in steps], total_features]
+    for i in range(0,len(steps)):
+        begin = parts[i]
+        end = parts[i+1] - 1
+        df = tbl.filter([_[row_n] > begin, _[row_n] <= end]).to_pandas()
+        geo = geopandas.GeoDataFrame(df, geometry=df.geometry, crs=crs)
+        geo = zonal_stats(cog, geo, band_name, row_n)
+        geo.to_parquet(f"{dirname}/part_{i}.parquet")
+    
+
+
+# -
+
+# %%time
+cog = "https://data.source.coop/cboettig/mobi/species-richness-all/SpeciesRichness_All.tif"
+piecewise_zonal(cog, pad_labeled, crs, "richness")
+
+# Or be bold!
+df = pad_labeled.to_pandas()
+geo = geopandas.GeoDataFrame(df, geometry=df.geometry, crs=crs)
+
+# +
+import numpy as np
+
+def piecewise_zonal2(cog, geo, band_name = "mean", n = 10000, row_n = "row_n", dirname = "pad_parquet2"):
+    total = len(geo)
+    for i in range(0,total,n):
+        end = np.min([i + n,total])
+        geo_slice = geo.iloc[i:end]
+        geo_slice = zonal_stats(cog, geo_slice, band_name, row_n)
+        geo_slice.to_parquet(f"{dirname}/part_{i}.parquet")
+    
+
+
+# -
+
+# %%time
+piecewise_zonal2(cog, geo, "richness") # 6 min
+
+
+import geopandas
+gdf = geopandas.read_parquet("pad_parquet2")
+
+gdf.columns
+
+# %%time
+human_impacts_2021 = "https://data.source.coop/vizzuality/hfp-100/hfp_2021_100m_v1-2_cog.tif"
+gdf = piecewise_zonal2(human_impacts_2021, gdf, "human_impacts_2021")
+
 
-geo.to_parquet("pad-mobi.parquet")
 
-# Now we need to convert to PMTiles:
+
+
+
+# Lastly we need to convert to PMTiles:
 #
 # ```
 # ogr2ogr  -dsco MAX_SIZE=90000000 -dsco MAX_FEATURES=50000000 -dsco MAXZOOM=10 pad-mobi.pmtiles pad-mobi.parquet
 # ```
 
-geo.plot(column="mean", legend=True)
+geo.plot(column="richness", legend=True)
+
+import geopandas
+gdf = geopandas.read_parquet("pad-mobi.parquet")
+cog = "https://data.source.coop/cboettig/mobi/range-size-rarity-all/RSR_All.tif"
+
+
+human_impacts_2021 = "https://data.source.coop/vizzuality/hfp-100/hfp_2021_100m_v1-2_cog.tif"
+geo = zonal_stats(human_impacts_2021, geo, "human_impacts_2021")
+
+
+geo.to_parquet("pad-extended.parquet")
+
+# +
+# %%time
+geo = zonal_stats("https://data.source.coop/vizzuality/lg-land-carbon-data/deforest_carbon_100m_cog.tif", geo, "deforest_carbon")
+geo = zonal_stats("https://data.source.coop/vizzuality/lg-land-carbon-data/natcrop_fii_100m_cog.tif", geo, "fii")
+geo = zonal_stats("https://data.source.coop/vizzuality/lg-land-carbon-data/natcrop_bii_100m_cog.tif", geo, "bii")
+geo = zonal_stats("https://data.source.coop/vizzuality/lg-land-carbon-data/natcrop_expansion_100m_cog.tif", geo, "crop_expansion")
+geo = zonal_stats("https://data.source.coop/vizzuality/lg-land-carbon-data/natcrop_reduction_100m_cog.tif", geo, "crop_reduction")
+	
+geo = zonal_stats("https://data.source.coop/cboettig/carbon/cogs/irrecoverable_c_total_2018.tif", geo, "irrecoverable_c_total_2018")
+geo = zonal_stats("https://data.source.coop/cboettig/carbon/cogs/manageable_c_total_2018.tif", geo, "manageable_c_total_2018")
+
+cog = "https://data.source.coop/cboettig/mobi/range-size-rarity-all/RSR_All.tif"
+
+geo.to_parquet("pad-extended.parquet")