# -*- coding: utf-8 -*- import gc import glob import os from datetime import timedelta from typing import Dict import fiona import geopandas as gpd import netCDF4 import numpy as np import pandas as pd import rasterio from rasterio.mask import mask from shapely.validation import make_valid from libs.utils import verbose as vprint V = 1 V_IGNORE = [] # Debug, Warning, Error def read_shape_file( file_name: str, epsg_num: int = 4326, dissolve=True, # buffer: float = 100, *args, **kwargs, ) -> gpd.GeoDataFrame: """Read a shape file and return a geo-dataframe object. Parameters ---------- file_name : str path to shape file epsg_num : int, optional coordinate system number dissolve : bool, optional Dissolve to one geometry, e.g. get the most external shape. Useful when interested in the Returns ------- gpd.GeoDataFrame pandas geo-dataframe with the geometry data and other infos Raises ------ FileNotFoundError error if file is not found """ if not os.path.exists(file_name): raise FileNotFoundError geo_df = gpd.read_file(file_name) if epsg_num is not None: geo_df["geometry"] = geo_df["geometry"].to_crs(epsg=epsg_num) crs = geo_df.crs if crs is None: crs = 4326 print("Initial CRS:", crs) # check for multi geo_df.geometry = geo_df.apply( lambda row: make_valid(row.geometry) if not row.geometry.is_valid else row.geometry, axis=1, ) return geo_df def find_match_raster(model_path, farm_name): # navigate one directory up in model_path while not os.path.basename(model_path) == farm_name: model_path = os.path.dirname(model_path) root = model_path print("Looking for a matching raster in:", root) for root, dirs, _ in os.walk(root): # print(root,dirs,"\n\n") if "et_pp" in dirs: real_path = os.path.join(root, "et_pp") print("raster_path is", real_path) # for file in os.listdir(real_path): # if file.endswith(".tif"): # match_raster = os.path.join(real_path,file) # print("Found match raster:",match_raster) return real_path return None def find_shape_file(model_path, farm_name): # navigate one directory up in model_path until you reach the farm_name directory while not os.path.basename(model_path) == farm_name: model_path = os.path.dirname(model_path) print("Looking for a shape file in:", model_path) root = model_path print("Looking for a shape file in:", root) for root, dirs, files in os.walk(root): # print(root,dirs,"\n\n") for file in files: if file.endswith(".shp"): real_path = os.path.join(root, file) return real_path return None def get_range_agg( input_dir: str, window_start: str, window_end: str, layer_name: str, agg: str = "mean", ) -> np.ndarray: """Get the mean for a given window_start and window_end dates. Parameters ---------- input_dir : str Path to the directory containing the netcdf files. window_start : str Start date of the window. Format: YYYY-MM-DD. window_end : str End date of the window. Format: YYYY-MM-DD. layer_name : str Soil layer to consider for the mean. agg : str Aggregation method to use. Possible values: mean, median, max, min, std, or None. Returns ------- np.ndarray Mean raster for the given window_start and window_end dates. """ # Get the list of dates between two dates if date_from and date_to dates = pd.DataFrame( pd.date_range( pd.to_datetime(window_start), pd.to_datetime(window_end) - timedelta(days=1), freq="d", ), columns=["date"], ) # .strftime('%Y-%m-%d') dates["dayofyear"] = dates["date"].dt.dayofyear - 1 dates["year"] = dates["date"].dt.year dates["str_dates"] = dates["date"].dt.strftime("%Y-%m-%d") yearly_dates = dates.groupby("year")["dayofyear"].apply(list).to_dict() data_l = list() # For each year, get the data for layer_name for the dates specified in yearly_dates for year in yearly_dates: # read the year file nc_y = netCDF4.Dataset(os.path.join(input_dir, f"model_{year}.nc")) vprint( 1, V, V_IGNORE, Debug=f"getting data for year: {year} from layer: {layer_name}...", ) # Get the data for the layer_name data = nc_y.variables[layer_name][:, :, :] # Get the data for the dates days = yearly_dates[year] data = data[days, :, :] data_l.append(data) nc_y.close() del data gc.collect() # Concat data for all years data_concat = np.concatenate(data_l, axis=0) data_concat.shape if agg == "mean": # Get the mean raster for the range data_agg = np.mean(data_concat, axis=0) elif agg == "median": # Get the median raster for the range data_agg = np.median(data_concat, axis=0) elif agg == "max": # Get the max raster for the range data_agg = np.max(data_concat, axis=0) elif agg == "min": # Get the min raster for the range data_agg = np.min(data_concat, axis=0) elif agg == "std": # Get the std raster for the range data_agg = np.std(data_concat, axis=0) elif agg == "var": # Get the var raster for the range data_agg = np.var(data_concat, axis=0) elif agg == "sum": # Get the sum raster for the range data_agg = np.sum(data_concat, axis=0) elif agg is None: data_agg = data_concat.copy() else: raise ValueError( f"agg should be one of 'mean', 'median', 'max', 'min', 'std', 'var', 'sum', or a None value. {agg} was provided." ) print("done.") return data_agg def get_historic_agg( input_dir: str, historic_years: int, current_window_start: str, current_window_end: str, layer_name: str, agg_window: str = "mean", agg_history: str = "mean", ) -> np.ndarray: """Get the historic mean for a given window_start and window_end dates. Parameters ---------- input_dir : str Path to the directory containing the netcdf files. historic_years : int Number of historic years to consider for the mean. current_window_start : str Start date of the current window. Format: YYYY-MM-DD. current_window_end : str End date of the current window. Format: YYYY-MM-DD. layer_name : str Soil layer to consider for the mean. agg_window : str Aggregation method for the window (applies to both current and historic). Default is "mean". Possible values: "mean", "median", "max", "min", "std", "var", None (None will just make a 3D matrix for each year). agg_history : str Aggregation method for the historic years. This defines the way all years are combined into one. Default is "mean". Possible values: "mean", "median", "max", "min", "std", "var", "quantiles". If `quantile` is selected, the historic cube will be sent back as a 3D matrix (with non-agregated window and concatenated on axis 0 instead) for quantile comparison. Returns ------- np.ndarray Array of the historic mean for the given window_start and window_end dates for the historic years. Raises ------ FileNotFoundError If the file for the historic year is not found. Possible solutions: - The historic year should be modelled before calling this function. - The path to the historic year should be changed. - Calculate for a more recent historic year by reducing historic_years value. """ # Get the window_start year window_start_year = pd.to_datetime(current_window_start).year window_end_year = pd.to_datetime(current_window_end).year # Get the first year first_year = window_start_year - historic_years # Check if file exists for this year if os.path.exists(os.path.join(input_dir, f"model_{first_year}.nc")): # Get the list of historic windows historic_agg = {} for year in range(1, historic_years + 1): args = { "input_dir": input_dir, "window_start": f"{window_start_year-year}{current_window_start[4:]}", "window_end": f"{window_end_year-year}{current_window_end[4:]}", "layer_name": layer_name, "agg": agg_window, } # Get the range mean historic_agg[window_start_year - year] = get_range_agg(**args) historic_agg_np = np.array([historic_agg[year] for year in historic_agg]) # Get the aggregation of the historic years if agg_history == "mean": historic_agg_np = np.mean(historic_agg_np, axis=0) elif agg_history == "median": historic_agg_np = np.median(historic_agg_np, axis=0) elif agg_history == "max": historic_agg_np = np.max(historic_agg_np, axis=0) elif agg_history == "min": historic_agg_np = np.min(historic_agg_np, axis=0) elif agg_history == "std": historic_agg_np = np.std(historic_agg_np, axis=0) elif agg_history == "var": historic_agg_np = np.var(historic_agg_np, axis=0) elif agg_history == "sum": historic_agg_np = np.sum(historic_agg_np, axis=0) elif agg_history is None: historic_agg_np = np.concatenate(list(historic_agg.values()), axis=0) else: raise ValueError( f"Invalid aggregation method: {agg_history}. Possible values: mean, median, max, min, std, var, sum." ) return historic_agg_np else: raise FileNotFoundError( f"File not found for the historic data: {os.path.join(input_dir,f'model_{first_year}.nc')}. Make sure the path is correct and the historic year for the requested year is modelled before calling this function." ) def save(path, array, profile): """Save the array as a raster. Parameters ---------- path : str Path to the raster to save. array : np.ndarray Array to save as a raster. profile : dict Profile of the raster to save. """ with rasterio.open(path, "w", **profile) as dst: dst.write(array, 1) def analyse( input, window_start, window_end, historic_years: int, layer: str, match_raster: str = None, output: str = None, agg_history: str = "mean", agg_window: str = "mean", farm_name: str = None, **kwargs, ) -> Dict[str, str]: """Main function to run the script. Parameters ---------- input : str Path to the input raster. window_start : str Start date of the window. Format: YYYY-MM-DD. window_end : str End date of the window. Format: YYYY-MM-DD. historic_years : int Number of historic years to use for the comparison. layer : str Soil layer to consider for the comparison. match_raster : str Path to the match raster. Default: None. If None, the match raster will be searched in the et_pp directory based on the input directory. output : str Path to the output raster. Default: None. If None, the output raster will be saved in the same directory as the input raster. agg_history : str Aggregation method to use for the historic years. Possible values: 'mean', 'median', 'max', 'min', 'std', None. Default: 'mean'. agg_window : str Aggregation method to use for the window. Possible values: 'mean', 'median', 'max', 'min', 'std', None. Default: 'mean'. farm_name : str Name of the farm. Should be provided. Default: None. Returns ------- Dict[str,str] Dictionary with the path to the output rasters. """ if output is None: output = os.path.join(input, "analysis") # Create the output directory if it does not exist if not os.path.exists(output): os.makedirs(output) if match_raster is None: match_raster = find_match_raster(input, farm_name) print("match_raster is:", match_raster) if match_raster is not None: print("Found match raster:", match_raster) files = glob.glob(os.path.join(match_raster, f"{window_start[:7]}*.tif")) if len(files) == 0: files = glob.glob(os.path.join(match_raster, f"{window_end[:7]}*.tif")) if len(files) == 0: vprint( 1, V, V_IGNORE, Debug=f"Expanding the search for match raster file to find e closer date to {window_start[:5]}...", ) files = glob.glob(os.path.join(match_raster, f"{window_start[:5]}*.tif")) if len(files) == 0: vprint( 1, V, V_IGNORE, Debug=f"Expanding the search further for match raster file to find e closer date to {window_end[:5]}...", ) files = glob.glob(os.path.join(match_raster, f"{window_end[:5]}*.tif")) if len(files) == 0: raise FileNotFoundError( f"Could not find any matching raster in {match_raster} for the rage of dates given at {window_start} / {window_end}!" ) print(f"Found {len(files)} matching raster file {files[0]}.") match_raster = files[0] with rasterio.open(match_raster) as src: profile = src.profile # Get the layers layer = layer # Get the historic aggregated data # Get aggregated current window data current_data = get_range_agg( input_dir=input, window_start=window_start, window_end=window_end, agg=agg_window, layer_name=layer, ) historic_data = get_historic_agg( input_dir=input, historic_years=historic_years, current_window_start=window_start, current_window_end=window_end, agg_window=agg_window, agg_history=agg_history, layer_name=layer, ) historic_data_quant = get_historic_agg( input_dir=input, historic_years=historic_years, current_window_start=window_start, current_window_end=window_end, agg_window=None, agg_history=None, layer_name=layer, ) # Compare the two rasters delta = current_data - historic_data quantile = current_data.copy() print("\nCalculating quantiles...", "\n=========================") print("Data shape:", current_data.shape) print("Historic data shape:", historic_data_quant.shape) print("=========================\n") pixel_now = [] pixel_hist = [] pixel_quant = [] for i in range(current_data.shape[0]): for j in range(current_data.shape[1]): sorted_scores = np.array(sorted(historic_data_quant[:, i, j])) quantile[i, j] = ( (sorted_scores.searchsorted(current_data[i, j])) / sorted_scores.shape[0] * 100 ) pixel_now.append(current_data[i, j]) pixel_hist.append(sorted_scores) pixel_quant.append(quantile[i, j]) df_quants = pd.DataFrame(pixel_hist) df_quants["pixel_now"] = pixel_now df_quants["pixel_quant"] = pixel_quant print("shapes are:", len(pixel_now), len(pixel_quant)) df_quants = df_quants.sort_values(by=["pixel_quant"], ascending=False) print(df_quants) df_quants.to_csv( os.path.join( output, f"quantiles-{window_start.replace('-','_')}-{window_end.replace('-','_')}-{layer}-w_{agg_window}-h_concat-y_{historic_years}.csv", ) ) # Search for a shape file ending with .shp shape_file = find_shape_file(input, farm_name) print("Shape file:", shape_file) # Save the rasters historic_raster = os.path.join( output, f"historic-{window_start.replace('-','_')}-{window_end.replace('-','_')}-{layer}-w_{agg_window}-h_{agg_history}-y_{historic_years}.tif", ) current_raster = os.path.join( output, f"current-{window_start.replace('-','_')}-{window_end.replace('-','_')}-{layer}-w_{agg_window}.tif", ) delta_raster = os.path.join( output, f"delta-{window_start.replace('-','_')}-{window_end.replace('-','_')}-{layer}-w_{agg_window}-h_{agg_history}-y_{historic_years}.tif", ) quant_raster = os.path.join( output, f"quantile-{window_start.replace('-','_')}-{window_end.replace('-','_')}-{layer}-w_{agg_window}-h_concat-y_{historic_years}.tif", ) save(historic_raster, historic_data, profile) save(current_raster, current_data, profile) save(delta_raster, delta, profile) save(quant_raster, quantile, profile) # Open the rasters # with rasterio.open(historic_raster) as src: # historic_raster = src.read(1) # Clip the rasters to the shape file if shape_file is not None: print("Found shape file:", shape_file) # shapes = read_shape_file( # shape_file, epsg_num=None, dissolve=False # ).geometry try: with fiona.open(shape_file, "r") as shapefile: shapes = [feature["geometry"] for feature in shapefile] except Exception as e: print("Error reading shape file:", e) try: with rasterio.open(historic_raster) as src: out_image, transformed = mask(src, shapes, crop=True, filled=True) out_profile = src.profile.copy() out_profile.update( { "width": out_image.shape[2], "height": out_image.shape[1], "transform": transformed, } ) with rasterio.open(historic_raster, "w", **out_profile) as dst: dst.write(out_image) except Exception as e: print("Error clipping historic raster:", e) try: with rasterio.open(current_raster) as src: out_image, transformed = mask(src, shapes, crop=True, filled=True) out_profile = src.profile.copy() out_profile.update( { "width": out_image.shape[2], "height": out_image.shape[1], "transform": transformed, } ) with rasterio.open(current_raster, "w", **out_profile) as dst: dst.write(out_image) except Exception as e: print("Error clipping current raster:", e) try: with rasterio.open(delta_raster) as src: out_image, transformed = mask(src, shapes, crop=True, filled=True) out_profile = src.profile.copy() out_profile.update( { "width": out_image.shape[2], "height": out_image.shape[1], "transform": transformed, } ) with rasterio.open(delta_raster, "w", **out_profile) as dst: dst.write(out_image) except Exception as e: print("Error clipping delta raster:", e) try: with rasterio.open(quant_raster) as src: out_image, transformed = mask(src, shapes, crop=True, filled=True) out_profile = src.profile.copy() out_profile.update( { "width": out_image.shape[2], "height": out_image.shape[1], "transform": transformed, } ) with rasterio.open(quant_raster, "w", **out_profile) as dst: dst.write(out_image) except Exception as e: print("Error clipping quantile raster:", e) # current_data, _ = rasterio.mask.mask(current_data, shapes, crop=True) # historic_data, _ = rasterio.mask.mask(historic_data, shapes, crop=True) # delta, _ = rasterio.mask.mask(delta, shapes, crop=True) # quantile, _ = rasterio.mask.mask(quantile, shapes, crop=True) print("done.") return { "historic_raster": historic_raster, "current_raster": current_raster, "delta_raster": delta_raster, "quant_raster": quant_raster, } def find_analyses(path): """Find all the analysis files in a directory. Parameters ---------- path: str Path to the directory containing the analysis files Returns ------- files: list List of analysis files """ files = [f for f in os.listdir(path) if f.endswith(".tif")] return files def open_image(path): """Open a raster image and return the data and coordinates. Parameters ---------- path: str path to the raster image Returns ------- band1: np.array The raster data lons: np.array The longitude coordinates lats: np.array The latitude coordinates """ with rasterio.open(path) as src: band1 = src.read(1) print("Band1 has shape", band1.shape) height = band1.shape[0] width = band1.shape[1] cols, rows = np.meshgrid(np.arange(width), np.arange(height)) xs, ys = rasterio.transform.xy(src.transform, rows, cols) lons = np.array(xs) lats = np.array(ys) return band1, lons, lats def perform_analysis( input, window_start, window_end, historic_years: int, layer: str, match_raster: str = None, output: str = None, agg_history: str = "mean", agg_window: str = "mean", comparison: str = "diff", farm_name: str = None, **args, ) -> Dict[str, str]: """Perform the analysis. This is a wrapper function for the analysis module. It takes the input parameters and passes them to the analysis module. Parameters ---------- input : str path to the input data window_start : str start date of the window window_end : str end date of the window historic_years : int number of years to use for the historic data layer : str layer to use for the analysis match_raster : str, optional path to the raster to match the output to, by default None output : str, optional path to the output file, by default None agg_history : str, optional aggregation method for the historic data, by default "mean" agg_window : str, optional aggregation method for the window data, by default "mean" comparison : str, optional comparison method for the window and historic data, by default "diff" Returns ------- files: dict Dict of analysis files """ files = analyse( input=input, window_start=window_start, window_end=window_end, historic_years=historic_years, agg_window=agg_window, agg_history=agg_history, comparison=comparison, layer=layer, output=output, match_raster=match_raster, farm_name=farm_name, ) return files def layout(WAIT_IMAGE): import datetime import plotly.express as px from dash import dcc, html today = datetime.datetime.today() colorscales = px.colors.named_colorscales() layout = html.Div( [ # html.Div( # className="dashapp-header", # children=[ # html.Div('Soil Moisture Comparison Tool', className="dashapp-header--title") # ] # ), dcc.Store(id="farm-name-session", storage_type="session"), html.Div( [ html.P( """This tool will use the produced datacubes to compare the soil moisture of a farm against historic data. Please select the desired comaprison method and dates to make the comparison as in section A. Then choose the visualisation in section B to see the results.""", style={"font-size": "larger"}, ), html.Hr(), html.H3("A"), ], className="col-lg-12", style={"padding-top": "1%", "padding-left": "1%"}, ), html.Div( [ html.Div( [ # html.P("Write farm name/ID:"), dcc.Input( id="farm-name", type="text", placeholder="Farm name", style={"width": "80%"}, ), html.Img( id="farm-image", src=WAIT_IMAGE, style={"width": "30px", "margin-left": "15px"}, ), ], className="col-lg-5", # style = {'padding-top':'1%', 'padding-left':'1%'} ), html.Div( [ html.P(), ], className="col-lg-7", # style = {'padding-top':'1%', 'padding-left':'1%'} ), ], className="row", style={"padding-top": "1%", "padding-left": "1%"}, ), html.Div( [ html.Div( [ html.P("Select soil layer:"), dcc.Dropdown( id="layer-dropdown", options=[ {"label": "SM1", "value": "SM1"}, {"label": "SM2", "value": "SM2"}, {"label": "SM3", "value": "SM3"}, {"label": "SM4", "value": "SM4"}, {"label": "SM5", "value": "SM5"}, {"label": "DD", "value": "DD"}, ], value="SM2", ), ], className="col-lg-4", style={"padding": "1%"}, ), html.Div( [ html.P("Select the historic years to compare against:"), dcc.Dropdown( id="historic-dropdown", options=[ {"label": year, "value": year} for year in range(1, 20) ], value=2, ), ], className="col-lg-4", style={"padding": "1%"}, ), html.Div( [ html.P( "Select the most recent window of dates to analyse:" ), dcc.DatePickerRange( id="window-select", min_date_allowed=datetime.date(2000, 1, 1), max_date_allowed=today.strftime("%Y-%m-%d"), initial_visible_month=datetime.date(2023, 1, 1), clearable=False, display_format="YYYY-MM-DD", start_date_placeholder_text="Start date", end_date_placeholder_text="End date", style={"width": "100%"}, ), ], className="col-lg-4", style={"padding": "1%"}, ), ], className="row", style={"padding-top": "1%"}, ), html.Div( [ html.Div( [ html.P("Select window aggregation method:"), dcc.Dropdown( id="w-aggregation-dropdown", options=[ {"label": "Mean", "value": "mean"}, {"label": "Median", "value": "median"}, {"label": "Max", "value": "max"}, {"label": "Min", "value": "min"}, {"label": "Sum", "value": "sum"}, {"label": "std", "value": "std"}, {"label": "var", "value": "var"}, ], value="mean", ), ], className="col-lg-6", style={"padding": "1%"}, ), html.Div( [ html.P("Select historic aggregation method:"), dcc.Dropdown( id="h-aggregation-dropdown", options=[ {"label": "Mean", "value": "mean"}, {"label": "Median", "value": "median"}, {"label": "Max", "value": "max"}, {"label": "Min", "value": "min"}, {"label": "Sum", "value": "sum"}, {"label": "std", "value": "std"}, {"label": "var", "value": "var"}, {"label": "quantile", "value": "quantile"}, ], value="mean", ), ], className="col-lg-6", style={"padding": "1%"}, ), ], className="row", # style = {'padding-top':'1%'} ), html.Div( [ html.Button("Generate Images", id="generate-button"), html.Br(), html.Hr(), ], className="col-lg-12", style={"margin-bottom": "1%"}, ), html.Div( [ html.H3("B"), ], className="col-lg-12", style={"padding-top": "1%", "padding-left": "1%"}, ), html.Div( [ html.Div( [ html.P("Select visualisation name:"), dcc.Dropdown(id="visualisation-select"), ], className="col-lg-6", style={"padding": "1%"}, ), html.Div( [ html.P("Select your palette:"), dcc.Dropdown( id="platter-dropdown", options=colorscales, value="viridis", ), ], className="col-lg-6", style={"padding": "1%"}, ), ], className="row", # style = {'padding-top':'1%'} ), html.Div( [ html.Hr(), html.H3("Results"), dcc.Graph(id="graph"), ], className="col-lg-12", style={"padding-top": "1%"}, ), # html.Div( # className="dashapp-footer", # children=[ # html.Div(f"Copyright @ {today.strftime('%Y')} Sydney Informatics Hub (SIH)", className="dashapp-footer--copyright") # ] # ), ], className="container-fluid", ) return layout