Sahand
mask method change
c8c4008
# -*- coding: utf-8 -*-
import gc
import glob
import os
from datetime import timedelta
from typing import Dict
import fiona
import geopandas as gpd
import netCDF4
import numpy as np
import pandas as pd
import rasterio
from rasterio.mask import mask
from shapely.validation import make_valid
from libs.utils import verbose as vprint
V = 1
V_IGNORE = [] # Debug, Warning, Error
def read_shape_file(
file_name: str,
epsg_num: int = 4326,
dissolve=True,
# buffer: float = 100,
*args,
**kwargs,
) -> gpd.GeoDataFrame:
"""Read a shape file and return a geo-dataframe object.
Parameters
----------
file_name : str
path to shape file
epsg_num : int, optional
coordinate system number
dissolve : bool, optional
Dissolve to one geometry, e.g. get the most external shape.
Useful when interested in the
Returns
-------
gpd.GeoDataFrame
pandas geo-dataframe with the geometry data and other infos
Raises
------
FileNotFoundError
error if file is not found
"""
if not os.path.exists(file_name):
raise FileNotFoundError
geo_df = gpd.read_file(file_name)
if epsg_num is not None:
geo_df["geometry"] = geo_df["geometry"].to_crs(epsg=epsg_num)
crs = geo_df.crs
if crs is None:
crs = 4326
print("Initial CRS:", crs)
# check for multi
geo_df.geometry = geo_df.apply(
lambda row: make_valid(row.geometry)
if not row.geometry.is_valid
else row.geometry,
axis=1,
)
return geo_df
def find_match_raster(model_path, farm_name):
# navigate one directory up in model_path
while not os.path.basename(model_path) == farm_name:
model_path = os.path.dirname(model_path)
root = model_path
print("Looking for a matching raster in:", root)
for root, dirs, _ in os.walk(root):
# print(root,dirs,"\n\n")
if "et_pp" in dirs:
real_path = os.path.join(root, "et_pp")
print("raster_path is", real_path)
# for file in os.listdir(real_path):
# if file.endswith(".tif"):
# match_raster = os.path.join(real_path,file)
# print("Found match raster:",match_raster)
return real_path
return None
def find_shape_file(model_path, farm_name):
# navigate one directory up in model_path until you reach the farm_name directory
while not os.path.basename(model_path) == farm_name:
model_path = os.path.dirname(model_path)
print("Looking for a shape file in:", model_path)
root = model_path
print("Looking for a shape file in:", root)
for root, dirs, files in os.walk(root):
# print(root,dirs,"\n\n")
for file in files:
if file.endswith(".shp"):
real_path = os.path.join(root, file)
return real_path
return None
def get_range_agg(
input_dir: str,
window_start: str,
window_end: str,
layer_name: str,
agg: str = "mean",
) -> np.ndarray:
"""Get the mean for a given window_start and window_end dates.
Parameters
----------
input_dir : str
Path to the directory containing the netcdf files.
window_start : str
Start date of the window. Format: YYYY-MM-DD.
window_end : str
End date of the window. Format: YYYY-MM-DD.
layer_name : str
Soil layer to consider for the mean.
agg : str
Aggregation method to use. Possible values: mean, median, max, min, std, or None.
Returns
-------
np.ndarray
Mean raster for the given window_start and window_end dates.
"""
# Get the list of dates between two dates if date_from and date_to
dates = pd.DataFrame(
pd.date_range(
pd.to_datetime(window_start),
pd.to_datetime(window_end) - timedelta(days=1),
freq="d",
),
columns=["date"],
) # .strftime('%Y-%m-%d')
dates["dayofyear"] = dates["date"].dt.dayofyear - 1
dates["year"] = dates["date"].dt.year
dates["str_dates"] = dates["date"].dt.strftime("%Y-%m-%d")
yearly_dates = dates.groupby("year")["dayofyear"].apply(list).to_dict()
data_l = list()
# For each year, get the data for layer_name for the dates specified in yearly_dates
for year in yearly_dates:
# read the year file
nc_y = netCDF4.Dataset(os.path.join(input_dir, f"model_{year}.nc"))
vprint(
1,
V,
V_IGNORE,
Debug=f"getting data for year: {year} from layer: {layer_name}...",
)
# Get the data for the layer_name
data = nc_y.variables[layer_name][:, :, :]
# Get the data for the dates
days = yearly_dates[year]
data = data[days, :, :]
data_l.append(data)
nc_y.close()
del data
gc.collect()
# Concat data for all years
data_concat = np.concatenate(data_l, axis=0)
data_concat.shape
if agg == "mean":
# Get the mean raster for the range
data_agg = np.mean(data_concat, axis=0)
elif agg == "median":
# Get the median raster for the range
data_agg = np.median(data_concat, axis=0)
elif agg == "max":
# Get the max raster for the range
data_agg = np.max(data_concat, axis=0)
elif agg == "min":
# Get the min raster for the range
data_agg = np.min(data_concat, axis=0)
elif agg == "std":
# Get the std raster for the range
data_agg = np.std(data_concat, axis=0)
elif agg == "var":
# Get the var raster for the range
data_agg = np.var(data_concat, axis=0)
elif agg == "sum":
# Get the sum raster for the range
data_agg = np.sum(data_concat, axis=0)
elif agg is None:
data_agg = data_concat.copy()
else:
raise ValueError(
f"agg should be one of 'mean', 'median', 'max', 'min', 'std', 'var', 'sum', or a None value. {agg} was provided."
)
print("done.")
return data_agg
def get_historic_agg(
input_dir: str,
historic_years: int,
current_window_start: str,
current_window_end: str,
layer_name: str,
agg_window: str = "mean",
agg_history: str = "mean",
) -> np.ndarray:
"""Get the historic mean for a given window_start and window_end dates.
Parameters
----------
input_dir : str
Path to the directory containing the netcdf files.
historic_years : int
Number of historic years to consider for the mean.
current_window_start : str
Start date of the current window. Format: YYYY-MM-DD.
current_window_end : str
End date of the current window. Format: YYYY-MM-DD.
layer_name : str
Soil layer to consider for the mean.
agg_window : str
Aggregation method for the window (applies to both current and historic). Default is "mean". Possible values: "mean", "median", "max", "min", "std", "var", None (None will just make a 3D matrix for each year).
agg_history : str
Aggregation method for the historic years. This defines the way all years are combined into one. Default is "mean". Possible values: "mean", "median", "max", "min", "std", "var", "quantiles".
If `quantile` is selected, the historic cube will be sent back as a 3D matrix (with non-agregated window and concatenated on axis 0 instead) for quantile comparison.
Returns
-------
np.ndarray
Array of the historic mean for the given window_start and window_end dates for the historic years.
Raises
------
FileNotFoundError
If the file for the historic year is not found. Possible solutions:
- The historic year should be modelled before calling this function.
- The path to the historic year should be changed.
- Calculate for a more recent historic year by reducing historic_years value.
"""
# Get the window_start year
window_start_year = pd.to_datetime(current_window_start).year
window_end_year = pd.to_datetime(current_window_end).year
# Get the first year
first_year = window_start_year - historic_years
# Check if file exists for this year
if os.path.exists(os.path.join(input_dir, f"model_{first_year}.nc")):
# Get the list of historic windows
historic_agg = {}
for year in range(1, historic_years + 1):
args = {
"input_dir": input_dir,
"window_start": f"{window_start_year-year}{current_window_start[4:]}",
"window_end": f"{window_end_year-year}{current_window_end[4:]}",
"layer_name": layer_name,
"agg": agg_window,
}
# Get the range mean
historic_agg[window_start_year - year] = get_range_agg(**args)
historic_agg_np = np.array([historic_agg[year] for year in historic_agg])
# Get the aggregation of the historic years
if agg_history == "mean":
historic_agg_np = np.mean(historic_agg_np, axis=0)
elif agg_history == "median":
historic_agg_np = np.median(historic_agg_np, axis=0)
elif agg_history == "max":
historic_agg_np = np.max(historic_agg_np, axis=0)
elif agg_history == "min":
historic_agg_np = np.min(historic_agg_np, axis=0)
elif agg_history == "std":
historic_agg_np = np.std(historic_agg_np, axis=0)
elif agg_history == "var":
historic_agg_np = np.var(historic_agg_np, axis=0)
elif agg_history == "sum":
historic_agg_np = np.sum(historic_agg_np, axis=0)
elif agg_history is None:
historic_agg_np = np.concatenate(list(historic_agg.values()), axis=0)
else:
raise ValueError(
f"Invalid aggregation method: {agg_history}. Possible values: mean, median, max, min, std, var, sum."
)
return historic_agg_np
else:
raise FileNotFoundError(
f"File not found for the historic data: {os.path.join(input_dir,f'model_{first_year}.nc')}. Make sure the path is correct and the historic year for the requested year is modelled before calling this function."
)
def save(path, array, profile):
"""Save the array as a raster.
Parameters
----------
path : str
Path to the raster to save.
array : np.ndarray
Array to save as a raster.
profile : dict
Profile of the raster to save.
"""
with rasterio.open(path, "w", **profile) as dst:
dst.write(array, 1)
def analyse(
input,
window_start,
window_end,
historic_years: int,
layer: str,
match_raster: str = None,
output: str = None,
agg_history: str = "mean",
agg_window: str = "mean",
farm_name: str = None,
**kwargs,
) -> Dict[str, str]:
"""Main function to run the script.
Parameters
----------
input : str
Path to the input raster.
window_start : str
Start date of the window. Format: YYYY-MM-DD.
window_end : str
End date of the window. Format: YYYY-MM-DD.
historic_years : int
Number of historic years to use for the comparison.
layer : str
Soil layer to consider for the comparison.
match_raster : str
Path to the match raster. Default: None. If None, the match raster will be searched in the et_pp directory based on the input directory.
output : str
Path to the output raster. Default: None. If None, the output raster will be saved in the same directory as the input raster.
agg_history : str
Aggregation method to use for the historic years. Possible values: 'mean', 'median', 'max', 'min', 'std', None. Default: 'mean'.
agg_window : str
Aggregation method to use for the window. Possible values: 'mean', 'median', 'max', 'min', 'std', None. Default: 'mean'.
farm_name : str
Name of the farm. Should be provided. Default: None.
Returns
-------
Dict[str,str]
Dictionary with the path to the output rasters.
"""
if output is None:
output = os.path.join(input, "analysis")
# Create the output directory if it does not exist
if not os.path.exists(output):
os.makedirs(output)
if match_raster is None:
match_raster = find_match_raster(input, farm_name)
print("match_raster is:", match_raster)
if match_raster is not None:
print("Found match raster:", match_raster)
files = glob.glob(os.path.join(match_raster, f"{window_start[:7]}*.tif"))
if len(files) == 0:
files = glob.glob(os.path.join(match_raster, f"{window_end[:7]}*.tif"))
if len(files) == 0:
vprint(
1,
V,
V_IGNORE,
Debug=f"Expanding the search for match raster file to find e closer date to {window_start[:5]}...",
)
files = glob.glob(os.path.join(match_raster, f"{window_start[:5]}*.tif"))
if len(files) == 0:
vprint(
1,
V,
V_IGNORE,
Debug=f"Expanding the search further for match raster file to find e closer date to {window_end[:5]}...",
)
files = glob.glob(os.path.join(match_raster, f"{window_end[:5]}*.tif"))
if len(files) == 0:
raise FileNotFoundError(
f"Could not find any matching raster in {match_raster} for the rage of dates given at {window_start} / {window_end}!"
)
print(f"Found {len(files)} matching raster file {files[0]}.")
match_raster = files[0]
with rasterio.open(match_raster) as src:
profile = src.profile
# Get the layers
layer = layer
# Get the historic aggregated data
# Get aggregated current window data
current_data = get_range_agg(
input_dir=input,
window_start=window_start,
window_end=window_end,
agg=agg_window,
layer_name=layer,
)
historic_data = get_historic_agg(
input_dir=input,
historic_years=historic_years,
current_window_start=window_start,
current_window_end=window_end,
agg_window=agg_window,
agg_history=agg_history,
layer_name=layer,
)
historic_data_quant = get_historic_agg(
input_dir=input,
historic_years=historic_years,
current_window_start=window_start,
current_window_end=window_end,
agg_window=None,
agg_history=None,
layer_name=layer,
)
# Compare the two rasters
delta = current_data - historic_data
quantile = current_data.copy()
print("\nCalculating quantiles...", "\n=========================")
print("Data shape:", current_data.shape)
print("Historic data shape:", historic_data_quant.shape)
print("=========================\n")
pixel_now = []
pixel_hist = []
pixel_quant = []
for i in range(current_data.shape[0]):
for j in range(current_data.shape[1]):
sorted_scores = np.array(sorted(historic_data_quant[:, i, j]))
quantile[i, j] = (
(sorted_scores.searchsorted(current_data[i, j]))
/ sorted_scores.shape[0]
* 100
)
pixel_now.append(current_data[i, j])
pixel_hist.append(sorted_scores)
pixel_quant.append(quantile[i, j])
df_quants = pd.DataFrame(pixel_hist)
df_quants["pixel_now"] = pixel_now
df_quants["pixel_quant"] = pixel_quant
print("shapes are:", len(pixel_now), len(pixel_quant))
df_quants = df_quants.sort_values(by=["pixel_quant"], ascending=False)
print(df_quants)
df_quants.to_csv(
os.path.join(
output,
f"quantiles-{window_start.replace('-','_')}-{window_end.replace('-','_')}-{layer}-w_{agg_window}-h_concat-y_{historic_years}.csv",
)
)
# Search for a shape file ending with .shp
shape_file = find_shape_file(input, farm_name)
print("Shape file:", shape_file)
# Save the rasters
historic_raster = os.path.join(
output,
f"historic-{window_start.replace('-','_')}-{window_end.replace('-','_')}-{layer}-w_{agg_window}-h_{agg_history}-y_{historic_years}.tif",
)
current_raster = os.path.join(
output,
f"current-{window_start.replace('-','_')}-{window_end.replace('-','_')}-{layer}-w_{agg_window}.tif",
)
delta_raster = os.path.join(
output,
f"delta-{window_start.replace('-','_')}-{window_end.replace('-','_')}-{layer}-w_{agg_window}-h_{agg_history}-y_{historic_years}.tif",
)
quant_raster = os.path.join(
output,
f"quantile-{window_start.replace('-','_')}-{window_end.replace('-','_')}-{layer}-w_{agg_window}-h_concat-y_{historic_years}.tif",
)
save(historic_raster, historic_data, profile)
save(current_raster, current_data, profile)
save(delta_raster, delta, profile)
save(quant_raster, quantile, profile)
# Open the rasters
# with rasterio.open(historic_raster) as src:
# historic_raster = src.read(1)
# Clip the rasters to the shape file
if shape_file is not None:
print("Found shape file:", shape_file)
# shapes = read_shape_file(
# shape_file, epsg_num=None, dissolve=False
# ).geometry
try:
with fiona.open(shape_file, "r") as shapefile:
shapes = [feature["geometry"] for feature in shapefile]
except Exception as e:
print("Error reading shape file:", e)
try:
with rasterio.open(historic_raster) as src:
out_image, transformed = mask(src, shapes, crop=True, filled=True)
out_profile = src.profile.copy()
out_profile.update(
{
"width": out_image.shape[2],
"height": out_image.shape[1],
"transform": transformed,
}
)
with rasterio.open(historic_raster, "w", **out_profile) as dst:
dst.write(out_image)
except Exception as e:
print("Error clipping historic raster:", e)
try:
with rasterio.open(current_raster) as src:
out_image, transformed = mask(src, shapes, crop=True, filled=True)
out_profile = src.profile.copy()
out_profile.update(
{
"width": out_image.shape[2],
"height": out_image.shape[1],
"transform": transformed,
}
)
with rasterio.open(current_raster, "w", **out_profile) as dst:
dst.write(out_image)
except Exception as e:
print("Error clipping current raster:", e)
try:
with rasterio.open(delta_raster) as src:
out_image, transformed = mask(src, shapes, crop=True, filled=True)
out_profile = src.profile.copy()
out_profile.update(
{
"width": out_image.shape[2],
"height": out_image.shape[1],
"transform": transformed,
}
)
with rasterio.open(delta_raster, "w", **out_profile) as dst:
dst.write(out_image)
except Exception as e:
print("Error clipping delta raster:", e)
try:
with rasterio.open(quant_raster) as src:
out_image, transformed = mask(src, shapes, crop=True, filled=True)
out_profile = src.profile.copy()
out_profile.update(
{
"width": out_image.shape[2],
"height": out_image.shape[1],
"transform": transformed,
}
)
with rasterio.open(quant_raster, "w", **out_profile) as dst:
dst.write(out_image)
except Exception as e:
print("Error clipping quantile raster:", e)
# current_data, _ = rasterio.mask.mask(current_data, shapes, crop=True)
# historic_data, _ = rasterio.mask.mask(historic_data, shapes, crop=True)
# delta, _ = rasterio.mask.mask(delta, shapes, crop=True)
# quantile, _ = rasterio.mask.mask(quantile, shapes, crop=True)
print("done.")
return {
"historic_raster": historic_raster,
"current_raster": current_raster,
"delta_raster": delta_raster,
"quant_raster": quant_raster,
}
def find_analyses(path):
"""Find all the analysis files in a directory.
Parameters
----------
path: str
Path to the directory containing the analysis files
Returns
-------
files: list
List of analysis files
"""
files = [f for f in os.listdir(path) if f.endswith(".tif")]
return files
def open_image(path):
"""Open a raster image and return the data and coordinates.
Parameters
----------
path: str
path to the raster image
Returns
-------
band1: np.array
The raster data
lons: np.array
The longitude coordinates
lats: np.array
The latitude coordinates
"""
with rasterio.open(path) as src:
band1 = src.read(1)
print("Band1 has shape", band1.shape)
height = band1.shape[0]
width = band1.shape[1]
cols, rows = np.meshgrid(np.arange(width), np.arange(height))
xs, ys = rasterio.transform.xy(src.transform, rows, cols)
lons = np.array(xs)
lats = np.array(ys)
return band1, lons, lats
def perform_analysis(
input,
window_start,
window_end,
historic_years: int,
layer: str,
match_raster: str = None,
output: str = None,
agg_history: str = "mean",
agg_window: str = "mean",
comparison: str = "diff",
farm_name: str = None,
**args,
) -> Dict[str, str]:
"""Perform the analysis.
This is a wrapper function for the analysis module. It takes the input parameters and passes them to the analysis module.
Parameters
----------
input : str
path to the input data
window_start : str
start date of the window
window_end : str
end date of the window
historic_years : int
number of years to use for the historic data
layer : str
layer to use for the analysis
match_raster : str, optional
path to the raster to match the output to, by default None
output : str, optional
path to the output file, by default None
agg_history : str, optional
aggregation method for the historic data, by default "mean"
agg_window : str, optional
aggregation method for the window data, by default "mean"
comparison : str, optional
comparison method for the window and historic data, by default "diff"
Returns
-------
files: dict
Dict of analysis files
"""
files = analyse(
input=input,
window_start=window_start,
window_end=window_end,
historic_years=historic_years,
agg_window=agg_window,
agg_history=agg_history,
comparison=comparison,
layer=layer,
output=output,
match_raster=match_raster,
farm_name=farm_name,
)
return files
def layout(WAIT_IMAGE):
import datetime
import plotly.express as px
from dash import dcc, html
today = datetime.datetime.today()
colorscales = px.colors.named_colorscales()
layout = html.Div(
[
# html.Div(
# className="dashapp-header",
# children=[
# html.Div('Soil Moisture Comparison Tool', className="dashapp-header--title")
# ]
# ),
dcc.Store(id="farm-name-session", storage_type="session"),
html.Div(
[
html.P(
"""This tool will use the produced datacubes to compare the soil moisture of a farm against historic data.
Please select the desired comaprison method and dates to make the comparison as in section A.
Then choose the visualisation in section B to see the results.""",
style={"font-size": "larger"},
),
html.Hr(),
html.H3("A"),
],
className="col-lg-12",
style={"padding-top": "1%", "padding-left": "1%"},
),
html.Div(
[
html.Div(
[
# html.P("Write farm name/ID:"),
dcc.Input(
id="farm-name",
type="text",
placeholder="Farm name",
style={"width": "80%"},
),
html.Img(
id="farm-image",
src=WAIT_IMAGE,
style={"width": "30px", "margin-left": "15px"},
),
],
className="col-lg-5",
# style = {'padding-top':'1%', 'padding-left':'1%'}
),
html.Div(
[
html.P(),
],
className="col-lg-7",
# style = {'padding-top':'1%', 'padding-left':'1%'}
),
],
className="row",
style={"padding-top": "1%", "padding-left": "1%"},
),
html.Div(
[
html.Div(
[
html.P("Select soil layer:"),
dcc.Dropdown(
id="layer-dropdown",
options=[
{"label": "SM1", "value": "SM1"},
{"label": "SM2", "value": "SM2"},
{"label": "SM3", "value": "SM3"},
{"label": "SM4", "value": "SM4"},
{"label": "SM5", "value": "SM5"},
{"label": "DD", "value": "DD"},
],
value="SM2",
),
],
className="col-lg-4",
style={"padding": "1%"},
),
html.Div(
[
html.P("Select the historic years to compare against:"),
dcc.Dropdown(
id="historic-dropdown",
options=[
{"label": year, "value": year}
for year in range(1, 20)
],
value=2,
),
],
className="col-lg-4",
style={"padding": "1%"},
),
html.Div(
[
html.P(
"Select the most recent window of dates to analyse:"
),
dcc.DatePickerRange(
id="window-select",
min_date_allowed=datetime.date(2000, 1, 1),
max_date_allowed=today.strftime("%Y-%m-%d"),
initial_visible_month=datetime.date(2023, 1, 1),
clearable=False,
display_format="YYYY-MM-DD",
start_date_placeholder_text="Start date",
end_date_placeholder_text="End date",
style={"width": "100%"},
),
],
className="col-lg-4",
style={"padding": "1%"},
),
],
className="row",
style={"padding-top": "1%"},
),
html.Div(
[
html.Div(
[
html.P("Select window aggregation method:"),
dcc.Dropdown(
id="w-aggregation-dropdown",
options=[
{"label": "Mean", "value": "mean"},
{"label": "Median", "value": "median"},
{"label": "Max", "value": "max"},
{"label": "Min", "value": "min"},
{"label": "Sum", "value": "sum"},
{"label": "std", "value": "std"},
{"label": "var", "value": "var"},
],
value="mean",
),
],
className="col-lg-6",
style={"padding": "1%"},
),
html.Div(
[
html.P("Select historic aggregation method:"),
dcc.Dropdown(
id="h-aggregation-dropdown",
options=[
{"label": "Mean", "value": "mean"},
{"label": "Median", "value": "median"},
{"label": "Max", "value": "max"},
{"label": "Min", "value": "min"},
{"label": "Sum", "value": "sum"},
{"label": "std", "value": "std"},
{"label": "var", "value": "var"},
{"label": "quantile", "value": "quantile"},
],
value="mean",
),
],
className="col-lg-6",
style={"padding": "1%"},
),
],
className="row",
# style = {'padding-top':'1%'}
),
html.Div(
[
html.Button("Generate Images", id="generate-button"),
html.Br(),
html.Hr(),
],
className="col-lg-12",
style={"margin-bottom": "1%"},
),
html.Div(
[
html.H3("B"),
],
className="col-lg-12",
style={"padding-top": "1%", "padding-left": "1%"},
),
html.Div(
[
html.Div(
[
html.P("Select visualisation name:"),
dcc.Dropdown(id="visualisation-select"),
],
className="col-lg-6",
style={"padding": "1%"},
),
html.Div(
[
html.P("Select your palette:"),
dcc.Dropdown(
id="platter-dropdown",
options=colorscales,
value="viridis",
),
],
className="col-lg-6",
style={"padding": "1%"},
),
],
className="row",
# style = {'padding-top':'1%'}
),
html.Div(
[
html.Hr(),
html.H3("Results"),
dcc.Graph(id="graph"),
],
className="col-lg-12",
style={"padding-top": "1%"},
),
# html.Div(
# className="dashapp-footer",
# children=[
# html.Div(f"Copyright @ {today.strftime('%Y')} Sydney Informatics Hub (SIH)", className="dashapp-footer--copyright")
# ]
# ),
],
className="container-fluid",
)
return layout