initial commit

CHANGELON.md

@@ -0,0 +1,12 @@
+## 09/08/2023
+WIP - services are being added. Added a draft of job management service.
+
+
+## 08/08/2023
+
+* Added diagrams
+* Added db model
+* Dash App is a working prototype. To be added is the quantiles.
+* Added a new file to the repo: `requirements.txt` to be used for deployment.
+* Intial app structure has gone through design and a draft is ready.
+* There are currently 2 versions of the app. One is messy and works. THe app-v2 is a cleaner version but is not working yet.

Dockerfile.dash

@@ -0,0 +1,6 @@
+FROM python:3.9
+WORKDIR /
+COPY ./requirements/requirements-dev-vis.txt /requirements-dev-vis.txt
+RUN pip install --no-cache-dir --upgrade -r /requirements-dev-vis.txt
+COPY . .
+CMD ["python", "app.py","--input","Data/{}"]

admin.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+from flask import Flask, render_template
+from flask_admin import Admin
+from flask_sqlalchemy import SQLAlchemy
+
+# Consts
+SQLPROTOCOL = "mysql://"  # "sqlite:////"
+SQLUNAME = "root"
+SQLPASS = "Sahy1990!"
+SQLHOST = "localhost"
+SQLDB = "nowcasting"
+
+# App
+app = Flask(__name__)
+app.config["SECRET_KEY"] = "your-secret-key"
+app.config[
+    "SQLALCHEMY_DATABASE_URI"
+] = f"{SQLPROTOCOL}{SQLUNAME}:{SQLPASS}@{SQLHOST}/{SQLDB}"
+db = SQLAlchemy(app)
+admin = Admin(app, name="My Admin Panel", template_mode="bootstrap4")
+
+
+@app.route("/")
+def home():
+    return render_template(
+        "admin.html"
+    )  # "Welcome to Nowcasting App.<br><br>The dashapp is still under development..."
+
+
+if __name__ == "__main__":
+    app.run(debug=True)

app-v2.py

@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+import argparse
+import logging
+import os
+import traceback
+
+import routes
+from flask import Flask
+from flask_sqlalchemy import SQLAlchemy
+
+from libs.utils import setup_logging
+
+# from marshmallow import Schema, fields
+
+
+setup_logging()
+log = logging.getLogger(__name__)
+
+
+def create_the_database(db):
+    db.create_all()
+
+
+# ===============================================================================
+
+SQLPROTOCOL = "mysql://"  # "sqlite:////"
+SQLUNAME = "root"
+SQLPASS = ""
+SQLHOST = "localhost"
+SQLDB = "nowcasting"
+
+app = Flask(__name__)
+app.config["SECRET_KEY"] = "A secret for nowcasting in SIH"
+
+all_methods = ["GET", "POST"]
+
+# Home page (where you will add a new user)
+app.add_url_rule("/", view_func=routes.index)
+# "Thank you for submitting your form" page
+app.add_url_rule("/submitted", methods=all_methods, view_func=routes.submitted)
+# Viewing all the content in the database page
+app.add_url_rule("/database", view_func=routes.view_database)
+app.add_url_rule(
+    "/modify<the_id>/<modified_category>",
+    methods=all_methods,
+    view_func=routes.modify_database,
+)
+app.add_url_rule("/delete<the_id>", methods=all_methods, view_func=routes.delete)
+
+app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False  # no warning messages
+app.config[
+    "SQLALCHEMY_DATABASE_URI"
+] = "${SQLPROTOCOL}${SQLUNAME}:${SQLPASS}@${SQLHOST}/${SQLDB}"
+db = SQLAlchemy(app)
+
+# ===============================================================================
+
+
+# ===============================================================================
+
+if __name__ == "__main__":
+    # Load Configs
+    parser = argparse.ArgumentParser(
+        description="Download rainfall data from Google Earth Engine for a range of dates.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        "-i",
+        "--input",
+        help="Absolute or relative path to the netcdf data directory for each farm. Should be in this format: '/path/to/farm/{}/soilwatermodel'",
+        default=os.path.join(
+            os.path.expanduser("~"), "Data/results_default/{}/soilwatermodel"
+        ),
+    )
+    parser.add_argument(
+        "-d",
+        "--debug",
+        help="Debug mode as True or False. Default is True.",
+        default=True,
+    )
+
+    args = parser.parse_args()
+    INPUT = args.input
+
+    try:
+        # dashapp.run_server(debug=args.debug)
+        app.run(debug=args.debug)
+    except Exception as e:
+        print("Error in main.py:", e)
+        traceback.print_exc()
+        raise e

app.py

@@ -0,0 +1,359 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import argparse
+import datetime
+import logging
+import os
+import traceback
+from typing import Dict, List
+
+# import dash_html_components as html
+# import flask
+import numpy as np
+import plotly.express as px
+from dash import Dash, Input, Output, State  # , dcc  # ,html
+from dash.exceptions import PreventUpdate
+from flask import Flask, render_template
+
+from libs.dashapp import layout, open_image, perform_analysis
+from libs.utils import setup_logging
+from libs.utils import verbose as vprint
+
+setup_logging()
+log = logging.getLogger(__name__)
+CONFIG = {}
+V = 1
+V_IGNORE = []  # Debug, Warning, Error
+MODEL_PATH = ""
+
+# ===============================================================================
+# The Main App
+# ===============================================================================
+app = Flask(__name__)
+
+
+@app.route("/")
+def home():
+    return render_template(
+        "index.html"
+    )  # "Welcome to Nowcasting App.<br><br>The dashapp is still under development..."
+
+
+@app.route("/register")
+def register():
+    return "Welcome to Nowcasting signup page.<br><br>The dashapp is still under development..."
+
+
+@app.route("/addjob")
+def addjob():
+    return "Welcome to Nowcasting signup page.<br><br>The dashapp is still under development..."
+
+
+@app.route("/dashboard")
+def dashboard():
+    return "Welcome to Nowcasting dashboard page.<br><br>This page will contain dashboard for the prediction pipelines, and allow you to manage and add new predictions.<br>The dashapp is still under development..."
+    # dashapp
+
+
+# ===============================================================================
+# Soil Moisture Comparison Tool App Layout
+# ===============================================================================
+# external JavaScript files
+external_scripts = [
+    "https://www.google-analytics.com/analytics.js",
+    {"src": "https://cdn.polyfill.io/v2/polyfill.min.js"},
+    {
+        "src": "https://cdnjs.cloudflare.com/ajax/libs/lodash.js/4.17.10/lodash.core.js",
+        "integrity": "sha256-Qqd/EfdABZUcAxjOkMi8eGEivtdTkh3b65xCZL4qAQA=",
+        "crossorigin": "anonymous",
+    },
+]
+
+# external CSS stylesheets
+external_stylesheets = [
+    "https://codepen.io/chriddyp/pen/bWLwgP.css",
+    {
+        "href": "https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css",
+        "rel": "stylesheet",
+        "integrity": "sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO",
+        "crossorigin": "anonymous",
+    },
+]
+
+
+dashapp = Dash(
+    __name__,
+    server=app,
+    routes_pathname_prefix="/dashapp/",
+    external_scripts=external_scripts,
+    external_stylesheets=external_stylesheets,
+    title="Soil Moisture Comparison Tool",
+    update_title="Loading the tool...",
+)
+
+# farm_name = "Arawa"
+# layer = "SM2"
+
+time_delta = datetime.timedelta(days=20)
+FAIL_IMAGE = dashapp.get_asset_url("icons/fail.png")
+SUCCESS_IMAGE = dashapp.get_asset_url("icons/success.png")
+WAIT_IMAGE = dashapp.get_asset_url("icons/wait.png")
+
+current_working_directory = os.getcwd()
+dashapp.index_template = os.path.join(
+    current_working_directory, "templates", "dashapp.html"
+)
+dashapp.layout = layout(WAIT_IMAGE)
+
+
+def find_model_path(real_path):
+    for root, dirs, _ in os.walk(real_path):
+        # print(root,dirs,"\n\n")
+        if "soilwatermodel" in dirs:
+            real_path = os.path.join(root, "soilwatermodel")
+            print("real_path is", real_path)
+            return real_path
+    return real_path
+
+
+# ====================================================================================================
+# Callbacks
+# ====================================================================================================
+@dashapp.callback(
+    [
+        Output("farm-name-session", "data"),
+        Output("farm-image", "src"),
+    ],
+    [Input("farm-name", "value"), State("farm-name-session", "data")],
+)
+def update_session(farm_name, session):
+    global MODEL_PATH
+    session = farm_name
+    if farm_name is None or farm_name == "":
+        session = ""
+        image = WAIT_IMAGE
+    else:
+        print(f"Getting some data about farm: {farm_name}")
+
+        # if the path does not exist, do not update the session
+        real_path = INPUT.format(farm_name)
+
+        print(f"Checking: {real_path}")
+        try:
+            path = find_model_path(real_path)
+            real_path = path
+        except Exception:
+            print("Exception raised while searching for the root")
+
+        MODEL_PATH = path
+
+        if os.path.exists(real_path):
+            session = farm_name
+            image = SUCCESS_IMAGE
+        else:
+            session = ""
+            image = FAIL_IMAGE
+
+    print(f"\n\nSession updated to {session}")
+    print(f"Image updated to {image}\n\n")
+
+    return session, image
+
+
+@dashapp.callback(
+    Output("farm-name", "value"),
+    Input("farm-name-session", "modified_timestamp"),
+    State("farm-name-session", "data"),
+)
+def display_name_from_session(timestamp, name):
+    print(f"Updating the farm name from the session: {name}")
+    if timestamp is not None:
+        return name
+    else:
+        return ""
+
+
+@dashapp.callback(
+    Output("visualisation-select", "options"),
+    # Input("farm-name", "value"),
+    Input("layer-dropdown", "value"),
+    Input("window-select", "start_date"),
+    Input("window-select", "end_date"),
+    Input("historic-dropdown", "value"),
+    Input("w-aggregation-dropdown", "value"),
+    Input("h-aggregation-dropdown", "value"),
+    Input("generate-button", "n_clicks"),
+    State("farm-name-session", "data"),
+)
+def get_analysis(
+    layer, window_start, window_end, historic_years, w_agg, h_agg, n_clicks, farm_name
+) -> List[Dict[str, str]]:
+    """Get the analysis files and return them as a list of dicts.
+
+    Parameters
+    ----------
+    layer : str
+        layer to use for the analysis
+    window_start : str
+        start date of the window
+    window_end : str
+        end date of the window
+    historic_years : int
+        number of years to use for the historic data
+    w_agg : str
+        aggregation method for the window data
+    h_agg : str
+        aggregation method for the historic data
+    n_clicks : int
+        number of times the generate button has been clicked
+
+    Returns
+    -------
+    files : list
+        list of dicts of analysis files
+    """
+    global MODEL_PATH
+    print("\nAnalysis callback triggered")
+
+    if n_clicks == 0 or n_clicks is None:
+        raise PreventUpdate
+
+    # window_start = datetime.datetime.strptime(window_start, '%Y-%m-%d')
+    # window_end = datetime.datetime.strptime(window_end, '%Y-%m-%d')
+    print(f"\nPath: {MODEL_PATH}\n")
+
+    files = perform_analysis(
+        input=MODEL_PATH,
+        window_start=window_start,
+        window_end=window_end,
+        historic_years=historic_years,
+        layer=layer,
+        agg_window=w_agg,
+        agg_history=h_agg,
+        comparison="diff",
+        output=None,
+        match_raster=None,
+    )
+
+    print(MODEL_PATH)
+    print(
+        f"n_clicks: {n_clicks}\n"
+        + f"window_start: {window_start}\n"
+        + f"window_end: {window_end}\n"
+        + f"historic_years: {historic_years}\n"
+        + f"layer: {layer}\n"
+        + f"agg_window: {w_agg}\n"
+        + f"agg_history: {h_agg}\n"
+        + "comparison: 'diff'\n"
+        + f"output: {None}\n"
+        + f"match_raster: {None}\n"
+    )
+    print(files)
+    files = {
+        i: [
+            " ".join(files[i].split("/")[-1].split(".")[0].split("-")).capitalize(),
+            files[i],
+        ]
+        for i in files
+    }
+    print(files)
+    options = [{"label": files[i][0], "value": files[i][1]} for i in files]
+
+    return options
+
+
+@dashapp.callback(
+    Output("graph", "figure"),
+    Input("visualisation-select", "value"),
+    Input("platter-dropdown", "value"),
+    Input("generate-button", "n_clicks"),
+)
+def change_colorscale(file, palette, n_clicks):
+    """Display the selected visualisation and change the colorscale of the
+    visualisation.
+
+    Parameters
+    ----------
+    file : str
+        path to the visualisation file
+    palette : str
+        name of the colorscale to use
+
+    Returns
+    -------
+    fig : plotly.graph_objects.Figure
+        plotly figure object
+    """
+    if n_clicks == 0 or n_clicks is None or file is None:
+        raise PreventUpdate
+
+    band1, lons_a, lats_a = open_image(file)
+
+    # Get the second dimension of the lons
+    lats = lats_a[:, 0]
+    lons = lons_a[0, :]
+    if "quantile" in file:
+        value_type = "Percentile"
+    else:
+        value_type = "SM"
+
+    print(lons.shape, lons)
+    print(lats.shape, lats)
+    print(band1.shape, band1)
+    print(file)
+    fig = px.imshow(band1, x=lons, y=lats, color_continuous_scale=palette)
+    fig.update(
+        data=[
+            {
+                "customdata": np.stack((band1, lats_a, lons_a), axis=-1),
+                "hovertemplate": f"<b>{value_type}"
+                + "</b>: %{customdata[0]}<br>"
+                + "<b>Lat</b>: %{customdata[1]}<br>"
+                + "<b>Lon</b>: %{customdata[2]}<br>"
+                + "<extra></extra>",
+            }
+        ]
+    )
+    print("Render successful")
+    return fig
+
+
+# ==============================================================================
+# Main
+# ==============================================================================
+
+if __name__ == "__main__":
+    # Load Configs
+    parser = argparse.ArgumentParser(
+        description="Download rainfall data from Google Earth Engine for a range of dates.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        "-i",
+        "--input",
+        help="Absolute or relative path to the netcdf data directory for each farm. Should be in this format: '/path/to/farm/{}/soilwatermodel'",
+        default=os.path.join(os.path.expanduser("~"), "Data/results_default/{}"),
+    )
+    parser.add_argument(
+        "-d",
+        "--debug",
+        help="Debug mode as True or False. Default is True.",
+        default=True,
+    )
+
+    args = parser.parse_args()
+    INPUT = args.input
+
+    try:
+        # dashapp.run_server(debug=args.debug)
+        app.run(debug=args.debug)
+    except Exception as e:
+        vprint(
+            0,
+            V,
+            V_IGNORE,
+            Error="Failed to execute the main function:",
+            ErrorMessage=e,
+        )
+        traceback.print_exc()
+        raise e

app.yaml

@@ -0,0 +1,15 @@
+# Copyright 2023 Sydney Informatics Hub (SIH)
+
+runtime: python39
+
+handlers:
+  # This configures Google App Engine to serve the files in the app's static
+  # directory.
+- url: /static
+  static_dir: static
+
+  # This handler routes all requests not caught above to your main app. It is
+  # required when static routes are defined, but can be omitted (along with
+  # the entire handlers section) when there are no static files defined.
+- url: /.*
+  script: auto

libs/dashapp.py

@@ -0,0 +1,947 @@
+# -*- coding: utf-8 -*-
+import gc
+import glob
+import os
+from datetime import timedelta
+from typing import Dict
+
+import fiona
+import geopandas as gpd
+import netCDF4
+import numpy as np
+import pandas as pd
+import rasterio
+from shapely.validation import make_valid
+
+from libs.utils import verbose as vprint
+
+V = 1
+V_IGNORE = []  # Debug, Warning, Error
+
+
+def read_shape_file(
+    file_name: str,
+    epsg_num: int = 4326,
+    dissolve=True,
+    # buffer: float = 100,
+    *args,
+    **kwargs,
+) -> gpd.GeoDataFrame:
+    """Read a shape file and return a geo-dataframe object.
+
+    Parameters
+    ----------
+    file_name : str
+        path to shape file
+    epsg_num : int, optional
+        coordinate system number
+    dissolve : bool, optional
+        Dissolve to one geometry, e.g. get the most external shape.
+        Useful when interested in the
+
+    Returns
+    -------
+    gpd.GeoDataFrame
+        pandas geo-dataframe with the geometry data and other infos
+
+    Raises
+    ------
+    FileNotFoundError
+        error if file is not found
+    """
+    if not os.path.exists(file_name):
+        raise FileNotFoundError
+
+    geo_df = gpd.read_file(file_name)
+    if epsg_num is not None:
+        geo_df["geometry"] = geo_df["geometry"].to_crs(epsg=epsg_num)
+
+    crs = geo_df.crs
+    if crs is None:
+        crs = 4326
+
+    print("Initial CRS:", crs)
+
+    # check for multi
+    geo_df.geometry = geo_df.apply(
+        lambda row: make_valid(row.geometry)
+        if not row.geometry.is_valid
+        else row.geometry,
+        axis=1,
+    )
+    return geo_df
+
+
+def find_match_raster(model_path):
+    # navigate one directory up in model_path
+    root = os.path.dirname(model_path)
+    root = os.path.dirname(root)
+    root = os.path.dirname(root)
+    print("Looking for a matching raster in:", root)
+    for root, dirs, _ in os.walk(root):
+        # print(root,dirs,"\n\n")
+        if "et_pp" in dirs:
+            real_path = os.path.join(root, "et_pp")
+            print("raster_path is", real_path)
+            # for file in os.listdir(real_path):
+            #     if file.endswith(".tif"):
+            #         match_raster = os.path.join(real_path,file)
+            #         print("Found match raster:",match_raster)
+            return real_path
+    return None
+
+
+def find_shape_file(model_path):
+    # navigate one directory up in model_path
+    root = os.path.dirname(model_path)
+    root = os.path.dirname(root)
+    root = os.path.dirname(root)
+    print("Looking for a shape file in:", root)
+    for root, dirs, files in os.walk(root):
+        # print(root,dirs,"\n\n")
+        for file in files:
+            if file.endswith(".shp"):
+                real_path = os.path.join(root, file)
+                return real_path
+    return None
+
+
+def get_range_agg(
+    input_dir: str,
+    window_start: str,
+    window_end: str,
+    layer_name: str,
+    agg: str = "mean",
+) -> np.ndarray:
+    """Get the mean for a given window_start and window_end dates.
+
+    Parameters
+    ----------
+    input_dir : str
+        Path to the directory containing the netcdf files.
+    window_start : str
+        Start date of the window. Format: YYYY-MM-DD.
+    window_end : str
+        End date of the window. Format: YYYY-MM-DD.
+    layer_name : str
+        Soil layer to consider for the mean.
+    agg : str
+        Aggregation method to use. Possible values: mean, median, max, min, std, or None.
+
+    Returns
+    -------
+    np.ndarray
+        Mean raster for the given window_start and window_end dates.
+    """
+
+    # Get the list of dates between two dates if date_from and date_to
+    dates = pd.DataFrame(
+        pd.date_range(
+            pd.to_datetime(window_start),
+            pd.to_datetime(window_end) - timedelta(days=1),
+            freq="d",
+        ),
+        columns=["date"],
+    )  # .strftime('%Y-%m-%d')
+    dates["dayofyear"] = dates["date"].dt.dayofyear - 1
+    dates["year"] = dates["date"].dt.year
+    dates["str_dates"] = dates["date"].dt.strftime("%Y-%m-%d")
+
+    yearly_dates = dates.groupby("year")["dayofyear"].apply(list).to_dict()
+
+    data_l = list()
+    # For each year, get the data for layer_name for the dates specified in yearly_dates
+    for year in yearly_dates:
+        # read the year file
+        nc_y = netCDF4.Dataset(os.path.join(input_dir, f"model_{year}.nc"))
+
+        vprint(
+            1,
+            V,
+            V_IGNORE,
+            Debug=f"getting data for year: {year} from layer: {layer_name}...",
+        )
+        # Get the data for the layer_name
+        data = nc_y.variables[layer_name][:, :, :]
+
+        # Get the data for the dates
+        days = yearly_dates[year]
+
+        data = data[days, :, :]
+
+        data_l.append(data)
+        nc_y.close()
+        del data
+        gc.collect()
+
+    # Concat data for all years
+    data_concat = np.concatenate(data_l, axis=0)
+    data_concat.shape
+    if agg == "mean":
+        # Get the mean raster for the range
+        data_agg = np.mean(data_concat, axis=0)
+    elif agg == "median":
+        # Get the median raster for the range
+        data_agg = np.median(data_concat, axis=0)
+    elif agg == "max":
+        # Get the max raster for the range
+        data_agg = np.max(data_concat, axis=0)
+    elif agg == "min":
+        # Get the min raster for the range
+        data_agg = np.min(data_concat, axis=0)
+    elif agg == "std":
+        # Get the std raster for the range
+        data_agg = np.std(data_concat, axis=0)
+    elif agg == "var":
+        # Get the var raster for the range
+        data_agg = np.var(data_concat, axis=0)
+    elif agg == "sum":
+        # Get the sum raster for the range
+        data_agg = np.sum(data_concat, axis=0)
+    elif agg is None:
+        data_agg = data_concat.copy()
+    else:
+        raise ValueError(
+            f"agg should be one of 'mean', 'median', 'max', 'min', 'std', 'var', 'sum', or a None value. {agg} was provided."
+        )
+    print("done.")
+    return data_agg
+
+
+def get_historic_agg(
+    input_dir: str,
+    historic_years: int,
+    current_window_start: str,
+    current_window_end: str,
+    layer_name: str,
+    agg_window: str = "mean",
+    agg_history: str = "mean",
+) -> np.ndarray:
+    """Get the historic mean for a given window_start and window_end dates.
+
+    Parameters
+    ----------
+    input_dir : str
+        Path to the directory containing the netcdf files.
+    historic_years : int
+        Number of historic years to consider for the mean.
+    current_window_start : str
+        Start date of the current window. Format: YYYY-MM-DD.
+    current_window_end : str
+        End date of the current window. Format: YYYY-MM-DD.
+    layer_name : str
+        Soil layer to consider for the mean.
+    agg_window : str
+        Aggregation method for the window (applies to both current and historic). Default is "mean". Possible values: "mean", "median", "max", "min", "std", "var", None (None will just make a 3D matrix for each year).
+    agg_history : str
+        Aggregation method for the historic years. This defines the way all years are combined into one. Default is "mean". Possible values: "mean", "median", "max", "min", "std", "var", "quantiles".
+        If `quantile` is selected, the historic cube will be sent back as a 3D matrix (with non-agregated window and concatenated on axis 0 instead) for quantile comparison.
+    Returns
+    -------
+    np.ndarray
+        Array of the historic mean for the given window_start and window_end dates for the historic years.
+
+    Raises
+    ------
+    FileNotFoundError
+        If the file for the historic year is not found. Possible solutions:
+          - The historic year should be modelled before calling this function.
+          - The path to the historic year should be changed.
+          - Calculate for a more recent historic year by reducing historic_years value.
+    """
+
+    # Get the window_start year
+    window_start_year = pd.to_datetime(current_window_start).year
+    window_end_year = pd.to_datetime(current_window_end).year
+
+    # Get the first year
+    first_year = window_start_year - historic_years
+
+    # Check if file exists for this year
+    if os.path.exists(os.path.join(input_dir, f"model_{first_year}.nc")):
+        # Get the list of historic windows
+        historic_agg = {}
+        for year in range(1, historic_years + 1):
+            args = {
+                "input_dir": input_dir,
+                "window_start": f"{window_start_year-year}{current_window_start[4:]}",
+                "window_end": f"{window_end_year-year}{current_window_end[4:]}",
+                "layer_name": layer_name,
+                "agg": agg_window,
+            }
+            # Get the range mean
+            historic_agg[window_start_year - year] = get_range_agg(**args)
+        historic_agg_np = np.array([historic_agg[year] for year in historic_agg])
+        # Get the aggregation of the historic years
+        if agg_history == "mean":
+            historic_agg_np = np.mean(historic_agg_np, axis=0)
+        elif agg_history == "median":
+            historic_agg_np = np.median(historic_agg_np, axis=0)
+        elif agg_history == "max":
+            historic_agg_np = np.max(historic_agg_np, axis=0)
+        elif agg_history == "min":
+            historic_agg_np = np.min(historic_agg_np, axis=0)
+        elif agg_history == "std":
+            historic_agg_np = np.std(historic_agg_np, axis=0)
+        elif agg_history == "var":
+            historic_agg_np = np.var(historic_agg_np, axis=0)
+        elif agg_history == "sum":
+            historic_agg_np = np.sum(historic_agg_np, axis=0)
+        elif agg_history is None:
+            historic_agg_np = np.concatenate(list(historic_agg.values()), axis=0)
+        else:
+            raise ValueError(
+                f"Invalid aggregation method: {agg_history}. Possible values: mean, median, max, min, std, var, sum."
+            )
+        return historic_agg_np
+    else:
+        raise FileNotFoundError(
+            f"File not found for the historic data: {os.path.join(input_dir,f'model_{first_year}.nc')}. Make sure the path is correct and the historic year for the requested year is modelled before calling this function."
+        )
+
+
+def save(path, array, profile):
+    """Save the array as a raster.
+
+    Parameters
+    ----------
+    path : str
+        Path to the raster to save.
+    array : np.ndarray
+        Array to save as a raster.
+    profile : dict
+        Profile of the raster to save.
+    """
+
+    with rasterio.open(path, "w", **profile) as dst:
+        dst.write(array, 1)
+
+
+def analyse(
+    input,
+    window_start,
+    window_end,
+    historic_years: int,
+    layer: str,
+    match_raster: str = None,
+    output: str = None,
+    agg_history: str = "mean",
+    agg_window: str = "mean",
+    **kwargs,
+) -> Dict[str, str]:
+    """Main function to run the script.
+
+    Parameters
+    ----------
+    input : str
+        Path to the input raster.
+    window_start : str
+        Start date of the window. Format: YYYY-MM-DD.
+    window_end : str
+        End date of the window. Format: YYYY-MM-DD.
+    historic_years : int
+        Number of historic years to use for the comparison.
+    layer : str
+        Soil layer to consider for the comparison.
+    match_raster : str
+        Path to the match raster. Default: None. If None, the match raster will be searched in the et_pp directory based on the input directory.
+    output : str
+        Path to the output raster. Default: None. If None, the output raster will be saved in the same directory as the input raster.
+    agg_history : str
+        Aggregation method to use for the historic years. Possible values: 'mean', 'median', 'max', 'min', 'std', None. Default: 'mean'.
+    agg_window : str
+        Aggregation method to use for the window. Possible values: 'mean', 'median', 'max', 'min', 'std', None. Default: 'mean'.
+
+    Returns
+    -------
+    Dict[str,str]
+        Dictionary with the path to the output rasters.
+    """
+
+    if output is None:
+        output = os.path.join(input, "analysis")
+
+    # Create the output directory if it does not exist
+    if not os.path.exists(output):
+        os.makedirs(output)
+
+    if match_raster is None:
+        match_raster = find_match_raster(input)
+    print("match_raster is:", match_raster)
+
+    if match_raster is not None:
+        print("Found match raster:", match_raster)
+
+    files = glob.glob(os.path.join(match_raster, f"{window_start[:7]}*.tif"))
+    if len(files) == 0:
+        files = glob.glob(os.path.join(match_raster, f"{window_end[:7]}*.tif"))
+    if len(files) == 0:
+        vprint(
+            1,
+            V,
+            V_IGNORE,
+            Debug=f"Expanding the search for match raster file to find e closer date to {window_start[:5]}...",
+        )
+        files = glob.glob(os.path.join(match_raster, f"{window_start[:5]}*.tif"))
+    if len(files) == 0:
+        vprint(
+            1,
+            V,
+            V_IGNORE,
+            Debug=f"Expanding the search further for match raster file to find e closer date  to {window_end[:5]}...",
+        )
+        files = glob.glob(os.path.join(match_raster, f"{window_end[:5]}*.tif"))
+    if len(files) == 0:
+        raise FileNotFoundError(
+            f"Could not find any matching raster in {match_raster} for the rage of dates given at {window_start} / {window_end}!"
+        )
+    print(f"Found {len(files)} matching raster file {files[0]}.")
+    match_raster = files[0]
+
+    with rasterio.open(match_raster) as src:
+        profile = src.profile
+
+    # Get the layers
+    layer = layer
+    # Get the historic aggregated data
+
+    # Get aggregated current window data
+    current_data = get_range_agg(
+        input_dir=input,
+        window_start=window_start,
+        window_end=window_end,
+        agg=agg_window,
+        layer_name=layer,
+    )
+
+    historic_data = get_historic_agg(
+        input_dir=input,
+        historic_years=historic_years,
+        current_window_start=window_start,
+        current_window_end=window_end,
+        agg_window=agg_window,
+        agg_history=agg_history,
+        layer_name=layer,
+    )
+
+    historic_data_quant = get_historic_agg(
+        input_dir=input,
+        historic_years=historic_years,
+        current_window_start=window_start,
+        current_window_end=window_end,
+        agg_window=None,
+        agg_history=None,
+        layer_name=layer,
+    )
+
+    # Compare the two rasters
+    delta = current_data - historic_data
+
+    quantile = current_data.copy()
+    print("\nCalculating quantiles...", "\n=========================")
+    print("Data shape:", current_data.shape)
+    print("Historic data shape:", historic_data_quant.shape)
+    print("=========================\n")
+    pixel_now = []
+    pixel_hist = []
+    pixel_quant = []
+    for i in range(current_data.shape[0]):
+        for j in range(current_data.shape[1]):
+            sorted_scores = np.array(sorted(historic_data_quant[:, i, j]))
+            quantile[i, j] = (
+                (sorted_scores.searchsorted(current_data[i, j]))
+                / sorted_scores.shape[0]
+                * 100
+            )
+            pixel_now.append(current_data[i, j])
+            pixel_hist.append(sorted_scores)
+            pixel_quant.append(quantile[i, j])
+
+    df_quants = pd.DataFrame(pixel_hist)
+    df_quants["pixel_now"] = pixel_now
+    df_quants["pixel_quant"] = pixel_quant
+    print("shapes are:", len(pixel_now), len(pixel_quant))
+    df_quants = df_quants.sort_values(by=["pixel_quant"], ascending=False)
+    print(df_quants)
+    df_quants.to_csv(
+        os.path.join(
+            output,
+            f"quantiles-{window_start.replace('-','_')}-{window_end.replace('-','_')}-{layer}-w_{agg_window}-h_concat-y_{historic_years}.csv",
+        )
+    )
+
+    # Search for a shape file ending with .shp
+    shape_file = find_shape_file(input)
+    print("Shape file:", shape_file)
+
+    # Save the rasters
+    historic_raster = os.path.join(
+        output,
+        f"historic-{window_start.replace('-','_')}-{window_end.replace('-','_')}-{layer}-w_{agg_window}-h_{agg_history}-y_{historic_years}.tif",
+    )
+    current_raster = os.path.join(
+        output,
+        f"current-{window_start.replace('-','_')}-{window_end.replace('-','_')}-{layer}-w_{agg_window}.tif",
+    )
+    delta_raster = os.path.join(
+        output,
+        f"delta-{window_start.replace('-','_')}-{window_end.replace('-','_')}-{layer}-w_{agg_window}-h_{agg_history}-y_{historic_years}.tif",
+    )
+    quant_raster = os.path.join(
+        output,
+        f"quantile-{window_start.replace('-','_')}-{window_end.replace('-','_')}-{layer}-w_{agg_window}-h_concat-y_{historic_years}.tif",
+    )
+
+    save(historic_raster, historic_data, profile)
+    save(current_raster, current_data, profile)
+    save(delta_raster, delta, profile)
+    save(quant_raster, quantile, profile)
+
+    # Open the rasters
+    # with rasterio.open(historic_raster) as src:
+    #     historic_raster = src.read(1)
+
+    # Clip the rasters to the shape file
+    if shape_file is not None:
+        print("Found shape file:", shape_file)
+        # shapes = read_shape_file(
+        #     shape_file, epsg_num=None, dissolve=False
+        #     ).geometry
+        try:
+            with fiona.open(shape_file, "r") as shapefile:
+                shapes = [feature["geometry"] for feature in shapefile]
+
+            try:
+                with rasterio.open(historic_raster) as src:
+                    out_image, transformed = rasterio.mask.mask(
+                        src, shapes, crop=True, filled=True
+                    )
+                    out_profile = src.profile.copy()
+                out_profile.update(
+                    {
+                        "width": out_image.shape[2],
+                        "height": out_image.shape[1],
+                        "transform": transformed,
+                    }
+                )
+                with rasterio.open(historic_raster, "w", **out_profile) as dst:
+                    dst.write(out_image)
+            except Exception as e:
+                print("Failed to crop historic raster", historic_raster)
+                print(e)
+
+            try:
+                with rasterio.open(current_raster) as src:
+                    out_image, transformed = rasterio.mask.mask(
+                        src, shapes, crop=True, filled=True
+                    )
+                    out_profile = src.profile.copy()
+                out_profile.update(
+                    {
+                        "width": out_image.shape[2],
+                        "height": out_image.shape[1],
+                        "transform": transformed,
+                    }
+                )
+                with rasterio.open(current_raster, "w", **out_profile) as dst:
+                    dst.write(out_image)
+            except Exception as e:
+                print("Failed to crop current raster", current_raster)
+                print(e)
+
+            try:
+                with rasterio.open(delta_raster) as src:
+                    out_image, transformed = rasterio.mask.mask(
+                        src, shapes, crop=True, filled=True
+                    )
+                    out_profile = src.profile.copy()
+                out_profile.update(
+                    {
+                        "width": out_image.shape[2],
+                        "height": out_image.shape[1],
+                        "transform": transformed,
+                    }
+                )
+                with rasterio.open(delta_raster, "w", **out_profile) as dst:
+                    dst.write(out_image)
+            except Exception as e:
+                print("Failed to crop delta raster", delta_raster)
+                print(e)
+
+            try:
+                with rasterio.open(quant_raster) as src:
+                    out_image, transformed = rasterio.mask.mask(
+                        src, shapes, crop=True, filled=True
+                    )
+                    out_profile = src.profile.copy()
+                out_profile.update(
+                    {
+                        "width": out_image.shape[2],
+                        "height": out_image.shape[1],
+                        "transform": transformed,
+                    }
+                )
+                with rasterio.open(quant_raster, "w", **out_profile) as dst:
+                    dst.write(out_image)
+            except Exception as e:
+                print("Failed to crop quantile raster", quant_raster)
+                print(e)
+
+        except Exception as e:
+            print("Could not open shape file:", shape_file)
+            print(e)
+
+        # current_data, _ = rasterio.mask.mask(current_data, shapes, crop=True)
+        # historic_data, _ = rasterio.mask.mask(historic_data, shapes, crop=True)
+        # delta, _ = rasterio.mask.mask(delta, shapes, crop=True)
+        # quantile, _ = rasterio.mask.mask(quantile, shapes, crop=True)
+        print("done.")
+
+    return {
+        "historic_raster": historic_raster,
+        "current_raster": current_raster,
+        "delta_raster": delta_raster,
+        "quant_raster": quant_raster,
+    }
+
+
+def find_analyses(path):
+    """Find all the analysis files in a directory.
+
+    Parameters
+    ----------
+    path: str
+        Path to the directory containing the analysis files
+
+    Returns
+    -------
+    files: list
+        List of analysis files
+    """
+    files = [f for f in os.listdir(path) if f.endswith(".tif")]
+    return files
+
+
+def open_image(path):
+    """Open a raster image and return the data and coordinates.
+
+    Parameters
+    ----------
+    path: str
+        path to the raster image
+
+    Returns
+    -------
+    band1: np.array
+        The raster data
+    lons: np.array
+        The longitude coordinates
+    lats: np.array
+        The latitude coordinates
+    """
+    with rasterio.open(path) as src:
+        band1 = src.read(1)
+        print("Band1 has shape", band1.shape)
+        height = band1.shape[0]
+        width = band1.shape[1]
+        cols, rows = np.meshgrid(np.arange(width), np.arange(height))
+        xs, ys = rasterio.transform.xy(src.transform, rows, cols)
+        lons = np.array(xs)
+        lats = np.array(ys)
+
+    return band1, lons, lats
+
+
+def perform_analysis(
+    input,
+    window_start,
+    window_end,
+    historic_years: int,
+    layer: str,
+    match_raster: str = None,
+    output: str = None,
+    agg_history: str = "mean",
+    agg_window: str = "mean",
+    comparison: str = "diff",
+    **args,
+) -> Dict[str, str]:
+    """Perform the analysis.
+
+    This is a wrapper function for the analysis module. It takes the input parameters and passes them to the analysis module.
+
+    Parameters
+    ----------
+    input : str
+        path to the input data
+    window_start : str
+        start date of the window
+    window_end : str
+        end date of the window
+    historic_years : int
+        number of years to use for the historic data
+    layer : str
+        layer to use for the analysis
+    match_raster : str, optional
+        path to the raster to match the output to, by default None
+    output : str, optional
+        path to the output file, by default None
+    agg_history : str, optional
+        aggregation method for the historic data, by default "mean"
+    agg_window : str, optional
+        aggregation method for the window data, by default "mean"
+    comparison : str, optional
+        comparison method for the window and historic data, by default "diff"
+
+    Returns
+    -------
+        files: dict
+            Dict of analysis files
+    """
+    files = analyse(
+        input=input,
+        window_start=window_start,
+        window_end=window_end,
+        historic_years=historic_years,
+        agg_window=agg_window,
+        agg_history=agg_history,
+        comparison=comparison,
+        layer=layer,
+        output=output,
+        match_raster=match_raster,
+    )
+    return files
+
+
+def layout(WAIT_IMAGE):
+    import datetime
+
+    import plotly.express as px
+    from dash import dcc, html
+
+    today = datetime.datetime.today()
+
+    colorscales = px.colors.named_colorscales()
+
+    layout = html.Div(
+        [
+            # html.Div(
+            # className="dashapp-header",
+            # children=[
+            #     html.Div('Soil Moisture Comparison Tool', className="dashapp-header--title")
+            # ]
+            # ),
+            dcc.Store(id="farm-name-session", storage_type="session"),
+            html.Div(
+                [
+                    html.P(
+                        """This tool will use the produced datacubes to compare the soil moisture of a farm against historic data.
+                        Please select the desired comaprison method and dates to make the comparison as in section A.
+                        Then choose the visualisation in section B to see the results.""",
+                        style={"font-size": "larger"},
+                    ),
+                    html.Hr(),
+                    html.H3("A"),
+                ],
+                className="col-lg-12",
+                style={"padding-top": "1%", "padding-left": "1%"},
+            ),
+            html.Div(
+                [
+                    html.Div(
+                        [
+                            # html.P("Write farm name/ID:"),
+                            dcc.Input(
+                                id="farm-name",
+                                type="text",
+                                placeholder="Farm name",
+                                style={"width": "80%"},
+                            ),
+                            html.Img(
+                                id="farm-image",
+                                src=WAIT_IMAGE,
+                                style={"width": "30px", "margin-left": "15px"},
+                            ),
+                        ],
+                        className="col-lg-5",
+                        # style = {'padding-top':'1%', 'padding-left':'1%'}
+                    ),
+                    html.Div(
+                        [
+                            html.P(),
+                        ],
+                        className="col-lg-7",
+                        # style = {'padding-top':'1%', 'padding-left':'1%'}
+                    ),
+                ],
+                className="row",
+                style={"padding-top": "1%", "padding-left": "1%"},
+            ),
+            html.Div(
+                [
+                    html.Div(
+                        [
+                            html.P("Select soil layer:"),
+                            dcc.Dropdown(
+                                id="layer-dropdown",
+                                options=[
+                                    {"label": "SM1", "value": "SM1"},
+                                    {"label": "SM2", "value": "SM2"},
+                                    {"label": "SM3", "value": "SM3"},
+                                    {"label": "SM4", "value": "SM4"},
+                                    {"label": "SM5", "value": "SM5"},
+                                    {"label": "DD", "value": "DD"},
+                                ],
+                                value="SM2",
+                            ),
+                        ],
+                        className="col-lg-4",
+                        style={"padding": "1%"},
+                    ),
+                    html.Div(
+                        [
+                            html.P("Select the historic years to compare against:"),
+                            dcc.Dropdown(
+                                id="historic-dropdown",
+                                options=[
+                                    {"label": year, "value": year}
+                                    for year in range(1, 20)
+                                ],
+                                value=2,
+                            ),
+                        ],
+                        className="col-lg-4",
+                        style={"padding": "1%"},
+                    ),
+                    html.Div(
+                        [
+                            html.P(
+                                "Select the most recent window of dates to analyse:"
+                            ),
+                            dcc.DatePickerRange(
+                                id="window-select",
+                                min_date_allowed=datetime.date(2000, 1, 1),
+                                max_date_allowed=today.strftime("%Y-%m-%d"),
+                                initial_visible_month=datetime.date(2023, 1, 1),
+                                clearable=False,
+                                display_format="YYYY-MM-DD",
+                                start_date_placeholder_text="Start date",
+                                end_date_placeholder_text="End date",
+                                style={"width": "100%"},
+                            ),
+                        ],
+                        className="col-lg-4",
+                        style={"padding": "1%"},
+                    ),
+                ],
+                className="row",
+                style={"padding-top": "1%"},
+            ),
+            html.Div(
+                [
+                    html.Div(
+                        [
+                            html.P("Select window aggregation method:"),
+                            dcc.Dropdown(
+                                id="w-aggregation-dropdown",
+                                options=[
+                                    {"label": "Mean", "value": "mean"},
+                                    {"label": "Median", "value": "median"},
+                                    {"label": "Max", "value": "max"},
+                                    {"label": "Min", "value": "min"},
+                                    {"label": "Sum", "value": "sum"},
+                                    {"label": "std", "value": "std"},
+                                    {"label": "var", "value": "var"},
+                                ],
+                                value="mean",
+                            ),
+                        ],
+                        className="col-lg-6",
+                        style={"padding": "1%"},
+                    ),
+                    html.Div(
+                        [
+                            html.P("Select historic aggregation method:"),
+                            dcc.Dropdown(
+                                id="h-aggregation-dropdown",
+                                options=[
+                                    {"label": "Mean", "value": "mean"},
+                                    {"label": "Median", "value": "median"},
+                                    {"label": "Max", "value": "max"},
+                                    {"label": "Min", "value": "min"},
+                                    {"label": "Sum", "value": "sum"},
+                                    {"label": "std", "value": "std"},
+                                    {"label": "var", "value": "var"},
+                                    {"label": "quantile", "value": "quantile"},
+                                ],
+                                value="mean",
+                            ),
+                        ],
+                        className="col-lg-6",
+                        style={"padding": "1%"},
+                    ),
+                ],
+                className="row",
+                # style = {'padding-top':'1%'}
+            ),
+            html.Div(
+                [
+                    html.Button("Generate Images", id="generate-button"),
+                    html.Br(),
+                    html.Hr(),
+                ],
+                className="col-lg-12",
+                style={"margin-bottom": "1%"},
+            ),
+            html.Div(
+                [
+                    html.H3("B"),
+                ],
+                className="col-lg-12",
+                style={"padding-top": "1%", "padding-left": "1%"},
+            ),
+            html.Div(
+                [
+                    html.Div(
+                        [
+                            html.P("Select visualisation name:"),
+                            dcc.Dropdown(id="visualisation-select"),
+                        ],
+                        className="col-lg-6",
+                        style={"padding": "1%"},
+                    ),
+                    html.Div(
+                        [
+                            html.P("Select your palette:"),
+                            dcc.Dropdown(
+                                id="platter-dropdown",
+                                options=colorscales,
+                                value="viridis",
+                            ),
+                        ],
+                        className="col-lg-6",
+                        style={"padding": "1%"},
+                    ),
+                ],
+                className="row",
+                # style = {'padding-top':'1%'}
+            ),
+            html.Div(
+                [
+                    html.Hr(),
+                    html.H3("Results"),
+                    dcc.Graph(id="graph"),
+                ],
+                className="col-lg-12",
+                style={"padding-top": "1%"},
+            ),
+            # html.Div(
+            # className="dashapp-footer",
+            # children=[
+            #     html.Div(f"Copyright @ {today.strftime('%Y')} Sydney Informatics Hub (SIH)", className="dashapp-footer--copyright")
+            # ]
+            # ),
+        ],
+        className="container-fluid",
+    )
+
+    return layout

libs/utils.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+import logging
+
+
+def setup_logging():
+    logging.basicConfig(
+        format="%(asctime)s %(module)s %(levelname)s : %(message)s", level=logging.INFO
+    )
+
+
+# -*- coding: utf-8 -*-
+def verbose(value, thresh=1, ignore: list = [], **outputs):
+    """Verbose outputs.
+
+    Parameters
+    ----------
+    value : int
+        An integer showing the verbose level of an output. Higher verbose value means less important.
+        If the object's or script's initialized verbose threshold is equal to or lower than this value, it will print.
+    thresh : int
+    ignore : list
+        A list of outout types to ignore. (Debug, Warning, Error, etc.)
+
+    **outputs : kword arguments, 'key'='value'
+        Outputs with keys. Keys will be also printed. More than one key and output pair can be supplied.
+    """
+    for output in outputs.items():
+        if output[0] not in ignore:
+            if value <= thresh:
+                print("\n> " + str(output[0]) + ":", output[1])
+            # event_log.append([str(datetime.now())+'\n> '+str(output[0])+':',output[1]])

model/job.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+import datetime as dt
+
+from marshmallow import Schema, fields
+
+
+class Job(object):
+    def __init__(self, description, amount, type):
+        self.description = description
+        self.time_start = 1
+        self.created_at = dt.datetime.now()
+        self.type = type
+
+    def __repr__(self):
+        return "<Transaction(name={self.description!r})>".format(self=self)
+
+
+class TransactionSchema(Schema):
+    description = fields.Str()
+    amount = fields.Number()
+    created_at = fields.Date()
+    type = fields.Str()

requirements/requirements-dev-dash.txt

@@ -0,0 +1,10 @@
+matplotlib
+dash
+dash-bootstrap-components
+netCDF4
+tqdm
+fiona
+pandas
+rasterio
+shapely
+numpy

requirements/requirements-dev-vis.txt

@@ -0,0 +1,26 @@
+pillow
+numpy
+matplotlib
+basemap-data
+basemap-data-hires
+basemap
+netCDF4
+earthengine-api
+geopandas
+fiona
+pandas
+rasterio
+shapely
+tqdm
+holoviews 
+bokeh 
+datashader 
+panel 
+param
+dash
+dash-bootstrap-components
+flask
+flask-restful
+marshmallow
+Flask-SQLAlchemy
+flask-bcrypt

requirements/requirements-vis-system.txt

@@ -0,0 +1 @@
+sudo apt install python3-dev mysql-server default-libmysqlclient-dev build-essential

requirements/requuirements-gcp.txt

@@ -0,0 +1,5 @@
+google-api-core
+google-cloud-shell
+google-cloud-batch
+google-cloud-datastore
+

scripts/analyse.py

@@ -0,0 +1,468 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import argparse
+import gc
+import glob
+import logging
+import os
+import traceback
+from datetime import timedelta
+from typing import Dict
+
+import netCDF4
+import numpy as np
+import pandas as pd
+import rasterio
+
+from libs.utils import setup_logging
+from libs.utils import verbose as vprint
+
+setup_logging()
+log = logging.getLogger(__name__)
+CONFIG = {}
+V = 1
+V_IGNORE = []  # Debug, Warning, Error
+# print(os.getcwd())
+
+
+def get_historic_agg(
+    input_dir: str,
+    historic_years: int,
+    current_window_start: str,
+    current_window_end: str,
+    layer_name: str,
+    agg_window: str = "mean",
+    agg_history: str = "mean",
+) -> np.ndarray:
+    """Get the historic mean for a given window_start and window_end dates.
+
+    Parameters
+    ----------
+    input_dir : str
+        Path to the directory containing the netcdf files.
+    historic_years : int
+        Number of historic years to consider for the mean.
+    current_window_start : str
+        Start date of the current window. Format: YYYY-MM-DD.
+    current_window_end : str
+        End date of the current window. Format: YYYY-MM-DD.
+    layer_name : str
+        Soil layer to consider for the mean.
+    agg_window : str
+        Aggregation method for the window. Default is "mean". Possible values: "mean", "median", "max", "min", "std", "var".
+    agg_history : str
+        Aggregation method for the historic years. Default is "mean". Possible values: "mean", "median", "max", "min", "std", "var".
+
+    Returns
+    -------
+    np.ndarray
+        Array of the historic mean for the given window_start and window_end dates for the historic years.
+
+    Raises
+    ------
+    FileNotFoundError
+        If the file for the historic year is not found. Possible solutions:
+          - The historic year should be modelled before calling this function.
+          - The path to the historic year should be changed.
+          - Calculate for a more recent historic year by reducing historic_years value.
+    """
+
+    # Get the window_start year
+    window_start_year = pd.to_datetime(current_window_start).year
+    window_end_year = pd.to_datetime(current_window_end).year
+
+    # Get the first year
+    first_year = window_start_year - historic_years
+
+    # Check if file exists for this year
+    if os.path.exists(os.path.join(input_dir, f"model_{first_year}.nc")):
+        # Get the list of historic windows
+        historic_agg = {}
+        for year in range(1, historic_years + 1):
+            args = {
+                "input_dir": input_dir,
+                "window_start": f"{window_start_year-year}{current_window_start[4:]}",
+                "window_end": f"{window_end_year-year}{current_window_end[4:]}",
+                "layer_name": layer_name,
+                "agg": agg_window,
+            }
+            # Get the range mean
+            historic_agg[window_start_year - year] = get_range_agg(**args)
+        historic_agg_np = np.array([historic_agg[year] for year in historic_agg])
+        # Get the aggregation of the historic years
+        if agg_history == "mean":
+            historic_agg_np = np.mean(historic_agg_np, axis=0)
+        elif agg_history == "median":
+            historic_agg_np = np.median(historic_agg_np, axis=0)
+        elif agg_history == "max":
+            historic_agg_np = np.max(historic_agg_np, axis=0)
+        elif agg_history == "min":
+            historic_agg_np = np.min(historic_agg_np, axis=0)
+        elif agg_history == "std":
+            historic_agg_np = np.std(historic_agg_np, axis=0)
+        elif agg_history == "var":
+            historic_agg_np = np.var(historic_agg_np, axis=0)
+        elif agg_history == "sum":
+            historic_agg_np = np.sum(historic_agg_np, axis=0)
+        else:
+            raise ValueError(
+                f"Invalid aggregation method: {agg_history}. Possible values: mean, median, max, min, std, var, sum."
+            )
+        return historic_agg_np
+    else:
+        raise FileNotFoundError(
+            f"File not found for the historic data: {os.path.join(input_dir,f'model_{first_year}.nc')}. Make sure the path is correct and the historic year for the requested year is modelled before calling this function."
+        )
+
+
+def get_range_agg(
+    input_dir: str,
+    window_start: str,
+    window_end: str,
+    layer_name: str,
+    agg: str = "mean",
+) -> np.ndarray:
+    """Get the mean for a given window_start and window_end dates.
+
+    Parameters
+    ----------
+    input_dir : str
+        Path to the directory containing the netcdf files.
+    window_start : str
+        Start date of the window. Format: YYYY-MM-DD.
+    window_end : str
+        End date of the window. Format: YYYY-MM-DD.
+    layer_name : str
+        Soil layer to consider for the mean.
+    agg : str
+        Aggregation method to use. Possible values: mean, median, max, min, std.
+
+    Returns
+    -------
+    np.ndarray
+        Mean raster for the given window_start and window_end dates.
+    """
+
+    # Get the list of dates between two dates if date_from and date_to
+    dates = pd.DataFrame(
+        pd.date_range(
+            pd.to_datetime(window_start),
+            pd.to_datetime(window_end) - timedelta(days=1),
+            freq="d",
+        ),
+        columns=["date"],
+    )  # .strftime('%Y-%m-%d')
+    dates["dayofyear"] = dates["date"].dt.dayofyear - 1
+    dates["year"] = dates["date"].dt.year
+    dates["str_dates"] = dates["date"].dt.strftime("%Y-%m-%d")
+
+    yearly_dates = dates.groupby("year")["dayofyear"].apply(list).to_dict()
+
+    data_l = list()
+    # For each year, get the data for layer_name for the dates specified in yearly_dates
+    for year in yearly_dates:
+        # read the year file
+        nc_y = netCDF4.Dataset(os.path.join(input_dir, f"model_{year}.nc"))
+
+        vprint(
+            1,
+            V,
+            V_IGNORE,
+            Debug=f"getting data for year: {year} from layer: {layer_name}...",
+        )
+        # Get the data for the layer_name
+        data = nc_y.variables[layer_name][:, :, :]
+
+        # Get the data for the dates
+        days = yearly_dates[year]
+
+        data = data[days, :, :]
+
+        data_l.append(data)
+        nc_y.close()
+        del data
+        gc.collect()
+
+    # Concat data for all years
+    data_concat = np.concatenate(data_l, axis=0)
+    data_concat.shape
+    if agg == "mean":
+        # Get the mean raster for the range
+        data_agg = np.mean(data_concat, axis=0)
+    elif agg == "median":
+        # Get the median raster for the range
+        data_agg = np.median(data_concat, axis=0)
+    elif agg == "max":
+        # Get the max raster for the range
+        data_agg = np.max(data_concat, axis=0)
+    elif agg == "min":
+        # Get the min raster for the range
+        data_agg = np.min(data_concat, axis=0)
+    elif agg == "std":
+        # Get the std raster for the range
+        data_agg = np.std(data_concat, axis=0)
+    elif agg == "var":
+        # Get the var raster for the range
+        data_agg = np.var(data_concat, axis=0)
+    elif agg == "sum":
+        # Get the sum raster for the range
+        data_agg = np.sum(data_concat, axis=0)
+    else:
+        raise ValueError(
+            f"agg should be one of mean, median, max, min, std, var, sum. {agg} was provided."
+        )
+    print("done.")
+    return data_agg
+
+
+def save(path, array, profile):
+    """Save the array as a raster.
+
+    Parameters
+    ----------
+    path : str
+        Path to the raster to save.
+    array : np.ndarray
+        Array to save as a raster.
+    profile : dict
+        Profile of the raster to save.
+    """
+
+    with rasterio.open(path, "w", **profile) as dst:
+        dst.write(array, 1)
+
+
+def analyse(
+    input,
+    window_start,
+    window_end,
+    historic_years: int,
+    layer: str,
+    match_raster: str = None,
+    output: str = None,
+    agg_history: str = "mean",
+    agg_window: str = "mean",
+    comparison: str = "diff",
+) -> Dict[str, str]:
+    """Main function to run the script.
+
+    Parameters
+    ----------
+    input : str
+        Path to the input raster.
+    window_start : str
+        Start date of the window. Format: YYYY-MM-DD.
+    window_end : str
+        End date of the window. Format: YYYY-MM-DD.
+    historic_years : int
+        Number of historic years to use for the comparison.
+    layer : str
+        Soil layer to consider for the comparison.
+    match_raster : str
+        Path to the match raster. Default: None. If None, the match raster will be searched in the et_pp directory based on the input directory.
+    output : str
+        Path to the output raster. Default: None. If None, the output raster will be saved in the same directory as the input raster.
+    agg_history : str
+        Aggregation method to use for the historic years. Possible values: mean, median, max, min, std. Default: mean.
+    agg_window : str
+        Aggregation method to use for the window. Possible values: mean, median, max, min, std. Default: mean.
+    comparison : str
+        Comparison method to use. Possible values: diff, ratio. Default: diff.
+
+    Returns
+    -------
+    Dict[str,str]
+        Dictionary with the path to the output rasters.
+    """
+
+    if output is None:
+        output = os.path.join(input, "analysis")
+
+    # Create the output directory if it does not exist
+    if not os.path.exists(output):
+        os.makedirs(output)
+
+    if match_raster is None:
+        match_raster = os.path.join(os.path.dirname(input), "et_pp")
+
+    files = glob.glob(os.path.join(match_raster, f"{window_start[:7]}*.tif"))
+    if len(files) == 0:
+        files = glob.glob(os.path.join(match_raster, f"{window_end[:7]}*.tif"))
+    if len(files) == 0:
+        vprint(
+            1,
+            V,
+            V_IGNORE,
+            Debug=f"Expanding the search for match raster file to find e closer date to {window_start[:5]}...",
+        )
+        files = glob.glob(os.path.join(match_raster, f"{window_start[:5]}*.tif"))
+    if len(files) == 0:
+        vprint(
+            1,
+            V,
+            V_IGNORE,
+            Debug=f"Expanding the search further for match raster file to find e closer date  to {window_end[:5]}...",
+        )
+        files = glob.glob(os.path.join(match_raster, f"{window_end[:5]}*.tif"))
+    if len(files) == 0:
+        raise FileNotFoundError(
+            f"Could not find any matching raster in {match_raster} for the range of dates given at {window_start} / {window_end}!"
+        )
+    print(f"Found {len(files)} matching raster file {files[0]}.")
+    match_raster = files[0]
+
+    with rasterio.open(match_raster) as src:
+        profile = src.profile
+
+    # Get the layers
+    layer = layer
+    # Get the historic aggregated data
+    historic_data = get_historic_agg(
+        input_dir=input,
+        historic_years=historic_years,
+        current_window_start=window_start,
+        current_window_end=window_end,
+        agg_window=agg_window,
+        agg_history=agg_history,
+        layer_name=layer,
+    )
+    # Get aggregated current window data
+    current_data = get_range_agg(
+        input_dir=input,
+        window_start=window_start,
+        window_end=window_end,
+        agg=agg_window,
+        layer_name=layer,
+    )
+
+    # Compare the two rasters
+    if comparison == "diff":
+        delta = current_data - historic_data
+    else:
+        raise NotImplementedError(
+            f"comparison should be diff. {comparison} was provided."
+        )
+
+    # Save the rasters
+    historic_raster = os.path.join(
+        output,
+        f"historic-{window_start.replace('-','_')}-{window_end.replace('-','_')}-{layer}-w_{agg_window}-h_{agg_history}-y_{historic_years}.tif",
+    )
+    current_raster = os.path.join(
+        output,
+        f"current-{window_start.replace('-','_')}-{window_end.replace('-','_')}-{layer}-w_{agg_window}.tif",
+    )
+    delta_raster = os.path.join(
+        output,
+        f"delta-{window_start.replace('-','_')}-{window_end.replace('-','_')}-{layer}-w_{agg_window}-h_{agg_history}-y_{historic_years}.tif",
+    )
+
+    save(historic_raster, historic_data, profile)
+    save(current_raster, current_data, profile)
+    save(delta_raster, delta, profile)
+
+    # # Visualise the rasters
+    # # Read the saved rasters
+    # with rasterio.open(historic_raster) as src:
+    #     historic_raster = src.read(1)
+    # with rasterio.open(current_raster) as src:
+    #     current_raster = src.read(1)
+    # with rasterio.open(delta_raster) as src:
+    #     delta_raster = src.read(1)
+
+    # # Plot the rasters
+    return {
+        "historic_raster": historic_raster,
+        "current_raster": current_raster,
+        "delta_raster": delta_raster,
+    }
+
+
+if __name__ == "__main__":
+    # Load Configs
+    parser = argparse.ArgumentParser(
+        description="Download rainfall data from Google Earth Engine for a range of dates.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        "-i",
+        "--input",
+        help="Absolute or relative path to the netcdf data ending with .nc. By dfault it is set to data.nc",
+        default="data.nc",
+    )
+    parser.add_argument(
+        "-l",
+        "--layer",
+        help="Soil layer to visualise. Default is all. Select between SM1 to SM5 or DD.",
+        default="all",
+    )
+    parser.add_argument("-s", "--window_start", help="Window start date YYYY-MM-DD.")
+    parser.add_argument("-e", "--window_end", help="Window end date YYYY-MM-DD.")
+    parser.add_argument(
+        "-y", "--historic_years", help="Number of years to go back in time.", default=2
+    )
+    parser.add_argument(
+        "-a",
+        "--agg_history",
+        help="Aggregation method to use for the historic data. Possible values: mean, median, max, min, std, var, sum.",
+        default="mean",
+    )
+    parser.add_argument(
+        "-g",
+        "--agg_window",
+        help="Aggregation method to use for the window range data. Possible values: mean, median, max, min, std, var, sum.",
+        default="mean",
+    )
+    parser.add_argument(
+        "-c",
+        "--comparison",
+        help="Comparison method to use for the window range data. Possible values: show, diff.",
+        default="show",
+    )
+    parser.add_argument(
+        "-o",
+        "--output",
+        help="Output directory to save the output files. Default is the input directory.",
+    )
+    parser.add_argument(
+        "-m",
+        "--match_raster",
+        help="Raster to match the output to. Default is the input raster.",
+    )
+
+    args = parser.parse_args()
+
+    # args.input ="/home/sahand/Projects/PIPE-3788 GRDC SoilWaterNow Deployment/work/v3/Arawa 2019-2023/c8/1af25ced023e58c46f4403a155210d/soilwatermodel v3"
+    # args.window_start = "2022-12-20"
+    # args.window_end = "2023-01-10"
+    # args.historic_years = 3
+    # args.agg_window = "mean"
+    # args.agg_history = "mean"
+    # args.comparison = "diff"
+    # args.layer = "SM2"
+    # args.output = None
+
+    try:
+        analyse(
+            input=args.input,
+            window_start=args.window_start,
+            window_end=args.window_end,
+            historic_years=args.historic_years,
+            agg_window=args.agg_window,
+            agg_history=args.agg_history,
+            comparison=args.comparison,
+            layer=args.layer,
+            output=args.output,
+            match_raster=args.match_raster,
+        )
+
+    except Exception as e:
+        vprint(
+            0,
+            V,
+            V_IGNORE,
+            Error="Failed to execute the main function:",
+            ErrorMessage=e,
+        )
+        traceback.print_exc()
+        raise e

scripts/animate.py

@@ -0,0 +1,214 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import argparse
+import gc
+import glob
+import logging
+import multiprocessing
+import os
+import traceback
+
+import netCDF4
+import numpy as np
+import PIL
+from mpl_toolkits.basemap import Basemap
+
+from libs.utils import setup_logging
+from libs.utils import verbose as vprint
+
+setup_logging()
+log = logging.getLogger(__name__)
+CONFIG = {}
+V = 1
+V_IGNORE = []  # Debug, Warning, Error
+# print(os.getcwd())
+
+
+def vis(files, layers=["SM1", "SM2", "SM3", "SM4", "SM5", "DD"], days="all"):
+    """Save the visualisation of the soil moisture content for given layers,
+    year files, and days.
+
+    Parameters
+    ----------
+    files : list
+        Iterable of the absolute or relative paths to the netcdf data ending with .nc.
+    layers : list
+        Soil layers to visualise. Default is ["SM1","SM2","SM3","SM4","SM5","DD"].
+    days : list / str
+        Iterable days of year range to visualise. Default is 'all'.
+
+    Returns
+    -------
+        None
+    """
+    inputs = [(file, layer, days) for file in files for layer in layers]
+    num_processes = multiprocessing.cpu_count()
+    with multiprocessing.Pool(processes=max(1, int(num_processes / 2))) as pool:
+        # pool.starmap(vis_process, [(f, l, days) for f in files for l in layers])
+        # pool.apply_async(vis_process, args=inputs)
+        for result in pool.imap_unordered(vis_process, inputs):
+            print(f"Result file: {result}", flush=True)
+    # pool.close()
+    # pool.join()
+
+
+def vis_process(input_nc_layer_days):
+    """Save the visualisation of the soil moisture content for a given layer,
+    year file, and days.
+
+    Parameters
+    ----------
+    input_nc : str
+        Absolute or relative path to the netcdf data ending with .nc.
+    layer : str
+        Soil layer to visualise.
+    days : list / str
+        Iterable days of year range to visualise.
+
+    Returns
+    -------
+        None
+    """
+    import matplotlib
+    import matplotlib.pyplot as plt
+
+    input_nc, layer, days = input_nc_layer_days
+    nc = netCDF4.Dataset(input_nc)
+    # ref = nc.variables["spatial_ref"]
+    # vprint(1, V, V_IGNORE, Debug=ref)
+    lons = nc.variables["x"][:]
+    lats = nc.variables["y"][:]
+
+    mp = Basemap(
+        projection="merc",
+        llcrnrlon=lons.min() - 0.02,  # lower longitude
+        llcrnrlat=lats.min() - 0.02,  # lower latitude
+        urcrnrlon=lons.max() + 0.02,  # uppper longitude
+        urcrnrlat=lats.max() + 0.02,  # uppper latitude
+        resolution="i",
+    )
+    lon, lat = np.meshgrid(lons, lats)  # this converts coordinates into 2D arrray
+    x, y = mp(lon, lat)  # mapping them together
+
+    layer_data = nc.variables[layer][:]
+    if days == "all":
+        days = np.arange(0, layer_data.shape[0])
+
+    # Generate daily images
+    for i in days:
+        day = i + 1
+        matplotlib.use("Agg")
+        plt.figure(figsize=(6, 8))  # figure size
+        c_scheme = mp.pcolor(x, y, np.squeeze(layer_data[i, :, :]), cmap="jet")
+        # mp.etopo()
+        mp.shadedrelief()
+        mp.drawcoastlines()
+        mp.drawstates()
+        mp.drawcountries()
+
+        mp.colorbar(c_scheme, location="right", pad="10%")
+
+        plt.title("Soil moisture content for day " + str(day) + " of the year")
+        plt.clim(layer_data.min(), layer_data.max())
+
+        plt.savefig(f".tmp/{layer}_{str(day)}.jpg")
+        plt.clf()
+    plt.close("all")
+
+    # Generate gif animation
+    image_frames = []  # creating a empty list to be appended later on
+    for day in days:
+        new_fram = PIL.Image.open(f".tmp/{layer}_{str(day + 1)}.jpg")
+        image_frames.append(new_fram)
+    o_name = input_nc.replace(".nc", f"_{layer}.gif")
+    image_frames[0].save(
+        o_name,
+        format="GIF",
+        append_images=image_frames[1:],
+        save_all=True,
+        duration=150,
+        loop=0,
+    )
+
+    # Delete temporary files
+    del (
+        nc,
+        layer_data,
+        mp,
+        image_frames,
+        c_scheme,
+        plt,
+        new_fram,
+        lon,
+        lat,
+        x,
+        y,
+        lons,
+        lats,
+        day,
+        input_nc,
+        layer,
+        days,
+        i,
+    )
+    gc.collect()
+    return o_name
+
+
+if __name__ == "__main__":
+    # Load Configs
+    parser = argparse.ArgumentParser(
+        description="Download rainfall data from Google Earth Engine for a range of dates.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        "-i",
+        "--input",
+        help="Absolute or relative path to the netcdf data ending with .nc. By dfault it is set to data.nc",
+        default="data.nc",
+    )
+    parser.add_argument(
+        "-l",
+        "--layer",
+        help="Soil layer to visualise. Default is all. Select between SM1 to SM5 or DD.",
+        default="all",
+    )
+    parser.add_argument(
+        "-d",
+        "--days",
+        help="Iterable days of year range to visualise. Default is all.",
+        default="all",
+    )
+
+    args = parser.parse_args()
+
+    try:
+        # Get the files
+        if os.path.isdir(args.input):
+            files = glob.glob(os.path.join(args.input, "*.nc"))
+        else:
+            files = [args.input]
+
+        # Get the layers
+        if args.layer == "all":
+            layers = ["SM1", "SM2", "SM3", "SM4", "SM5", "DD"]
+        else:
+            layers = str(args.layer).split(",")
+
+        # Get the days
+        if args.days == "all":
+            days = "all"
+        else:
+            days = str(args.days).split(",")
+
+        vis(files, layers, days)
+    except Exception as e:
+        vprint(
+            0,
+            V,
+            V_IGNORE,
+            Error="Failed to execute the main function:",
+            ErrorMessage=e,
+        )
+        traceback.print_exc()
+        raise e

scripts/display.py

@@ -0,0 +1,651 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import argparse
+import datetime
+import logging
+import os
+import traceback
+from typing import Dict, List
+
+import numpy as np
+import plotly.express as px
+import rasterio
+from dash import Dash, Input, Output, State, dcc, html
+from dash.exceptions import PreventUpdate
+
+from libs.utils import setup_logging
+from libs.utils import verbose as vprint
+from scripts.analyse import analyse
+
+setup_logging()
+log = logging.getLogger(__name__)
+CONFIG = {}
+V = 1
+V_IGNORE = []  # Debug, Warning, Error
+
+# ===============================================================================
+# Soil Moisture Comparison Tool App Layout
+# ===============================================================================
+
+colorscales = px.colors.named_colorscales()
+# external JavaScript files
+external_scripts = [
+    "https://www.google-analytics.com/analytics.js",
+    {"src": "https://cdn.polyfill.io/v2/polyfill.min.js"},
+    {
+        "src": "https://cdnjs.cloudflare.com/ajax/libs/lodash.js/4.17.10/lodash.core.js",
+        "integrity": "sha256-Qqd/EfdABZUcAxjOkMi8eGEivtdTkh3b65xCZL4qAQA=",
+        "crossorigin": "anonymous",
+    },
+]
+
+# external CSS stylesheets
+external_stylesheets = [
+    "https://codepen.io/chriddyp/pen/bWLwgP.css",
+    {
+        "href": "https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css",
+        "rel": "stylesheet",
+        "integrity": "sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO",
+        "crossorigin": "anonymous",
+    },
+]
+
+
+app = Dash(
+    __name__,
+    external_scripts=external_scripts,
+    external_stylesheets=external_stylesheets,
+    title="Soil Moisture Comparison Tool",
+    update_title="Loading the tool...",
+)
+
+# farm_name = "Arawa"
+# layer = "SM2"
+today = datetime.datetime.today()
+time_delta = datetime.timedelta(days=20)
+
+FAIL_IMAGE = app.get_asset_url("icons/fail.png")
+SUCCESS_IMAGE = app.get_asset_url("icons/success.png")
+WAIT_IMAGE = app.get_asset_url("icons/wait.png")
+
+current_working_directory = os.getcwd()
+app.index_template = os.path.join(current_working_directory, "templates", "index.html")
+# app.index_string = """
+#                     <!DOCTYPE html>
+#                     <html>
+#                         <head>
+#                             {%metas%}
+#                             <title>{%title%}</title>
+#                             {%favicon%}
+#                             {%css%}
+#                         </head>
+#                         <body>
+#                             <div class="col-12">
+#                                 <br>
+#                                 <h2>Soil Moisture Comparison Tool</h2>
+#                                 <br>
+#                                 <hr>
+#                             </div>
+#                             {%app_entry%}
+#                             <footer>
+#                                 {%config%}
+#                                 {%scripts%}
+#                                 {%renderer%}
+#                             </footer>
+#                             <div class="col-12">
+#                                 <hr>
+#                                 <br>
+#                                 Copyright @ 2023 Sydney Informatics Hub (SIH)
+#                                 <br>
+#                             </div>
+#                         </body>
+#                     </html>
+#                     """
+
+app.layout = html.Div(
+    [
+        # html.Div(
+        # className="app-header",
+        # children=[
+        #     html.Div('Soil Moisture Comparison Tool', className="app-header--title")
+        # ]
+        # ),
+        dcc.Store(id="farm-name-session", storage_type="session"),
+        html.Div(
+            [
+                html.P(
+                    """This tool will use the produced datacubes to compare the soil moisture of a farm against historic data.
+                    Please select the desired comaprison method and dates to make the comparison as in section A.
+                    Then choose the visualisation in section B to see the results.""",
+                    style={"font-size": "larger"},
+                ),
+                html.Hr(),
+                html.H3("A"),
+            ],
+            className="col-lg-12",
+            style={"padding-top": "1%", "padding-left": "1%"},
+        ),
+        html.Div(
+            [
+                html.Div(
+                    [
+                        # html.P("Write farm name/ID:"),
+                        dcc.Input(
+                            id="farm-name",
+                            type="text",
+                            placeholder="Farm name",
+                            style={"width": "80%"},
+                        ),
+                        html.Img(
+                            id="farm-image",
+                            src=WAIT_IMAGE,
+                            style={"width": "30px", "margin-left": "15px"},
+                        ),
+                    ],
+                    className="col-lg-5",
+                    # style = {'padding-top':'1%', 'padding-left':'1%'}
+                ),
+                html.Div(
+                    [
+                        html.P(),
+                    ],
+                    className="col-lg-7",
+                    # style = {'padding-top':'1%', 'padding-left':'1%'}
+                ),
+            ],
+            className="row",
+            style={"padding-top": "1%", "padding-left": "1%"},
+        ),
+        html.Div(
+            [
+                html.Div(
+                    [
+                        html.P("Select soil layer:"),
+                        dcc.Dropdown(
+                            id="layer-dropdown",
+                            options=[
+                                {"label": "SM1", "value": "SM1"},
+                                {"label": "SM2", "value": "SM2"},
+                                {"label": "SM3", "value": "SM3"},
+                                {"label": "SM4", "value": "SM4"},
+                                {"label": "SM5", "value": "SM5"},
+                                {"label": "DD", "value": "DD"},
+                            ],
+                            value="SM2",
+                        ),
+                    ],
+                    className="col-lg-4",
+                    style={"padding": "1%"},
+                ),
+                html.Div(
+                    [
+                        html.P("Select the historic years to compare against:"),
+                        dcc.Dropdown(
+                            id="historic-dropdown",
+                            options=[
+                                {"label": year, "value": year} for year in range(1, 20)
+                            ],
+                            value=2,
+                        ),
+                    ],
+                    className="col-lg-4",
+                    style={"padding": "1%"},
+                ),
+                html.Div(
+                    [
+                        html.P("Select the most recent window of dates to analyse:"),
+                        dcc.DatePickerRange(
+                            id="window-select",
+                            min_date_allowed=datetime.date(2000, 1, 1),
+                            max_date_allowed=today.strftime("%Y-%m-%d"),
+                            initial_visible_month=datetime.date(2023, 1, 1),
+                            clearable=False,
+                            display_format="YYYY-MM-DD",
+                            start_date_placeholder_text="Start date",
+                            end_date_placeholder_text="End date",
+                            style={"width": "100%"},
+                        ),
+                    ],
+                    className="col-lg-4",
+                    style={"padding": "1%"},
+                ),
+            ],
+            className="row",
+            style={"padding-top": "1%"},
+        ),
+        html.Div(
+            [
+                html.Div(
+                    [
+                        html.P("Select window aggregation method:"),
+                        dcc.Dropdown(
+                            id="w-aggregation-dropdown",
+                            options=[
+                                {"label": "Mean", "value": "mean"},
+                                {"label": "Median", "value": "median"},
+                                {"label": "Max", "value": "max"},
+                                {"label": "Min", "value": "min"},
+                                {"label": "Sum", "value": "sum"},
+                                {"label": "std", "value": "std"},
+                                {"label": "var", "value": "var"},
+                            ],
+                            value="mean",
+                        ),
+                    ],
+                    className="col-lg-6",
+                    style={"padding": "1%"},
+                ),
+                html.Div(
+                    [
+                        html.P("Select historic aggregation method:"),
+                        dcc.Dropdown(
+                            id="h-aggregation-dropdown",
+                            options=[
+                                {"label": "Mean", "value": "mean"},
+                                {"label": "Median", "value": "median"},
+                                {"label": "Max", "value": "max"},
+                                {"label": "Min", "value": "min"},
+                                {"label": "Sum", "value": "sum"},
+                                {"label": "std", "value": "std"},
+                                {"label": "var", "value": "var"},
+                            ],
+                            value="mean",
+                        ),
+                    ],
+                    className="col-lg-6",
+                    style={"padding": "1%"},
+                ),
+            ],
+            className="row",
+            # style = {'padding-top':'1%'}
+        ),
+        html.Div(
+            [
+                html.Button("Generate Images", id="generate-button"),
+                html.Br(),
+                html.Hr(),
+            ],
+            className="col-lg-12",
+            style={"margin-bottom": "1%"},
+        ),
+        html.Div(
+            [
+                html.H3("B"),
+            ],
+            className="col-lg-12",
+            style={"padding-top": "1%", "padding-left": "1%"},
+        ),
+        html.Div(
+            [
+                html.Div(
+                    [
+                        html.P("Select visualisation name:"),
+                        dcc.Dropdown(id="visualisation-select"),
+                    ],
+                    className="col-lg-6",
+                    style={"padding": "1%"},
+                ),
+                html.Div(
+                    [
+                        html.P("Select your palette:"),
+                        dcc.Dropdown(
+                            id="platter-dropdown", options=colorscales, value="viridis"
+                        ),
+                    ],
+                    className="col-lg-6",
+                    style={"padding": "1%"},
+                ),
+            ],
+            className="row",
+            # style = {'padding-top':'1%'}
+        ),
+        html.Div(
+            [
+                html.Hr(),
+                html.H3("Results"),
+                dcc.Graph(id="graph"),
+            ],
+            className="col-lg-12",
+            style={"padding-top": "1%"},
+        ),
+        # html.Div(
+        # className="app-footer",
+        # children=[
+        #     html.Div(f"Copyright @ {today.strftime('%Y')} Sydney Informatics Hub (SIH)", className="app-footer--copyright")
+        # ]
+        # ),
+    ],
+    className="container-fluid",
+)
+
+# ==================================================================================================
+# Functions
+# ==================================================================================================
+
+
+def find_analyses(path):
+    """Find all the analysis files in a directory.
+
+    Parameters
+    ----------
+    path: str
+        Path to the directory containing the analysis files
+
+    Returns
+    -------
+    files: list
+        List of analysis files
+    """
+    files = [f for f in os.listdir(path) if f.endswith(".tif")]
+    return files
+
+
+def open_image(path):
+    """Open a raster image and return the data and coordinates.
+
+    Parameters
+    ----------
+    path: str
+        path to the raster image
+
+    Returns
+    -------
+    band1: np.array
+        The raster data
+    lons: np.array
+        The longitude coordinates
+    lats: np.array
+        The latitude coordinates
+    """
+    with rasterio.open(path) as src:
+        band1 = src.read(1)
+        print("Band1 has shape", band1.shape)
+        height = band1.shape[0]
+        width = band1.shape[1]
+        cols, rows = np.meshgrid(np.arange(width), np.arange(height))
+        xs, ys = rasterio.transform.xy(src.transform, rows, cols)
+        lons = np.array(xs)
+        lats = np.array(ys)
+
+    return band1, lons, lats
+
+
+def perform_analysis(
+    input,
+    window_start,
+    window_end,
+    historic_years: int,
+    layer: str,
+    match_raster: str = None,
+    output: str = None,
+    agg_history: str = "mean",
+    agg_window: str = "mean",
+    comparison: str = "diff",
+    **args,
+) -> Dict[str, str]:
+    """Perform the analysis.
+
+    This is a wrapper function for the analysis module. It takes the input parameters and passes them to the analysis module.
+
+    Parameters
+    ----------
+    input : str
+        path to the input data
+    window_start : str
+        start date of the window
+    window_end : str
+        end date of the window
+    historic_years : int
+        number of years to use for the historic data
+    layer : str
+        layer to use for the analysis
+    match_raster : str, optional
+        path to the raster to match the output to, by default None
+    output : str, optional
+        path to the output file, by default None
+    agg_history : str, optional
+        aggregation method for the historic data, by default "mean"
+    agg_window : str, optional
+        aggregation method for the window data, by default "mean"
+    comparison : str, optional
+        comparison method for the window and historic data, by default "diff"
+
+    Returns
+    -------
+        files: dict
+            Dict of analysis files
+    """
+    files = analyse(
+        input=input,
+        window_start=window_start,
+        window_end=window_end,
+        historic_years=historic_years,
+        agg_window=agg_window,
+        agg_history=agg_history,
+        comparison=comparison,
+        layer=layer,
+        output=output,
+        match_raster=match_raster,
+    )
+    return files
+
+
+# ====================================================================================================
+# Callbacks
+# ====================================================================================================
+
+
+@app.callback(
+    [
+        Output("farm-name-session", "data"),
+        Output("farm-image", "src"),
+    ],
+    [Input("farm-name", "value"), State("farm-name-session", "data")],
+)
+def update_session(farm_name, session):
+    session = farm_name
+    if farm_name is None or farm_name == "":
+        session = ""
+        image = WAIT_IMAGE
+    else:
+        print(f"Getting some data about farm: {farm_name}")
+
+        # if the path does not exist, do not update the session
+        real_path = INPUT.format(farm_name)
+        print(f"Checking {real_path}")
+        if os.path.exists(real_path):
+            session = farm_name
+            image = SUCCESS_IMAGE
+        else:
+            session = ""
+            image = FAIL_IMAGE
+
+    print(f"\n\nSession updated to {session}")
+    print(f"Image updated to {image}\n\n")
+
+    return session, image
+
+
+@app.callback(
+    Output("farm-name", "value"),
+    Input("farm-name-session", "modified_timestamp"),
+    State("farm-name-session", "data"),
+)
+def display_name_from_session(timestamp, name):
+    print(f"Updating the farm name from the session: {name}")
+    if timestamp is not None:
+        return name
+    else:
+        return ""
+
+
+@app.callback(
+    Output("visualisation-select", "options"),
+    # Input("farm-name", "value"),
+    Input("layer-dropdown", "value"),
+    Input("window-select", "start_date"),
+    Input("window-select", "end_date"),
+    Input("historic-dropdown", "value"),
+    Input("w-aggregation-dropdown", "value"),
+    Input("h-aggregation-dropdown", "value"),
+    Input("generate-button", "n_clicks"),
+    State("farm-name-session", "data"),
+)
+def get_analysis(
+    layer, window_start, window_end, historic_years, w_agg, h_agg, n_clicks, farm_name
+) -> List[Dict[str, str]]:
+    """Get the analysis files and return them as a list of dicts.
+
+    Parameters
+    ----------
+    layer : str
+        layer to use for the analysis
+    window_start : str
+        start date of the window
+    window_end : str
+        end date of the window
+    historic_years : int
+        number of years to use for the historic data
+    w_agg : str
+        aggregation method for the window data
+    h_agg : str
+        aggregation method for the historic data
+    n_clicks : int
+        number of times the generate button has been clicked
+
+    Returns
+    -------
+    files : list
+        list of dicts of analysis files
+    """
+    print("\nAnalysis callback triggered")
+
+    if n_clicks == 0 or n_clicks is None:
+        raise PreventUpdate
+
+    path = f"/home/sahand/Data/results_default/{farm_name}/soilwatermodel"
+    # window_start = datetime.datetime.strptime(window_start, '%Y-%m-%d')
+    # window_end = datetime.datetime.strptime(window_end, '%Y-%m-%d')
+    print(f"\nPath: {path}\n")
+
+    files = perform_analysis(
+        input=path,
+        window_start=window_start,
+        window_end=window_end,
+        historic_years=historic_years,
+        layer=layer,
+        agg_window=w_agg,
+        agg_history=h_agg,
+        comparison="diff",
+        output=None,
+        match_raster=None,
+    )
+    print(path)
+    print(
+        f"n_clicks: {n_clicks}\n"
+        + f"window_start: {window_start}\n"
+        + f"window_end: {window_end}\n"
+        + f"historic_years: {historic_years}\n"
+        + f"layer: {layer}\n"
+        + f"agg_window: {w_agg}\n"
+        + f"agg_history: {h_agg}\n"
+        + "comparison: 'diff'\n"
+        + f"output: {None}\n"
+        + f"match_raster: {None}\n"
+    )
+    print(files)
+    files = {
+        i: [
+            " ".join(files[i].split("/")[-1].split(".")[0].split("-")).capitalize(),
+            files[i],
+        ]
+        for i in files
+    }
+    print(files)
+    options = [{"label": files[i][0], "value": files[i][1]} for i in files]
+
+    return options
+
+
+@app.callback(
+    Output("graph", "figure"),
+    Input("visualisation-select", "value"),
+    Input("platter-dropdown", "value"),
+)
+def change_colorscale(file, palette):
+    """Display the selected visualisation and change the colorscale of the
+    visualisation.
+
+    Parameters
+    ----------
+    file : str
+        path to the visualisation file
+    palette : str
+        name of the colorscale to use
+
+    Returns
+    -------
+    fig : plotly.graph_objects.Figure
+        plotly figure object
+    """
+
+    band1, lons_a, lats_a = open_image(file)
+
+    # Get the second dimension of the lons
+    lats = lats_a[:, 0]
+    lons = lons_a[0, :]
+
+    print(lons.shape, lons)
+    print(lats.shape, lats)
+    print(band1.shape, band1)
+
+    fig = px.imshow(band1, x=lons, y=lats, color_continuous_scale=palette)
+    fig.update(
+        data=[
+            {
+                "customdata": np.stack((band1, lats_a, lons_a), axis=-1),
+                "hovertemplate": "<b>SM</b>: %{customdata[0]}<br>"
+                + "<b>Lat</b>: %{customdata[1]}<br>"
+                + "<b>Lon</b>: %{customdata[2]}<br>"
+                + "<extra></extra>",
+            }
+        ]
+    )
+    print("Render successful")
+    return fig
+
+
+# ==============================================================================
+# Main
+# ==============================================================================
+
+if __name__ == "__main__":
+    # Load Configs
+    parser = argparse.ArgumentParser(
+        description="Download rainfall data from Google Earth Engine for a range of dates.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        "-i",
+        "--input",
+        help="Absolute or relative path to the netcdf data directory for each farm. Should be in this format: '/path/to/farm/{}/soilwatermodel'",
+        default=os.path.join(
+            os.path.expanduser("~"), "Data/results_default/{}/soilwatermodel"
+        ),
+    )
+
+    args = parser.parse_args()
+    INPUT = args.input
+
+    try:
+        app.run_server(debug=True)
+    except Exception as e:
+        vprint(
+            0,
+            V,
+            V_IGNORE,
+            Error="Failed to execute the main function:",
+            ErrorMessage=e,
+        )
+        traceback.print_exc()
+        raise e

scripts/sankey.py

@@ -0,0 +1,185 @@
+# -*- coding: utf-8 -*-
+# import json
+# import urllib
+
+import plotly.graph_objects as go
+from dash import Dash, Input, Output, dcc, html
+from dash.exceptions import PreventUpdate
+
+app = Dash(__name__)
+
+app.layout = html.Div(
+    [
+        html.H1("Sankey diagram"),
+        html.P("Data"),
+        dcc.Input(id="SM1", placeholder="SM1"),
+        dcc.Input(id="SM2", placeholder="SM2"),
+        dcc.Input(id="SM3", placeholder="SM3"),
+        dcc.Input(id="SM4", placeholder="SM4"),
+        dcc.Input(id="SM5", placeholder="SM5"),
+        dcc.Input(id="BS1", placeholder="BS1"),
+        dcc.Input(id="BS2", placeholder="BS2"),
+        dcc.Input(id="BS3", placeholder="BS3"),
+        dcc.Input(id="BS4", placeholder="BS4"),
+        dcc.Input(id="BS5", placeholder="BS5"),
+        dcc.Input(id="rainfall", placeholder="Rainfall"),
+        dcc.Input(id="et", placeholder="ET"),
+        html.Button("Calculate", id="submit"),
+        html.H4("Water conservation diagram"),
+        dcc.Graph(id="graph"),
+        html.P("Opacity"),
+        dcc.Slider(id="slider", min=0, max=1, value=0.5, step=0.1),
+    ]
+)
+
+
+def get_excess_clipped_moist_ETb(moisture, bucket_size):
+    excess = 0 if moisture <= bucket_size else moisture - bucket_size
+    moist_out = moisture if moisture <= bucket_size else bucket_size
+    moist_out = moist_out if moisture >= 0 else 0
+    ETb = 0 if moisture >= 0 else -moisture
+    return ETb, moist_out, excess
+
+
+def model(prev_state, R, ET, buckets):
+    old = {x + "_prev": y for x, y in prev_state.items()}
+
+    # layer 0 to 5 cm
+    SM1 = 0.2 * old["SM1_prev"] + R - 0.5 * ET
+    Exc1, SM1, ETb1 = get_excess_clipped_moist_ETb(SM1, buckets["BS1"])
+
+    # layer 5 to 15 cm
+    SM2 = 0.95 * old["SM2_prev"] + 0.8 * old["SM1_prev"] - 0.2 * ETb1 + Exc1
+    Exc2, SM2, ETb2 = get_excess_clipped_moist_ETb(SM2, buckets["BS2"])
+
+    # Run Off
+    if (SM1 > buckets["BS1"]) * (SM2 > buckets["BS2"]):
+        Exc2 = 0
+        run_off = Exc2
+    else:
+        run_off = 0
+
+    # layer 15 to 30 cm
+    SM3 = 0.95 * old["SM3_prev"] + 0.05 * old["SM2_prev"] - 0.15 * ETb2 + Exc2
+    Exc3, SM3, ETb3 = get_excess_clipped_moist_ETb(SM3, buckets["BS3"])
+
+    # layer 30 to 60 cm
+    SM4 = 0.95 * old["SM4_prev"] + 0.05 * old["SM3_prev"] - 0.1 * ETb3 + Exc3
+    Exc4, SM4, ETb4 = get_excess_clipped_moist_ETb(SM4, buckets["BS4"])
+
+    # layer 60 to 100 cm
+    SM5 = 0.99 * old["SM5_prev"] + 0.01 * old["SM4_prev"] - 0.05 * ETb4 + Exc4
+    Exc5, SM5, ETb5 = get_excess_clipped_moist_ETb(SM5, buckets["BS5"])
+
+    DD = 0.01 * old["SM5_prev"] + Exc5
+
+    return [
+        {
+            "SM1": SM1,
+            "SM2": SM2,
+            "SM3": SM3,
+            "SM4": SM4,
+            "SM5": SM5,
+            "DD": DD,
+        },
+        {
+            "ETb1": ETb1,
+            "ETb2": ETb2,
+            "ETb3": ETb3,
+            "ETb4": ETb4,
+            "ETb5": ETb5,
+        },
+        {
+            "Exc1": Exc1,
+            "Exc2": Exc2,
+            "Exc3": Exc3,
+            "Exc4": Exc4,
+            "Exc5": Exc5,
+        },
+        run_off,
+    ]
+
+
+@app.callback(
+    Output("graph", "figure"),
+    Input("slider", "value"),
+    Input("SM1", "value"),
+    Input("SM2", "value"),
+    Input("SM3", "value"),
+    Input("SM4", "value"),
+    Input("SM5", "value"),
+    Input("BS1", "value"),
+    Input("BS2", "value"),
+    Input("BS3", "value"),
+    Input("BS4", "value"),
+    Input("BS5", "value"),
+    Input("rainfall", "value"),
+    Input("et", "value"),
+    Input("submit", "n_clicks"),
+)
+def display_sankey(
+    opacity, SM1, SM2, SM3, SM4, SM5, BS1, BS2, BS3, BS4, BS5, R, ET, submit
+):
+    if submit == 0 or submit is None:
+        raise PreventUpdate
+    buckets = {
+        "BS1": BS1,
+        "BS2": BS2,
+        "BS3": BS3,
+        "BS4": BS4,
+        "BS5": BS5,
+    }
+    prev_state = {
+        "SM1": SM1,
+        "SM2": SM2,
+        "SM3": SM3,
+        "SM4": SM4,
+        "SM5": SM5,
+    }
+    SMs, ETbs, Excs, run_off = model(prev_state, R, ET, buckets)
+
+    node = dict(
+        pad=15,
+        thickness=20,
+        line=dict(color="black", width=0.5),
+        label=[
+            "SM1_prev",
+            "SM2_prev",
+            "SM3_prev",
+            "SM4_prev",
+            "SM5_prev",  # 0-4
+            "SM1",
+            "SM2",
+            "SM3",
+            "SM4",
+            "SM5",  # 5-9
+            "ETb1",
+            "ETb2",
+            "ETb3",
+            "ETb4",
+            "ETb5",  # 10-14
+            "Exc1",
+            "Exc2",
+            "Exc3",
+            "Exc4",
+            "Exc5",  # 15-19
+            "R",
+            "ET",
+            "DD",
+            "runoff",
+        ],  # 20, 21, 22, 23
+        color="blue",
+    )
+
+    link = dict(
+        source=[0, 0, 1, 1, 2, 2, 3, 3, 4, 4],  # indices correspond to labels
+        target=[5, 6, 6, 7, 7, 8, 8, 9, 9, 22],
+        value=[],
+    )
+
+    fig = go.Figure(go.Sankey(link=link, node=node))
+    fig.update_layout(font_size=10)
+    return fig
+
+
+app.run_server(debug=True)

scripts/simple_html.py

@@ -0,0 +1,391 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import argparse
+import os
+
+import numpy as np
+import plotly.express as px
+import plotly.graph_objs as go
+import plotly.io as pio
+import rasterio
+from matplotlib.colors import LightSource, LinearSegmentedColormap
+
+
+def main(f_path, f_name, farm_name):
+    """Main function to visualise the soil moisture content for a given layer,
+    year file, and days.
+
+    Parameters
+    ----------
+    f_path : str
+        Path to the file.
+    f_name : str
+        Name of the file.
+    farm_name : str
+        Name of the farm.
+
+    Returns
+    -------
+    None
+    """
+
+    # f_path = ""#"/home/sahand/Projects/PIPE-3788 GRDC SoilWaterNow Deployment/work/v3/Arawa 2019-2023/c8/1af25ced023e58c46f4403a155210d/soilwatermodel v3/analysis"
+    # f_name = ""#"delta-2022-12-20_2023-01-10-SM2-w_mean-h_mean.tif"
+    file = os.path.join(f_path, f_name)
+    # farm_name = "Arawa"
+    # layer = "SM2"
+
+    with rasterio.open(file) as src:
+        band1 = src.read(1)
+        print("Band1 has shape", band1.shape)
+        height = band1.shape[0]
+        width = band1.shape[1]
+        cols, rows = np.meshgrid(np.arange(width), np.arange(height))
+        xs, ys = rasterio.transform.xy(src.transform, rows, cols)
+        lons = np.array(xs)
+        lats = np.array(ys)
+
+    bamako = [
+        [0.0, "rgb(0, 63, 76)"],  # a scientific colorscale for dem data
+        [0.1, "rgb(29, 81, 59)"],
+        [0.2, "rgb(55, 98, 43)"],
+        [0.3, "rgb(79, 114, 30)"],
+        [0.4, "rgb(103, 129, 16)"],
+        [0.5, "rgb(136, 142, 2)"],
+        [0.6, "rgb(169, 154, 21)"],
+        [0.7, "rgb(192, 171, 45)"],
+        [0.8, "rgb(214, 188, 74)"],
+        [0.9, "rgb(234, 209, 112)"],
+        [1.0, "rgb(254, 229, 152)"],
+    ]
+
+    # fig= px.imshow(band1, color_continuous_scale=bamako)
+    # my_layout= dict(title_text=f"{farm_name}--{f_name.split('.')[0]}", title_x=0.5, width =700, height=500, template='none',
+    #                   coloraxis_colorbar=dict(len=0.75, thickness=25))
+    # fig.update_layout(**my_layout)
+
+    ve = 1
+    ls = LightSource(azdeg=315, altdeg=45)
+    hilsh = ls.hillshade(band1, vert_exag=ve)  # , dx=dx, dy=dy)
+    fig = px.imshow(hilsh, color_continuous_scale=bamako)
+
+    # fig.update_traces(customdata=elevation, hovertemplate='moisture: %{customdata}<extra></extra>')
+    # fig.update_layout(**my_layout)
+    # fig.show()#
+
+    cm_data = [
+        [0.0011753, 0.25004, 0.3],
+        [0.0039003, 0.25157, 0.29861],
+        [0.006602, 0.25305, 0.29722],
+        [0.0092914, 0.25456, 0.29581],
+        [0.012175, 0.25604, 0.2944],
+        [0.014801, 0.25755, 0.293],
+        [0.01745, 0.25907, 0.29161],
+        [0.020096, 0.26057, 0.29018],
+        [0.022743, 0.26208, 0.28878],
+        [0.025398, 0.26361, 0.28735],
+        [0.028064, 0.26511, 0.28592],
+        [0.030746, 0.26664, 0.2845],
+        [0.033437, 0.26816, 0.28309],
+        [0.036369, 0.26971, 0.28165],
+        [0.039136, 0.27124, 0.28021],
+        [0.041941, 0.27278, 0.27878],
+        [0.044597, 0.27432, 0.27734],
+        [0.047216, 0.27589, 0.27591],
+        [0.049816, 0.27743, 0.27443],
+        [0.052305, 0.279, 0.27298],
+        [0.054846, 0.28053, 0.27151],
+        [0.057331, 0.28213, 0.27007],
+        [0.059903, 0.28369, 0.2686],
+        [0.062324, 0.28526, 0.2671],
+        [0.064737, 0.28684, 0.26564],
+        [0.067177, 0.28844, 0.26416],
+        [0.069647, 0.29003, 0.26267],
+        [0.072007, 0.29165, 0.26118],
+        [0.074435, 0.29324, 0.25969],
+        [0.076803, 0.29484, 0.25817],
+        [0.079232, 0.29647, 0.25667],
+        [0.081738, 0.29808, 0.25518],
+        [0.084163, 0.29972, 0.25365],
+        [0.086556, 0.30134, 0.25213],
+        [0.089008, 0.30298, 0.25059],
+        [0.091487, 0.30463, 0.24907],
+        [0.093906, 0.3063, 0.24754],
+        [0.096327, 0.30793, 0.24598],
+        [0.098845, 0.30961, 0.24443],
+        [0.10132, 0.31127, 0.24288],
+        [0.10384, 0.31294, 0.24134],
+        [0.10637, 0.31461, 0.23978],
+        [0.10884, 0.31632, 0.23821],
+        [0.1114, 0.318, 0.23666],
+        [0.11391, 0.3197, 0.23509],
+        [0.11648, 0.32139, 0.23347],
+        [0.11904, 0.32311, 0.23191],
+        [0.12156, 0.32481, 0.2303],
+        [0.12416, 0.32653, 0.22873],
+        [0.12676, 0.32826, 0.22712],
+        [0.12938, 0.33, 0.22551],
+        [0.132, 0.33173, 0.22389],
+        [0.13465, 0.33348, 0.22228],
+        [0.13728, 0.33524, 0.22068],
+        [0.13988, 0.33699, 0.21904],
+        [0.14258, 0.33874, 0.21741],
+        [0.14528, 0.34051, 0.21578],
+        [0.14793, 0.34229, 0.21413],
+        [0.15062, 0.34406, 0.21248],
+        [0.15331, 0.34584, 0.21083],
+        [0.15607, 0.34765, 0.20917],
+        [0.15879, 0.34944, 0.20751],
+        [0.16154, 0.35125, 0.20585],
+        [0.16429, 0.35306, 0.20419],
+        [0.16705, 0.35489, 0.20248],
+        [0.16983, 0.35671, 0.20078],
+        [0.17259, 0.35853, 0.1991],
+        [0.1754, 0.36036, 0.19742],
+        [0.17824, 0.36222, 0.19572],
+        [0.18102, 0.36406, 0.19403],
+        [0.18387, 0.36593, 0.1923],
+        [0.18672, 0.36779, 0.19058],
+        [0.18959, 0.36966, 0.18887],
+        [0.19246, 0.37154, 0.18714],
+        [0.19535, 0.37344, 0.18542],
+        [0.19825, 0.37533, 0.18364],
+        [0.20116, 0.37724, 0.18189],
+        [0.20413, 0.37915, 0.18014],
+        [0.20706, 0.38108, 0.17841],
+        [0.21001, 0.383, 0.1766],
+        [0.21297, 0.38493, 0.17486],
+        [0.21599, 0.38688, 0.17306],
+        [0.21898, 0.38884, 0.17127],
+        [0.22199, 0.39081, 0.1695],
+        [0.22501, 0.39277, 0.16768],
+        [0.22809, 0.39476, 0.16585],
+        [0.23115, 0.39674, 0.16407],
+        [0.23423, 0.39875, 0.16224],
+        [0.23735, 0.40078, 0.16037],
+        [0.24043, 0.40279, 0.15857],
+        [0.24357, 0.40482, 0.15671],
+        [0.24673, 0.40687, 0.15484],
+        [0.24989, 0.40893, 0.15296],
+        [0.2531, 0.411, 0.15109],
+        [0.25629, 0.41308, 0.14922],
+        [0.25954, 0.41517, 0.14733],
+        [0.26278, 0.41728, 0.14546],
+        [0.26606, 0.41939, 0.14352],
+        [0.26935, 0.42151, 0.14161],
+        [0.27265, 0.42366, 0.13963],
+        [0.276, 0.42581, 0.13775],
+        [0.27935, 0.42799, 0.13575],
+        [0.28274, 0.43017, 0.13382],
+        [0.28611, 0.43238, 0.13185],
+        [0.28956, 0.43459, 0.12989],
+        [0.293, 0.43682, 0.12788],
+        [0.29648, 0.43906, 0.12588],
+        [0.29998, 0.44132, 0.12382],
+        [0.3035, 0.44361, 0.12179],
+        [0.30706, 0.44589, 0.11977],
+        [0.31066, 0.44821, 0.11773],
+        [0.31426, 0.45054, 0.11568],
+        [0.3179, 0.45289, 0.11362],
+        [0.32156, 0.45525, 0.11153],
+        [0.32526, 0.45763, 0.10944],
+        [0.32901, 0.46003, 0.10731],
+        [0.33277, 0.46245, 0.10513],
+        [0.33654, 0.4649, 0.10304],
+        [0.34037, 0.46736, 0.10085],
+        [0.34422, 0.46984, 0.098693],
+        [0.34812, 0.47233, 0.096453],
+        [0.35204, 0.47486, 0.094325],
+        [0.35601, 0.47738, 0.092102],
+        [0.36, 0.47994, 0.089849],
+        [0.36403, 0.48251, 0.087617],
+        [0.36811, 0.4851, 0.085276],
+        [0.37221, 0.48772, 0.082993],
+        [0.37638, 0.49034, 0.080686],
+        [0.38057, 0.49296, 0.078304],
+        [0.3848, 0.49562, 0.075955],
+        [0.38909, 0.49829, 0.073656],
+        [0.39341, 0.50095, 0.071224],
+        [0.39779, 0.50362, 0.068802],
+        [0.4022, 0.5063, 0.066264],
+        [0.40668, 0.50898, 0.063774],
+        [0.4112, 0.51164, 0.061172],
+        [0.41578, 0.51431, 0.058651],
+        [0.4204, 0.51695, 0.05607],
+        [0.42509, 0.51956, 0.053359],
+        [0.42981, 0.52214, 0.050712],
+        [0.4346, 0.5247, 0.047969],
+        [0.43943, 0.52722, 0.045272],
+        [0.44432, 0.52967, 0.042483],
+        [0.44924, 0.53207, 0.039697],
+        [0.45421, 0.5344, 0.036906],
+        [0.45922, 0.53666, 0.034001],
+        [0.46428, 0.53884, 0.031435],
+        [0.46935, 0.54092, 0.028957],
+        [0.47444, 0.54291, 0.026597],
+        [0.47956, 0.5448, 0.024363],
+        [0.48468, 0.54659, 0.022265],
+        [0.48982, 0.54827, 0.020312],
+        [0.49497, 0.54985, 0.018512],
+        [0.5001, 0.55132, 0.016876],
+        [0.50524, 0.55269, 0.015412],
+        [0.51038, 0.55397, 0.014132],
+        [0.5155, 0.55517, 0.013033],
+        [0.52063, 0.55628, 0.01218],
+        [0.52575, 0.55735, 0.011416],
+        [0.53088, 0.55835, 0.010839],
+        [0.53601, 0.55933, 0.010585],
+        [0.54115, 0.5603, 0.010612],
+        [0.54633, 0.56126, 0.010945],
+        [0.55153, 0.56225, 0.011649],
+        [0.55677, 0.56328, 0.012516],
+        [0.56207, 0.56438, 0.01365],
+        [0.56742, 0.56555, 0.015118],
+        [0.57285, 0.56684, 0.016927],
+        [0.57835, 0.56824, 0.019098],
+        [0.58394, 0.56978, 0.021653],
+        [0.58963, 0.57149, 0.024617],
+        [0.59541, 0.57336, 0.028014],
+        [0.60129, 0.5754, 0.03187],
+        [0.60728, 0.57763, 0.036408],
+        [0.61335, 0.58006, 0.041287],
+        [0.61954, 0.58268, 0.046299],
+        [0.62581, 0.58551, 0.051565],
+        [0.63218, 0.58853, 0.057023],
+        [0.63862, 0.59176, 0.062625],
+        [0.64514, 0.59517, 0.068288],
+        [0.65173, 0.59876, 0.074092],
+        [0.65837, 0.60254, 0.079969],
+        [0.66506, 0.60647, 0.085984],
+        [0.6718, 0.61057, 0.0922],
+        [0.67856, 0.6148, 0.098456],
+        [0.68534, 0.61918, 0.10476],
+        [0.69213, 0.62366, 0.11129],
+        [0.69891, 0.62825, 0.11783],
+        [0.70569, 0.63293, 0.12447],
+        [0.71245, 0.63769, 0.13125],
+        [0.71918, 0.64252, 0.1381],
+        [0.72588, 0.6474, 0.14503],
+        [0.73253, 0.65232, 0.15201],
+        [0.73912, 0.65728, 0.15912],
+        [0.74564, 0.66226, 0.16629],
+        [0.7521, 0.66724, 0.17352],
+        [0.75847, 0.67223, 0.18082],
+        [0.76475, 0.67721, 0.18823],
+        [0.77094, 0.68215, 0.19562],
+        [0.77702, 0.68709, 0.20308],
+        [0.78299, 0.69199, 0.21059],
+        [0.78886, 0.69684, 0.21813],
+        [0.79459, 0.70164, 0.2257],
+        [0.80021, 0.7064, 0.23325],
+        [0.8057, 0.7111, 0.24084],
+        [0.81106, 0.71574, 0.24845],
+        [0.81631, 0.72031, 0.25601],
+        [0.82142, 0.72482, 0.26361],
+        [0.8264, 0.72926, 0.27115],
+        [0.83127, 0.73364, 0.2787],
+        [0.83602, 0.73795, 0.28619],
+        [0.84066, 0.74221, 0.29369],
+        [0.84519, 0.74639, 0.30114],
+        [0.84961, 0.75052, 0.30859],
+        [0.85394, 0.75459, 0.31598],
+        [0.85818, 0.7586, 0.32333],
+        [0.86234, 0.76255, 0.33065],
+        [0.86641, 0.76646, 0.33794],
+        [0.87042, 0.77032, 0.3452],
+        [0.87436, 0.77414, 0.35241],
+        [0.87824, 0.77792, 0.3596],
+        [0.88207, 0.78167, 0.36676],
+        [0.88585, 0.78537, 0.37388],
+        [0.88958, 0.78905, 0.38098],
+        [0.89328, 0.7927, 0.38804],
+        [0.89694, 0.79632, 0.39508],
+        [0.90057, 0.79992, 0.40209],
+        [0.90417, 0.8035, 0.40909],
+        [0.90775, 0.80706, 0.41606],
+        [0.9113, 0.8106, 0.423],
+        [0.91484, 0.81412, 0.42995],
+        [0.91835, 0.81764, 0.43686],
+        [0.92186, 0.82114, 0.44376],
+        [0.92534, 0.82462, 0.45064],
+        [0.92882, 0.8281, 0.45751],
+        [0.93228, 0.83157, 0.46438],
+        [0.93573, 0.83502, 0.47121],
+        [0.93918, 0.83847, 0.47804],
+        [0.94261, 0.84192, 0.48486],
+        [0.94603, 0.84535, 0.49169],
+        [0.94945, 0.84878, 0.49849],
+        [0.95286, 0.85221, 0.50528],
+        [0.95627, 0.85563, 0.51207],
+        [0.95966, 0.85904, 0.51886],
+        [0.96305, 0.86246, 0.52563],
+        [0.96643, 0.86586, 0.53241],
+        [0.96981, 0.86927, 0.53917],
+        [0.97318, 0.87268, 0.54594],
+        [0.97654, 0.87608, 0.55269],
+        [0.9799, 0.87948, 0.55945],
+        [0.98325, 0.88288, 0.5662],
+        [0.98659, 0.88628, 0.57296],
+        [0.98993, 0.88968, 0.57971],
+        [0.99326, 0.89308, 0.58645],
+        [0.99658, 0.89648, 0.5932],
+        [0.9999, 0.89988, 0.59995],
+    ]
+
+    bamako_map = LinearSegmentedColormap.from_list("bamako", cm_data)
+    rgb = ls.shade(
+        band1, cmap=bamako_map, blend_mode="overlay", vert_exag=ve
+    )  # , dx=dx, dy=dy)
+    img = np.array((255 * rgb[:, :, :3]), int)
+
+    fig = go.Figure(
+        go.Image(
+            z=img,
+            colormodel="rgb",
+            customdata=np.stack((band1, lats, lons), axis=-1),
+            hovertemplate="<b>SM</b>: %{customdata[0]}<br>"
+            + "<b>Lat</b>: %{customdata[1]}<br>"
+            + "<b>Lon</b>: %{customdata[2]}<br>"
+            + "<extra></extra>",
+        )
+    )
+    fig.update_layout(
+        title_text=f"{farm_name}--{f_name.split('.')[0]}",
+        title_x=0.5,
+        width=700,
+        height=500,
+        template="none",
+        yaxis_autorange="reversed",
+    )
+    # fig.show()
+    pio.write_html(fig, file=f"{file.replace('.tif','')}plot.html", auto_open=True)
+
+
+if __name__ == "__main__":
+    # Load Configs
+    parser = argparse.ArgumentParser(
+        description="Perform mass conservation analysis on the soilwatermodel logs. This should be called after running the model module with `-m True` argument.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        "-f",
+        "--file",
+        help="file name",
+    )
+    parser.add_argument(
+        "-d",
+        "--dir",
+        help="directory name",
+    )
+    parser.add_argument(
+        "-n",
+        "--name",
+        help="farm name",
+    )
+
+    args = parser.parse_args()
+    f_name = args.file
+    d_name = args.dir
+    farm_name = args.name
+
+    main(f_name, d_name, farm_name)

secrets/README.md

@@ -0,0 +1 @@
+Store private keys and other secrets in this directory.

services/job_management.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+# from google.cloud import datastore
+
+# from google.api_core import core
+from google.cloud import shell_v1  # datastore
+from google.cloud.shell_v1.services.cloud_shell_service import (  # CloudShellServiceAsyncClient,; transports,
+    CloudShellServiceClient,
+)
+from google.oauth2 import service_account
+
+service_account_secret = "secrets/nowcasting-390306-e80acb3285b4.json"
+credentials = service_account.Credentials.from_service_account_file(
+    service_account_secret
+)
+print(credentials)
+# client = datastore.Client()
+client = CloudShellServiceClient(credentials=credentials)
+
+
+async def sample_authorize_environment(credentials=credentials):
+    # Create a client
+    client = shell_v1.CloudShellServiceAsyncClient(credentials=credentials)
+
+    # Initialize request argument(s)
+    request = shell_v1.AuthorizeEnvironmentRequest()
+
+    # Make the request
+    operation = client.authorize_environment(request=request)
+
+    print("Waiting for operation to complete...")
+
+    response = (await operation).result()
+
+    # Handle the response
+    print(response)
+
+
+def sample_authorize_environment_sync(credentials=credentials):
+    # Create a client
+    client = shell_v1.CloudShellServiceClient(credentials=credentials)
+
+    # Initialize request argument(s)
+    request = shell_v1.AuthorizeEnvironmentRequest()
+    # Make the request
+    operation = client.authorize_environment(request=request)
+    response = operation.result()
+    # Handle the response
+    print(response)
+
+    print(client.common_billing_account_path)
+
+
+sample_authorize_environment(credentials)
+sample_authorize_environment_sync(credentials)
+# operation = my_api_client.long_running_method()
+# result = operation.result()
+
+
+client = shell_v1.CloudShellServiceClient()
+service_account_secret = "secrets/nowcasting-390306-e80acb3285b4.json"
+client = client.from_service_account_file(service_account_secret)
+client.get_environment()
+request = shell_v1.AuthorizeEnvironmentRequest()
+operation = client.authorize_environment(request=request)
+response = operation.result()
+print(response)

setup.cfg

@@ -0,0 +1,20 @@
+[metadata]
+name = nowcasting_app_sahandv
+version = 0.0.3
+author = Sahand Vahidnia
+author_email = [email protected]
+description = Nowcasting software to detect soil water capacity
+long_description = file: README.md
+long_description_content_type = text/markdown
+url = https://github.sydney.edu.au/informatics/PIPE-3788-GRDC-SoilWaterNow-Deployment-App
+project_urls =
+    Bug Tracker = https://github.sydney.edu.au/informatics/PIPE-3788-GRDC-SoilWaterNow-Deployment-App/issues
+classifiers =
+    Programming Language :: Python :: 3
+    Operating System :: OS Independent
+
+[flake8]
+ignore = E501, W503, E203, E265
+
+[isort]
+profile = black

templates/admin.html

@@ -0,0 +1,28 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    {% block meta %} {% endblock %}
+    <title>Nowcasting Admin</title>
+    <style>
+        nav a {
+            color: #343434; 
+            font-size: 1em;
+            margin-left: 50px;
+            text-decoration: none;
+        }
+        nav a:hover{
+            color: #7a7a7a;
+        }
+    </style>
+</head>
+<body>
+    <nav>
+        <a href="{{ url_for('home') }}">Home</a>
+    </nav>
+    <hr>
+    <div class="content">
+        {% block content %} {% endblock %}
+    </div>
+</body>
+</html>

templates/base.html

@@ -0,0 +1,31 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    {% block meta %} {% endblock %}
+    <title>{% block title %} {% endblock %}</title>
+    <style>
+        nav a {
+            color: #343434; 
+            font-size: 1em;
+            margin-left: 50px;
+            text-decoration: none;
+        }
+        nav a:hover{
+            color: #7a7a7a;
+        }
+    </style>
+</head>
+<body>
+    <nav>
+        <a href="{{ url_for('home') }}">Home</a>
+        <a href="{{ url_for('/dashapp/') }}">Dashapp Analyser</a>
+        <a href="{{ url_for('dashboard') }}">Dashboard</a>
+        <a href="{{ url_for('addjob') }}">Add Job</a>
+    </nav>
+    <hr>
+    <div class="content">
+        {% block content %} {% endblock %}
+    </div>
+</body>
+</html>

templates/dash_app.html

@@ -0,0 +1,29 @@
+<!DOCTYPE html>
+<html>
+    <head>
+        {%metas%}
+        <title>{%title%}</title>
+        {%favicon%}
+        {%css%}
+    </head>
+    <body>
+        <div class="col-12">
+            <br>
+            <h2>Soil Moisture Comparison Tool</h2>
+            <br>
+            <hr>
+        </div>
+        {%app_entry%}
+        <footer>
+            {%config%}
+            {%scripts%}
+            {%renderer%}
+        </footer>
+        <div class="col-12">
+            <hr>
+            <br>
+            Copyright @ 2023 Sydney Informatics Hub (SIH)
+            <br>
+        </div>
+    </body>
+</html>

templates/index.html

@@ -0,0 +1,7 @@
+{% extends 'base.html' %}
+
+{% block content %}
+    <h1>{% block title %} Home page of the Nowcasting app {% endblock %}</h1>
+    <h1>Hello World!</h1>
+    <h2>Welcome to the FlaskApp!</h2>
+{% endblock %}

templates/job_add.html

@@ -0,0 +1,34 @@
+<html>
+<head>
+<title>Flask RESTful API Form</title>
+</head>
+<body>
+
+    <form action="/jobadd" method="post">
+        <input type="text" name="name">
+        <input type="text" name="age">
+        <input type="submit" value="Submit">
+    </form>
+    
+    <script>
+    function generateJSON() {
+        var formData = {};
+        var formElements = document.querySelectorAll("input[name]");
+        for (var i = 0; i < formElements.length; i++) {
+            var formElement = formElements[i];
+            var name = formElement.name;
+            var value = formElement.value;
+            formData[name] = value;
+        }
+        return JSON.stringify(formData);
+    }
+    
+    document.querySelector("form").addEventListener("submit", function(event) {
+        event.preventDefault();
+        var jsonData = generateJSON();
+        alert(jsonData);
+    });
+    </script>
+
+</body>
+</html>