Updated app.py with chunking of data based on selection of date range. Optimized the data flow.

app.py

@@ -1,5 +1,6 @@
 import panel as pn
 import pandas as pd
+import os
 
 from consts import INVERTER_ID_MAPPING, TEMPERATURE_COLUMNS_TO_USE
 from plotting import (
@@ -10,26 +11,14 @@ from plotting import (
     create_iv_plot_with_temperature_curves,
 )
 
-# Read metadata to get the date range
-timeseries_data = pd.read_csv(
-    r"multi_index_timeseries_data.csv",
-    index_col=0,
-    header=[0, 2],
-    parse_dates=True,
-)
-other_features_data = pd.read_csv(
-    r"other_features_data.csv", index_col=0, header=0, parse_dates=True
-)
-other_features_data = other_features_data[TEMPERATURE_COLUMNS_TO_USE]
-kpi_data = pd.read_csv(
-    r"kpi_data.csv", index_col=0, header=0, parse_dates=True
-)
-daily_timeseries_data = pd.read_csv(
-    r"daily_aggregated_timeseries_data.csv",
-    index_col=0,
-    header=0,
-    parse_dates=True,
-)
+# Function to load data in chunks
+def load_data_in_chunks(file_path, columns, start_date=None, end_date=None, chunksize=10000):
+    chunks = []
+    for chunk in pd.read_csv(file_path, usecols=['datetime'] + columns, chunksize=chunksize, parse_dates=['datetime']):
+        if start_date and end_date:
+            chunk = chunk[(chunk['datetime'] >= start_date) & (chunk['datetime'] <= end_date)]
+        chunks.append(chunk)
+    return pd.concat(chunks)
 
 # Initialize Panel extension
 pn.extension("plotly")
@@ -59,7 +48,6 @@ heatmap_temperature = pn.widgets.Checkbox(
 # Create a loading spinner
 loading_spinner = pn.indicators.LoadingSpinner(width=50, height=50)
 
-
 # Panel interactive functions
 @pn.depends(
     inverter_ids.param.value,
@@ -70,27 +58,37 @@ def update_iv_plot(inverter_ids, plot_power_curves, plot_temperature_curves):
     if not inverter_ids:
         return pn.pane.Markdown("No Inverters selected for Plotting.")
     else:
+        columns = []
+        if plot_power_curves:
+            columns += [CURRENT, IRRADIANCE, VOLTAGE, POWER_DC, POWER_AC]
+        if plot_temperature_curves:
+            columns += [T_AMBIENT, T_MODULE, T_HEATSINK, T_CPU, T_BOARD]
+        if not plot_power_curves and not plot_temperature_curves:
+            columns += [CURRENT, IRRADIANCE, VOLTAGE]
+
+        data = load_data_in_chunks('multi_index_timeseries_data.csv', columns)
+        other_data = load_data_in_chunks('other_features_data.csv', TEMPERATURE_COLUMNS_TO_USE)
+
         # Plot IV + Power + Temperature Curves
         if plot_power_curves and plot_temperature_curves:
             print("Plotting IV + Power + Temperature Curves")
             return create_iv_plot_with_power_and_temperature_curves(
-                timeseries_data, other_features_data, inverter_ids
+                data, other_data, inverter_ids
             )
         # Plot IV + Temperature Curves
         elif (not plot_power_curves) and plot_temperature_curves:
             print("Plot IV + Temperature Curves")
             return create_iv_plot_with_temperature_curves(
-                timeseries_data, other_features_data, inverter_ids
+                data, other_data, inverter_ids
             )
         # Plot IV + Power Curves
         elif plot_power_curves and (not plot_temperature_curves):
             print("Plot IV + Power Curves")
-            return create_iv_plot_with_power_curves(timeseries_data, inverter_ids)
+            return create_iv_plot_with_power_curves(data, inverter_ids)
         # Plot only IV Curves
         else:
             print("Plot only IV Curves")
-            return create_iv_plot(timeseries_data, inverter_ids)
-
+            return create_iv_plot(data, inverter_ids)
 
 @pn.depends(heatmap_pr.param.value)
 def update_heatmap_pr(heatmap_pr):
@@ -101,7 +99,6 @@ def update_heatmap_pr(heatmap_pr):
         return pn.Row(pr_heatmap)
     return pn.pane.Markdown("")
 
-
 @pn.depends(heatmap_sy.param.value)
 def update_heatmap_sy(heatmap_sy):
     if heatmap_sy:
@@ -111,7 +108,6 @@ def update_heatmap_sy(heatmap_sy):
         return pn.Row(sy_heatmap)
     return pn.pane.Markdown("")
 
-
 @pn.depends(heatmap_current.param.value)
 def update_heatmap_current(heatmap_current):
     if heatmap_current:
@@ -121,7 +117,6 @@ def update_heatmap_current(heatmap_current):
         return pn.Row(current_heatmap)
     return pn.pane.Markdown("")
 
-
 @pn.depends(heatmap_voltage.param.value)
 def update_heatmap_voltage(heatmap_voltage):
     if heatmap_voltage:
@@ -131,7 +126,6 @@ def update_heatmap_voltage(heatmap_voltage):
         return pn.Row(voltage_heatmap)
     return pn.pane.Markdown("")
 
-
 @pn.depends(heatmap_power.param.value)
 def update_heatmap_power(heatmap_power):
     if heatmap_power:
@@ -141,7 +135,6 @@ def update_heatmap_power(heatmap_power):
         return pn.Row(power_heatmap)
     return pn.pane.Markdown("")
 
-
 @pn.depends(heatmap_irradiance.param.value)
 def update_heatmap_irradiance(heatmap_irradiance):
     if heatmap_irradiance:
@@ -151,7 +144,6 @@ def update_heatmap_irradiance(heatmap_irradiance):
         return pn.Row(irradiance_heatmap)
     return pn.pane.Markdown("")
 
-
 @pn.depends(heatmap_temperature.param.value)
 def update_heatmap_temperature(heatmap_temperature):
     if heatmap_temperature:
@@ -161,7 +153,6 @@ def update_heatmap_temperature(heatmap_temperature):
         return pn.Row(temp_heatmap)
     return pn.pane.Markdown("")
 
-
 # Create dashboard layout
 dashboard = pn.Column(
     "# ENEL Dashboard",