Update app.py

app.py

@@ -100,18 +100,23 @@ if github_token:
     Data_FR=load_GitHub(github_token, 'FR_Entsoe_UTC.csv', hour, after_10_min)
     Data_NL=load_GitHub(github_token, 'NL_Entsoe_UTC.csv', hour, after_10_min)
     Data_DE=load_GitHub(github_token, 'DE_Entsoe_UTC.csv', hour, after_10_min)
+    Data_PT=load_GitHub(github_token, 'PT_Entsoe_UTC.csv', hour, after_10_min)
+    Data_ES=load_GitHub(github_token, 'ES_Entsoe_UTC.csv', hour, after_10_min)
+    Data_AT=load_GitHub(github_token, 'AT_Entsoe_UTC.csv', hour, after_10_min)
     
     Data_BE=convert_European_time(Data_BE, 'Europe/Brussels')
     Data_FR=convert_European_time(Data_FR, 'Europe/Paris')
     Data_NL=convert_European_time(Data_NL, 'Europe/Amsterdam')
     Data_DE=convert_European_time(Data_DE, 'Europe/Berlin')
+    Data_PT=convert_European_time(Data_PT, 'Europe/Lisbon')
+    Data_ES=convert_European_time(Data_ES, 'Europe/Madrid')
+    Data_AT=convert_European_time(Data_AT, 'Europe/Vienna')
 
 
 else:
     print("Please enter your GitHub Personal Access Token to proceed.")
 
 
-# Main layout of the app
 col1, col2 = st.columns([5, 2])  # Adjust the ratio to better fit your layout needs
 with col1:
     st.title("Transparency++")
@@ -125,19 +130,23 @@ with col2:
     with col2_2:
         st.image("energyville_logo.png", width=100) 
 
+
+st.write("**Evaluate and analyze ENTSO-E Transparency Platform data quality, forecast accuracy, and energy trends for Portugal, Spain, Belgium, France, Germany, Austria, and the Netherlands.**")
+
 upper_space.markdown("""
    
    
 """, unsafe_allow_html=True)
 
-
-
 countries = {
     'Overall': 'Overall',
-    'Netherlands': 'NL',
-    'Germany': 'DE',
-    'France': 'FR',
+    'Austria': 'AT',
     'Belgium': 'BE',
+    'France': 'FR',
+    'Germany': 'DE',
+    'Netherlands': 'NL',
+    'Portugal': 'PT',
+    'Spain': 'ES',
 }
 
 
@@ -159,12 +168,20 @@ if selected_country != 'Overall':
 else:
     section = None  # No section is shown when "Overall" is selected
 
+
+forecast_columns_with_wind_offshore = [
+    'Load_entsoe','Load_forecast_entsoe','Wind_onshore_entsoe','Wind_onshore_forecast_entsoe','Wind_offshore_entsoe','Wind_offshore_forecast_entsoe','Solar_entsoe','Solar_forecast_entsoe']
+
+forecast_columns_no_wind_offshore = [
+    'Load_entsoe','Load_forecast_entsoe','Wind_onshore_entsoe','Wind_onshore_forecast_entsoe','Solar_entsoe','Solar_forecast_entsoe']
+
 if selected_country == 'Overall':
     data = None  # You can set data to None or a specific dataset based on your logic
     section = None  # No section selected when "Overall" is chosen
 else:
     country_code = countries[selected_country]
     if country_code == 'BE':
+        forecast_columns=forecast_columns_with_wind_offshore
         data = Data_BE
         weather_columns = ['Temperature', 'Wind Speed Onshore', 'Wind Speed Offshore']
         data['Temperature'] = data['temperature_2m_8']
@@ -172,23 +189,45 @@ else:
         data['Wind Speed Onshore'] = data['wind_speed_100m_8']
 
     elif country_code == 'DE':
+        forecast_columns=forecast_columns_with_wind_offshore
         data = Data_DE
         weather_columns = ['Temperature', 'Wind Speed']
         data['Temperature'] = data['temperature_2m']
         data['Wind Speed'] = data['wind_speed_100m']
 
     elif country_code == 'NL':
+        forecast_columns=forecast_columns_with_wind_offshore
         data = Data_NL
         weather_columns = ['Temperature', 'Wind Speed']
         data['Temperature'] = data['temperature_2m']
         data['Wind Speed'] = data['wind_speed_100m']
 
     elif country_code == 'FR':
+        forecast_columns=forecast_columns_with_wind_offshore
         data = Data_FR
         weather_columns = ['Temperature', 'Wind Speed']
         data['Temperature'] = data['temperature_2m']
         data['Wind Speed'] = data['wind_speed_100m']
 
+    elif country_code == 'PT':
+        forecast_columns=forecast_columns_with_wind_offshore
+        data = Data_PT
+        weather_columns = ['Temperature', 'Wind Speed']
+        data['Temperature'] = data['temperature_2m']
+        data['Wind Speed'] = data['wind_speed_100m']
+    elif country_code == 'AT':
+        forecast_columns=forecast_columns_no_wind_offshore
+        data = Data_AT
+        weather_columns = ['Temperature', 'Wind Speed']
+        data['Temperature'] = data['temperature_2m']
+        data['Wind Speed'] = data['wind_speed_100m']
+    elif country_code == 'ES':
+        forecast_columns=forecast_columns_no_wind_offshore
+        data = Data_ES
+        weather_columns = ['Temperature', 'Wind Speed']
+        data['Temperature'] = data['temperature_2m']
+        data['Wind Speed'] = data['wind_speed_100m']
+
 def add_feature(df2, df_main):
     #df_main.index = pd.to_datetime(df_main.index)
     #df2.index = pd.to_datetime(df2.index)
@@ -202,16 +241,13 @@ def add_feature(df2, df_main):
 #data.index = data.index.tz_localize('UTC')
 
 
-forecast_columns = [
-    'Load_entsoe','Load_forecast_entsoe','Wind_onshore_entsoe','Wind_onshore_forecast_entsoe','Wind_offshore_entsoe','Wind_offshore_forecast_entsoe','Solar_entsoe','Solar_forecast_entsoe']
-
 if section == 'Data Quality':
    
     st.header('Data Quality')
     
     st.write('The table below presents the data quality metrics focusing on the percentage of missing values and the occurrence of extreme or nonsensical values for the selected country.')
 
-    yesterday_midnight = pd.Timestamp(datetime.now().date() - pd.Timedelta(days=1)).replace(hour=23, minute=59, second=59)
+    yesterday_midnight = pd.Timestamp(datetime.datetime.now().date() - pd.Timedelta(days=1)).replace(hour=23, minute=59, second=59)
 
     # Filter data until the end of yesterday (midnight)
     data_quality = data[data.index <= yesterday_midnight]
@@ -224,7 +260,10 @@ if section == 'Data Quality':
         'FR': { 'Solar': 17419, 'Wind Offshore': 1483, 'Wind Onshore': 22134},
         'DE': { 'Solar': 73821, 'Wind Offshore': 8386, 'Wind Onshore': 59915},
         'BE': { 'Solar': 8789, 'Wind Offshore': 2262, 'Wind Onshore': 3053},  
-        'NL': { 'Solar': 22590, 'Wind Offshore': 3220, 'Wind Onshore': 6190},  
+        'NL': { 'Solar': 22590, 'Wind Offshore': 3220, 'Wind Onshore': 6190},
+        'PT': { 'Solar': 1811, 'Wind Offshore': 25, 'Wind Onshore': 5333},
+        'ES': { 'Solar': 23867, 'Wind Onshore': 30159},
+        'AT': { 'Solar': 7294, 'Wind Onshore': 4021 }  
     }
 
     if country_code not in installed_capacities:
@@ -287,13 +326,21 @@ elif section == 'Forecasts Quality':
     st.write('The below plot shows the time series of forecasts vs. observations provided by the ENTSO-E Transparency platform from the past week.')
     
     # Options for selecting the data to display
-    variable_options = {
-        "Load": ("Load_entsoe", "Load_forecast_entsoe"),
-        "Solar": ("Solar_entsoe", "Solar_forecast_entsoe"),
-        "Wind Onshore": ("Wind_onshore_entsoe", "Wind_onshore_forecast_entsoe"),
-        "Wind Offshore": ("Wind_offshore_entsoe", "Wind_offshore_forecast_entsoe")
-    }
-    
+    if country_code!='ES' and country_code!='AT':
+
+        variable_options = {
+            "Load": ("Load_entsoe", "Load_forecast_entsoe"),
+            "Solar": ("Solar_entsoe", "Solar_forecast_entsoe"),
+            "Wind Onshore": ("Wind_onshore_entsoe", "Wind_onshore_forecast_entsoe"),
+            "Wind Offshore": ("Wind_offshore_entsoe", "Wind_offshore_forecast_entsoe")
+        }
+    else:
+        variable_options = {
+            "Load": ("Load_entsoe", "Load_forecast_entsoe"),
+            "Solar": ("Solar_entsoe", "Solar_forecast_entsoe"),
+            "Wind Onshore": ("Wind_onshore_entsoe", "Wind_onshore_forecast_entsoe"),
+        }
+        
     # Dropdown to select the variable
     selected_variable = st.selectbox("Select Variable for Line PLot", list(variable_options.keys()))
 
@@ -310,7 +357,7 @@ elif section == 'Forecasts Quality':
 
     # Scatter plots for error distribution
     st.subheader('Error Distribution')
-    st.write('The below scatter plots show the error distribution of all four fields: Solar, Wind Onshore, Wind Offshore and Load.')
+    st.write('The below scatter plots show the error distribution of all fields: Solar, Wind and Load.')
     selected_variable = st.selectbox("Select Variable for Error Distribution", list(variable_options.keys()))
 
     # Get the corresponding columns for the selected variable
@@ -329,7 +376,7 @@ elif section == 'Forecasts Quality':
         
         st.plotly_chart(fig)
 
-    
+
         
     st.subheader('Accuracy Metrics (Sorted by rMAE):')
 
@@ -351,7 +398,11 @@ elif section == 'Forecasts Quality':
     st.write(output_text)
     
     data = data.loc[start_date:end_date]
-    accuracy_metrics = pd.DataFrame(columns=['MAE', 'rMAE'], index=['Load', 'Solar', 'Wind Onshore', 'Wind Offshore'])
+
+    if country_code!='ES' and country_code!='AT':
+        accuracy_metrics = pd.DataFrame(columns=['MAE', 'rMAE'], index=['Load', 'Solar', 'Wind Onshore', 'Wind Offshore'])
+    else:
+        accuracy_metrics = pd.DataFrame(columns=['MAE', 'rMAE'], index=['Load', 'Solar', 'Wind Onshore'])
 
     for i in range(0, len(forecast_columns), 2):
         actual_col = forecast_columns[i]
@@ -433,7 +484,7 @@ elif section == 'Forecasts Quality':
         st.plotly_chart(fig, use_container_width=True, config={'displayModeBar': False}, className="small-chart")
 
     st.subheader('ACF plots of Errors')
-    st.write('The below plots show the ACF (Auto-Correlation Function) for the errors of all three data fields obtained from ENTSO-E: Solar, Wind Onshore, Wind Offshore and Load.')
+    st.write('The below plots show the ACF (Auto-Correlation Function) for the errors of all three data fields obtained from ENTSO-E: Solar, Wind and Load.')
 
     # Dropdown to select the variable
     selected_variable = st.selectbox("Select Variable for ACF of Errors", list(variable_options.keys()))
@@ -459,7 +510,7 @@ elif section == 'Forecasts Quality':
 elif section == 'Insights':
     st.header("Insights")
 
-    st.write('The scatter plots below are created to explore possible correlations between the data fields: Solar, Wind Onshore, Wind Offshore, Load, and Weather Features.')
+    st.write('The scatter plots below are created to explore possible correlations between the data fields: Solar, Wind Onshore, Wind Offshore (if any), Load, and Weather Features.')
     # Add a selection box for the data resolution (weekly, daily, hourly)
     data_2024 = data[data.index.year == 2024]
 
@@ -472,7 +523,11 @@ elif section == 'Insights':
         resampled_data = data_2024.resample('D').mean()  # Resample to daily mean
 
     # Select the necessary columns for the scatter plot
-    selected_columns = ['Load_entsoe', 'Solar_entsoe', 'Wind_offshore_entsoe', 'Wind_onshore_entsoe'] + weather_columns
+    if country_code!='ES' and country_code!='AT':
+        selected_columns = ['Load_entsoe', 'Solar_entsoe', 'Wind_offshore_entsoe', 'Wind_onshore_entsoe'] + weather_columns
+    else:
+        selected_columns = ['Load_entsoe', 'Solar_entsoe', 'Wind_onshore_entsoe'] + weather_columns
+
     selected_df = resampled_data[selected_columns]
     selected_df.columns = [col.replace('_entsoe', '').replace('_', ' ') for col in selected_df.columns]
 
@@ -491,22 +546,42 @@ elif selected_country == 'Overall':
     st.subheader("Net Load Error Map")
     st.write("""
         The net load error map highlights the error in the forecasted versus actual net load for each country. 
-        Hover over each country to see details on the latest net load error and the timestamp of the last recorded data.
+        Hover over each country to see details on the latest net load error and the timestamp (with the time zone of the corresponding country) of the last recorded data.
     """)
     
+    def get_forecast_columns(country_code):
+        if country_code in ['Belgium', 'Germany', 'Netherlands', 'France', 'Portugal']:
+            return ['Load_entsoe', 'Wind_onshore_entsoe', 'Solar_entsoe', 'Load_forecast_entsoe', 'Wind_onshore_forecast_entsoe', 'Solar_forecast_entsoe', 'Wind_offshore_entsoe', 'Wind_offshore_forecast_entsoe']
+        else:
+            return ['Load_entsoe', 'Wind_onshore_entsoe', 'Solar_entsoe', 'Load_forecast_entsoe', 'Wind_onshore_forecast_entsoe', 'Solar_forecast_entsoe']
+        
     def plot_net_load_error_map(data_dict):
         # Define forecast columns used in calculation
 
-        def calculate_net_load_error(df):
+        def calculate_net_load_error(df, country_code):
+            forecast_columns = get_forecast_columns(country_code)
             filter_df = df[forecast_columns].dropna()
-            net_load = filter_df['Load_entsoe'] - filter_df['Wind_onshore_entsoe'] - filter_df['Wind_offshore_entsoe'] - filter_df['Solar_entsoe']
-            net_load_forecast = filter_df['Load_forecast_entsoe'] - filter_df['Wind_onshore_forecast_entsoe'] - filter_df['Wind_offshore_forecast_entsoe'] - filter_df['Solar_forecast_entsoe']
+
+            # Initialize net_load and net_load_forecast with Load and other available data
+            net_load = filter_df['Load_entsoe'] - filter_df['Wind_onshore_entsoe'] - filter_df['Solar_entsoe']
+            net_load_forecast = filter_df['Load_forecast_entsoe'] - filter_df['Wind_onshore_forecast_entsoe'] - filter_df['Solar_forecast_entsoe']
+
+            # Subtract Wind_offshore_entsoe if the column exists
+            if 'Wind_offshore_entsoe' in filter_df.columns:
+                net_load -= filter_df['Wind_offshore_entsoe']
+
+            # Subtract Wind_offshore_forecast_entsoe if the column exists
+            if 'Wind_offshore_forecast_entsoe' in filter_df.columns:
+                net_load_forecast -= filter_df['Wind_offshore_forecast_entsoe']
+
+            # Calculate the error based on the latest values
             error = (net_load_forecast - net_load).iloc[-1]
             date = filter_df.index[-1].strftime("%Y-%m-%d %H:%M")  # Get the latest date in string format
+
             return error, date
 
         # Calculate net load errors and dates for each country
-        net_load_errors = {country_name: calculate_net_load_error(data) for country_name, data in data_dict.items()}
+        net_load_errors = {country_name: calculate_net_load_error(data, country_name) for country_name, data in data_dict.items()}
 
         # Create DataFrame for Folium with additional date column
         df_net_load_error = pd.DataFrame({
@@ -535,7 +610,7 @@ elif selected_country == 'Overall':
                 feature["properties"]["date"] = row.iloc[0]["date"]
 
         # Initialize the Folium map centered on Central Europe
-        m = folium.Map(location=[51, 10], zoom_start=5, tiles="cartodb positron")
+        m = folium.Map(location=[46.6034, 1.8883], zoom_start=4.5, tiles="cartodb positron")
 
         # Add choropleth layer to map net load errors by country
         folium.Choropleth(
@@ -544,7 +619,7 @@ elif selected_country == 'Overall':
             data=df_net_load_error,
             columns=["country", "net_load_error"],
             key_on="feature.properties.name",
-            fill_color="RdYlBu",  # Use a more vibrant color palette
+            fill_color= "RdYlBu", #"RdYlBu",  # Use a more vibrant color palette
             fill_opacity=0.7,
             line_opacity=0.5,
             line_color="black",  # Neutral border color
@@ -565,18 +640,22 @@ elif selected_country == 'Overall':
         # Display Folium map in Streamlit
         st_folium(m, width=700, height=600)
 
-    # Data dictionary with full country names
     data_dict = {
         'Belgium': Data_BE,
         'France': Data_FR,
         'Germany': Data_DE,
-        'Netherlands': Data_NL
+        'Netherlands': Data_NL,
+        'Portugal': Data_PT,
+        'Austria': Data_AT,
+        'Spain': Data_ES,
     }
+
     plot_net_load_error_map(data_dict)
 
     st.subheader("rMAE of Forecasts published on ENTSO-E TP")
-    st.write("""The rMAE of Forecasts chart compares the forecast accuracy of the predictions published by ENTSO-E Transparency Platform for Belgium, Germany, France, and the Netherlands. It shows the rMAE for onshore wind, offshore wind, solar, and load demand, highlighting how well forecasts perform relative to a basic persistence model across these countries and energy sectors.""")
+    st.write("""The rMAE of Forecasts chart compares the forecast accuracy of the predictions published by ENTSO-E Transparency Platform for Portugal, Spain, Belgium, France, Germany, Austria, and the Netherlands. It shows the rMAE for onshore wind, offshore wind (if any), solar, and load demand, highlighting how well forecasts perform relative to a basic persistence model across these countries and energy sectors.""")
 
+    # Function to calculate MAE
     def calculate_mae(actual, forecast):
         return np.mean(np.abs(actual - forecast))
 
@@ -589,32 +668,69 @@ elif selected_country == 'Overall':
         rmae = {}
         rmae['Load'] = calculate_mae(df['Load_entsoe'], df['Load_forecast_entsoe']) / calculate_persistence_mae(df['Load_entsoe'], 168)
         rmae['Wind_onshore'] = calculate_mae(df['Wind_onshore_entsoe'], df['Wind_onshore_forecast_entsoe']) / calculate_persistence_mae(df['Wind_onshore_entsoe'], 24)
-        rmae['Wind_offshore'] = calculate_mae(df['Wind_offshore_entsoe'], df['Wind_offshore_forecast_entsoe']) / calculate_persistence_mae(df['Wind_offshore_entsoe'], 24)
+        
+        # Only calculate Wind_offshore rMAE if the columns exist
+        if 'Wind_offshore_entsoe' in df.columns and 'Wind_offshore_forecast_entsoe' in df.columns:
+            rmae['Wind_offshore'] = calculate_mae(df['Wind_offshore_entsoe'], df['Wind_offshore_forecast_entsoe']) / calculate_persistence_mae(df['Wind_offshore_entsoe'], 24)
+        else:
+            rmae['Wind_offshore'] = None  # Mark as None if not applicable
+        
         rmae['Solar'] = calculate_mae(df['Solar_entsoe'], df['Solar_forecast_entsoe']) / calculate_persistence_mae(df['Solar_entsoe'], 24)
+        
         return rmae
 
     # Function to create rMAE DataFrame
     def create_rmae_dataframe(data_dict):
+
         rmae_values = {'Country': [], 'Load': [], 'Wind_onshore': [], 'Wind_offshore': [], 'Solar': []}
+        
         for country_name, df in data_dict.items():
+            forecast_columns=get_forecast_columns(country_name)
             df_filtered = df[forecast_columns].dropna()
             rmae = calculate_rmae_for_country(df_filtered)
+            
             rmae_values['Country'].append(country_name)
-            for key in rmae:
-                rmae_values[key].append(rmae[key])
+            rmae_values['Load'].append(rmae['Load'])
+            rmae_values['Wind_onshore'].append(rmae['Wind_onshore'])
+            rmae_values['Solar'].append(rmae['Solar'])
+            
+            # Append Wind_offshore rMAE only if it's not None (i.e., the country has offshore wind data)
+            if rmae['Wind_offshore'] is not None:
+                rmae_values['Wind_offshore'].append(rmae['Wind_offshore'])
+            else:
+                rmae_values['Wind_offshore'].append(np.nan)  # Insert NaN for countries without offshore wind
+        
         return pd.DataFrame(rmae_values)
 
     # Function to plot radar chart
     def plot_rmae_radar_chart(rmae_df):
         fig = go.Figure()
-        angles = ['Load', 'Wind_onshore', 'Wind_offshore', 'Solar']
+        
+        # Dynamically adjust angles to exclude Wind_offshore if all values are NaN
+        angles = ['Load', 'Wind_onshore', 'Solar']
+        if not rmae_df['Wind_offshore'].isna().all():  # Only include Wind_offshore if it's not NaN for all countries
+            angles.append('Wind_offshore')
+        
         for _, row in rmae_df.iterrows():
-            fig.add_trace(go.Scatterpolar(r=[row[angle] for angle in angles], theta=angles, fill='toself', name=row['Country']))
-        fig.update_layout(polar=dict(radialaxis=dict(visible=True, range=[0, 1.2])), showlegend=True, title="rMAE Radar Chart by Country")
+            fig.add_trace(go.Scatterpolar(
+                r=[row[angle] for angle in angles],
+                theta=angles,
+                fill='toself',
+                name=row['Country']
+            ))
+        
+        fig.update_layout(
+            polar=dict(
+                radialaxis=dict(visible=True, range=[0, 1.2])
+            ),
+            showlegend=True,
+            title="rMAE Radar Chart by Country"
+        )
         st.plotly_chart(fig)
 
     # Main execution to create and display radar plot
     rmae_df = create_rmae_dataframe(data_dict)
     plot_rmae_radar_chart(rmae_df)
 
+