app.py

import streamlit as st
import pandas as pd 
import plotly.graph_objects as go
import plotly.express as px
import numpy as np
from abc import ABC, abstractmethod


# Set the title of the app
st.title("Homer Simpson Meta-Learning with Hierarchical Reinforcement Learning Intrinsic Reward Lecture")

# Display the image with a caption
st.image("homer.webp", caption="Homer Simpson Meta-Learning HRL Lecture", use_column_width=True)

# Display and play the audio files
st.write("Audio Playback Meta-Learning with HRL Intrinsic Reward Lecture:")
st.audio("h0.wav", format="audio/wav")
st.audio("h1.wav", format="audio/wav")
st.write("Oh, sweet Homer's doughnuts! If that second .wav file ain't playin', just download the darn thing! Mmm... downloads...")
st.audio("h2.wav", format="audio/wav")
st.image("intrinsic_reward_formulation.png", caption='Intrinsic Reward Formulation')
st.write("Solving the first 5 equations @ (0,0):")

df_0_0 = pd.read_csv('df_0_0.csv')
st.write(df_0_0.shape)

st.write("Example 1 via Method 1:")
# Define parameters
eta = 0.1
N_st = 1
epsilon = 1e-5
# Intrinsic reward formulation
r_t_int = eta * (1 / (N_st + epsilon)**0.5)
# Display the formulation with parameters plugged in
st.latex(r"""
r_{t}^{int} = \eta \frac{1}{\sqrt{N(s_{t}) + \epsilon}} = 0.1 \frac{1}{\sqrt{1 + 1(10^{-5})}}
""")
st.write(f"Calculated intrinsic reward: {r_t_int}")
st.write(f"Calculated intrinsic reward rounded 2 decimal places:", np.round(r_t_int,2))
st.dataframe(df_0_0[:1])


# Display the formulation with parameters plugged in
st.write("Example 2 via Method 2:")
st.latex(r"""
r_{t}^{int} = \eta \frac{1}{\sqrt{N(s_{t}) + \epsilon}} = 0.1 \frac{1}{\sqrt{2 + 1(10^{-5})}}
""")

# Abstract Base Class for Intrinsic Reward Calculation
class IntrinsicRewardCalculator(ABC):  
    @abstractmethod  
    def calculate_intrinsic_reward(self, eta, count, epsilon):    
        pass
        
# Concrete Class for Intrinsic Reward Calculation
class ConcreteIntrinsicRewardCalculator(IntrinsicRewardCalculator):  
    def calculate_intrinsic_reward(self, eta, count, epsilon):    
        return eta * (1 / np.sqrt(count + epsilon))  
    
    def populate_df_0_0(self, df_0_0, eta, count, epsilon):        
        intrinsic_reward = self.calculate_intrinsic_reward(eta, count, epsilon)       
        df_0_0.at[0, 'Intrinsic Reward'] = intrinsic_reward        
        return df_0_0

# Example 2 parameters
eta = 0.1
count = 2
epsilon = 1e-5
x,y = 0,0

# Create instance for Intrinsic Reward Calculation
irc = ConcreteIntrinsicRewardCalculator()
intrinsic_reward = irc.calculate_intrinsic_reward(0.1, 2, 1e-5)
st.write(f"Intrinsic Reward @ {count} @ Coordinates {x,y}:", intrinsic_reward)
st.write(f"Intrinsic Reward @ {count} @ Coordinates {x,y} rounded 4 decimal places:", np.round(intrinsic_reward,4))
# Populate the DataFrame with the calculated intrinsic reward
df_0_0 = irc.populate_df_0_0(df_0_0, eta, count, epsilon)
# Display the updated DataFrame
st.dataframe(df_0_0[1:2])

st.write("Example 3 via Method 1:")
# Example 3 parameters
eta = 0.1
N_st = 3
epsilon = 1e-5
# Intrinsic reward formulation
r_t_int = eta * (1 / (N_st + epsilon)**0.5)
# Display the formulation with parameters plugged in
st.latex(r"""
r_{t}^{int} = \eta \frac{1}{\sqrt{N(s_{t}) + \epsilon}} = 0.1 \frac{1}{\sqrt{3 + 1(10^{-5})}}
""")
st.write(f"Calculated intrinsic reward: {r_t_int}")
st.write(f"Calculated intrinsic reward rounded 4 decimal places:", np.round(r_t_int,4))
st.dataframe(df_0_0[2:3])



# Display the formulation with parameters plugged in
st.write("Example 4 via Method 2:")
st.latex(r"""
r_{t}^{int} = \eta \frac{1}{\sqrt{N(s_{t}) + \epsilon}} = 0.1 \frac{1}{\sqrt{4 + 1(10^{-5})}}
""")

# Abstract Base Class for Intrinsic Reward Calculation
class IntrinsicRewardCalculator(ABC):  
    @abstractmethod  
    def calculate_intrinsic_reward(self, eta, count, epsilon):    
        pass
        
# Concrete Class for Intrinsic Reward Calculation
class ConcreteIntrinsicRewardCalculator(IntrinsicRewardCalculator):  
    def calculate_intrinsic_reward(self, eta, count, epsilon):    
        return eta * (1 / np.sqrt(count + epsilon))  
    
    def populate_df_0_0(self, df_0_0, eta, count, epsilon):        
        intrinsic_reward = self.calculate_intrinsic_reward(eta, count, epsilon)       
        df_0_0.at[0, 'Intrinsic Reward'] = intrinsic_reward        
        return df_0_0

# Example 4 parameters
eta = 0.1
count = 4
epsilon = 1e-5
x,y = 0,0

# Create instance for Intrinsic Reward Calculation
irc = ConcreteIntrinsicRewardCalculator()
intrinsic_reward = irc.calculate_intrinsic_reward(0.1, 4, 1e-5)
st.write(f"Intrinsic Reward @ {count} @ Coordinates {x,y}:", intrinsic_reward)
st.write(f"Intrinsic Reward @ {count} @ Coordinates {x,y} rounded 2 decimal places:", np.round(intrinsic_reward,2))
# Populate the DataFrame with the calculated intrinsic reward
df_0_0 = irc.populate_df_0_0(df_0_0, eta, count, epsilon)
# Display the updated DataFrame
st.dataframe(df_0_0[3:4])

st.write("Example 5 via Method 1:")
# Example 5 parameters
eta = 0.1
N_st = 5
epsilon = 1e-5
# Intrinsic reward formulation
r_t_int = eta * (1 / (N_st + epsilon)**0.5)
# Display the formulation with parameters plugged in
st.latex(r"""
r_{t}^{int} = \eta \frac{1}{\sqrt{N(s_{t}) + \epsilon}} = 0.1 \frac{1}{\sqrt{5 + 1(10^{-5})}}
""")
st.write(f"Calculated intrinsic reward: {r_t_int}")
st.write(f"Calculated intrinsic reward rounded 4 decimal places:", np.round(r_t_int,4))
st.dataframe(df_0_0[4:5])

st.audio("h3.wav", format="audio/wav")
st.write("Oh, sweet Krusty-licious! At coordinates (0,0) for that plotly visualization, we need a whopping 7036 intrinsic reward calculations to get things rollin'! And don't forget to update those State Visitations. Those were just the first five. Mmm... 7031 more to go... remember the oscillation starts at zero not 1 D'oh!")

# Display the formulation with parameters plugged in
st.write("Last Example 7035 via Method 2:")
st.latex(r"""
r_{t}^{int} = \eta \frac{1}{\sqrt{N(s_{t}) + \epsilon}} = 0.1 \frac{1}{\sqrt{7035 + 1(10^{-5})}}
""")


# Abstract Base Class for Intrinsic Reward Calculation
class IntrinsicRewardCalculator(ABC):  
    @abstractmethod  
    def calculate_intrinsic_reward(self, eta, count, epsilon):    
        pass
        
# Concrete Class for Intrinsic Reward Calculation
class ConcreteIntrinsicRewardCalculator(IntrinsicRewardCalculator):  
    def calculate_intrinsic_reward(self, eta, count, epsilon):    
        return eta * (1 / np.sqrt(count + epsilon))  
    
    def populate_df_0_0(self, df_0_0, eta, count, epsilon):        
        intrinsic_reward = self.calculate_intrinsic_reward(eta, count, epsilon)       
        df_0_0.at[0, 'Intrinsic Reward'] = intrinsic_reward        
        return df_0_0

# Example 4 parameters
eta = 0.1
count = 7035
epsilon = 1e-5
x,y = 0,0

# Create instance for Intrinsic Reward Calculation
irc = ConcreteIntrinsicRewardCalculator()
intrinsic_reward = irc.calculate_intrinsic_reward(0.1, 7035, 1e-5)
st.write(f"Intrinsic Reward @ {count} @ Coordinates {x,y}:", intrinsic_reward)
st.write(f"Intrinsic Reward @ {count} @ Coordinates {x,y} rounded 4 decimal places:", np.round(intrinsic_reward,4))
# Populate the DataFrame with the calculated intrinsic reward
df_0_0 = irc.populate_df_0_0(df_0_0, eta, count, epsilon)
# Display the updated DataFrame
st.dataframe(df_0_0[7034:7035])

st.image("homer_formula_7035.webp", caption='Homer Meta-Learning HRL Formula 7035')
st.image("homer_formula_7035_2.webp", caption='Homer Meta-Learning HRL Formula 7035 2')

# Define the grid and visitations
grid = np.zeros((6, 6))
visitations = {
    (0, 0): 7035, (1, 0): 3579, (2, 0): 1359, (2, 1): 1707, (3, 1): 520, (4, 1): 227,
    (4, 2): 243, (5, 1): 217, (5, 2): 181, (5, 0): 241, (4, 0): 267, (5, 3): 179,
    (4, 3): 1034, (3, 3): 2163, (2, 3): 2080, (0, 1): 3313, (1, 1): 3015, (0, 2): 1846,
    (0, 3): 1104, (0, 4): 351, (1, 4): 518, (1, 3): 1497, (1, 2): 2236, (2, 2): 2239,
    (2, 4): 842, (1, 5): 238, (2, 5): 217, (0, 5): 341, (3, 5): 382, (4, 5): 1872,
    (4, 4): 2038, (3, 4): 1684, (3, 0): 383, (3, 2): 1102, (5, 4): 198
}

# Fill the grid with visitations
for (x, y), count in visitations.items():
    grid[x, y] = count

# Calculate the total number of visitations
total_visitations = sum(visitations.values())

# Calculate the percentages
percentages = {state: (count / total_visitations) * 100 for state, count in visitations.items()}

# Print the percentages in the specified order
order = [
    (0, 0), (0, 1), (0, 2), (0, 3), (0, 4), (0, 5),
    (1, 0), (1, 1), (1, 2), (1, 3), (1, 4), (1, 5),
    (2, 0), (2, 1), (2, 2), (2, 3), (2, 4), (2, 5),
    (3, 0), (3, 1), (3, 2), (3, 3), (3, 4), (3, 5),
    (4, 0), (4, 1), (4, 2), (4, 3), (4, 4), (4, 5),
    (5, 0), (5, 1), (5, 2), (5, 3), (5, 4), (5, 5)
]

st.title("State Visitations Visualization")

st.write("### State Visitations Percentages:")
for state in order:
    st.write(f"State {state}: {percentages.get(state, 0):.2f}%")

# Create a pie chart
labels = [f"State {state}" for state in visitations.keys()]
values = list(visitations.values())

fig_pie = go.Figure(data=[go.Pie(labels=labels, values=values)])
fig_pie.update_layout(title_text="State Visitations Pie Chart")
st.plotly_chart(fig_pie)

# Create a heatmap
fig_heatmap = px.imshow(grid, labels=dict(x="Column", y="Row", color="Visitations"),
x=list(range(6)), y=list(range(6)), title="State Visitations Heatmap")
fig_heatmap.update_xaxes(side="top")
st.plotly_chart(fig_heatmap)

# Load the CSV data
df = pd.read_csv('goal_rows.csv')

# Aggregate the data to count the number of visits to each State_2D
visitation_counts = df['State_2D'].value_counts().reset_index()
visitation_counts.columns = ['State_2D', 'Visitation_Count']

# Create the Plotly bar chart
fig = px.bar(visitation_counts, x='State_2D', y='Visitation_Count', 
             title='Goal Position Visitation Counts',
             labels={'State_2D': 'State 2D', 'Visitation_Count': 'Visitation Count'})

# Display the plot using Streamlit
st.title('Goal Position Visitation Counts Visualization')
st.plotly_chart(fig)

# Duplicate Goal Visitation visualization
# Visualization only includes the (4,5) and (5,4) 6 x 6 Grid Map Positions
# Aggregate the data to count the number of visits to each State_2D
#visitation_counts = df['State_2D'].value_counts().reset_index()
#visitation_counts.columns = ['State_2D', 'Visitation_Count']

# Clean the State_2D column to remove any extra characters or spaces
#visitation_counts['State_2D'] = visitation_counts['State_2D'].str.replace(r'[^\d,]', '', regex=True)

# Split the cleaned State_2D into separate columns
#visitation_counts[['x', 'y']] = visitation_counts['State_2D'].str.split(',', expand=True).astype(int)

# Create the Plotly heatmap
#fig = px.density_heatmap(visitation_counts, x='x', y='y', z='Visitation_Count', 
                         #title='Goal Position Visitation Counts Heatmap',
                         #labels={'x': 'X Coordinate', 'y': 'Y Coordinate', 'Visitation_Count': 'Visitation Count'})

# Display the heatmap using Streamlit
#st.title('Goal Position Visitation Counts Heatmap Visualization')
#st.plotly_chart(fig)
# Duplicate Goal Visitation visualization 

df = pd.read_csv('intrinsic_analysis.csv')
st.write("Intrinsic Analysis DataFrame:")
st.dataframe(df)

# Visualization leveraging intrinsic_analysis.csv includes the entire (6x6) Grid-World (Map Positions) 
# Load the CSV data

df = pd.read_csv('goal_rows.csv')
st.write('Goal Rows DataFrame:')
st.dataframe(df)

# Display the image with a caption
st.image("homer_plotly_visualizations_1.webp", caption="Homer Simpson Meta-Learning HRL Lecture Plotly 1", use_column_width=True)

# Aggregate the data to count the number of visits to each State_2D
visitation_counts = df['State_2D'].value_counts().reset_index()
visitation_counts.columns = ['State_2D', 'Visitation_Count']

# Clean the State_2D column to remove any extra characters or spaces
visitation_counts['State_2D'] = visitation_counts['State_2D'].str.replace(r'[^\d,]', '', regex=True)

# Split the cleaned State_2D into separate columns
visitation_counts[['x', 'y']] = visitation_counts['State_2D'].str.split(',', expand=True).astype(int)

# Create the Plotly heatmap
fig = px.density_heatmap(visitation_counts, x='x', y='y', z='Visitation_Count', 
                         title='Goal Position Visitation Counts Heatmap',
                         labels={'x': 'X Coordinate', 'y': 'Y Coordinate', 'Visitation_Count': 'Visitation Count'})

# Display the heatmap using Streamlit
st.title('Goal Position Visitation Counts Heatmap Visualization')
st.plotly_chart(fig, key='unique_heatmap_key')

# Populates goal tiles at the top right corner at positions (4,5) and (5,4)
# Aggregate the data to count the number of visits to each State_2D
#visitation_counts = df['State_2D'].value_counts().reset_index()
#visitation_counts.columns = ['State_2D', 'Visitation_Count']

# Clean the State_2D column to remove any extra characters or spaces
#visitation_counts['State_2D'] = visitation_counts['State_2D'].str.replace(r'[^\d,]', '', regex=True)

# Split the cleaned State_2D into separate columns
#visitation_counts[['x', 'y']] = visitation_counts['State_2D'].str.split(',', expand=True).astype(int)

# Create a 6x6 grid with zero visitation counts for all positions
#grid_size = 6
#heatmap_data = pd.DataFrame({
    #'x': [x for x in range(grid_size) for y in range(grid_size)],
    #'y': [y for x in range(grid_size) for y in range(grid_size)],
    #'Visitation_Count': 0
#})

# Merge the visitation counts with the grid data
#heatmap_data = heatmap_data.merge(visitation_counts, on=['x', 'y'], how='left').fillna(0)

# Create the Plotly heatmap
#fig = px.density_heatmap(heatmap_data, x='x', y='y', z='Visitation_Count_y', 
                         #title='Goal Position Visitation Counts Heatmap',
                         #labels={'x': 'X Coordinate', 'y': 'Y Coordinate', 'Visitation_Count_y': 'Visitation Count'},
                         #nbinsx=grid_size, nbinsy=grid_size)

# Display the heatmap using Streamlit
#st.title('Goal Position Visitation Counts Heatmap Visualization')
#st.plotly_chart(fig)
# Populates goal tiles at the top right corner at positions (4,5) and (5,4)

# Populates the visualization with the goal tiles at the bottom right. But, the positions are set at (4,0) and (5,1)
# Visualization with the origin initializing at the top-left corner
# Goal tiles should appear on the bottom right at two sets of coordinates located at (4,5) and (5,4)
# Aggregate the data to count the number of visits to each State_2D
#visitation_counts = df['State_2D'].value_counts().reset_index()
#visitation_counts.columns = ['State_2D', 'Visitation_Count']

# Clean the State_2D column to remove any extra characters or spaces
#visitation_counts['State_2D'] = visitation_counts['State_2D'].str.replace(r'[^\d,]', '', regex=True)

# Split the cleaned State_2D into separate columns
#visitation_counts[['x', 'y']] = visitation_counts['State_2D'].str.split(',', expand=True).astype(int)

# Invert the y-coordinates to match the desired orientation
#grid_size = 6
#visitation_counts['y'] = grid_size - 1 - visitation_counts['y']

# Create a 6x6 grid with zero visitation counts for all positions
#heatmap_data = pd.DataFrame({
   #'x': [x for x in range(grid_size) for y in range(grid_size)],
    #'y': [y for x in range(grid_size) for y in range(grid_size)],
    #'Visitation_Count': 0
#})

# Merge the visitation counts with the grid data
#heatmap_data = heatmap_data.merge(visitation_counts, on=['x', 'y'], how='left').fillna(0)

# Create the Plotly heatmap
#fig = px.density_heatmap(heatmap_data, x='x', y='y', z='Visitation_Count_y', 
                         #title='Goal Position Visitation Counts Heatmap',
                         #labels={'x': 'X Coordinate', 'y': 'Y Coordinate', 'Visitation_Count_y': 'Visitation Count'},
                         #nbinsx=grid_size, nbinsy=grid_size)

# Display the heatmap using Streamlit with a unique key
#st.title('Goal Position Visitation Counts Heatmap Visualization')
#st.plotly_chart(fig, key='this_one_is_unique')
# Populates the visualization with the goal tiles at the bottom right. But, the positions are set at (4,0) and (5,1)

# Populates the visualization with the y-coordinates inverted. Also, reverses the y-axis coordinates. But, now the goal tiles appear 
# at the top-left corner at coordinates (4,0) and (5,1) 
# Aggregate the data to count the number of visits to each State_2D
#visitation_counts = df['State_2D'].value_counts().reset_index()
#visitation_counts.columns = ['State_2D', 'Visitation_Count']

# Clean the State_2D column to remove any extra characters or spaces
#visitation_counts['State_2D'] = visitation_counts['State_2D'].str.replace(r'[^\d,]', '', regex=True)

# Split the cleaned State_2D into separate columns
#visitation_counts[['x', 'y']] = visitation_counts['State_2D'].str.split(',', expand=True).astype(int)

# Invert the y-coordinates to match the desired orientation
#grid_size = 6
#visitation_counts['y'] = grid_size - 1 - visitation_counts['y']

# Create a 6x6 grid with zero visitation counts for all positions
#heatmap_data = pd.DataFrame({
   #'x': [x for x in range(grid_size) for y in range(grid_size)],
   # 'y': [y for x in range(grid_size) for y in range(grid_size)],
   # 'Visitation_Count': 0
#})

# Merge the visitation counts with the grid data
#heatmap_data = heatmap_data.merge(visitation_counts, on=['x', 'y'], how='left').fillna(0)

# Create the Plotly heatmap
#fig = px.density_heatmap(heatmap_data, x='x', y='y', z='Visitation_Count_y', 
                        # title='Goal Position Visitation Counts Heatmap',
                        # labels={'x': 'X Coordinate', 'y': 'Y Coordinate', 'Visitation_Count_y': 'Visitation Count'},
                        # nbinsx=grid_size, nbinsy=grid_size)

# Reverse the y-axis to display 0 to 5 from top to bottom
#fig.update_yaxes(autorange="reversed") 

# Display the heatmap using Streamlit with a unique key
#st.title('Goal Position Visitation Counts Heatmap Visualization')
#st.plotly_chart(fig, key='unique_diamond')
# Populates the visualization with the y-coordinates inverted. Also, reverses the y-axis coordinates. But, now the goal tiles appear 
# at the top-left corner at coordinates (4,0) and (5,1) 
# Populates goal tiles at the top right corner at positions (4,5) and (5,4)


# Aggregate the data to count the number of visits to each State_2D
visitation_counts = df['State_2D'].value_counts().reset_index()
visitation_counts.columns = ['State_2D', 'Visitation_Count']

# Clean the State_2D column to remove any extra characters or spaces
visitation_counts['State_2D'] = visitation_counts['State_2D'].str.replace(r'[^\d,]', '', regex=True)

# Split the cleaned State_2D into separate columns
visitation_counts[['x', 'y']] = visitation_counts['State_2D'].str.split(',', expand=True).astype(int)

# Create a 6x6 grid with zero visitation counts for all positions
grid_size = 6
heatmap_data = pd.DataFrame({
    'x': [x for x in range(grid_size) for y in range(grid_size)],
    'y': [y for x in range(grid_size) for y in range(grid_size)],
    'Visitation_Count': 0
})

# Merge the visitation counts with the grid data
heatmap_data = heatmap_data.merge(visitation_counts, on=['x', 'y'], how='left').fillna(0)

# Create the Plotly heatmap
fig = px.density_heatmap(heatmap_data, x='x', y='y', z='Visitation_Count_y', 
                         title='Goal Position Visitation Counts Heatmap',
                         labels={'x': 'X Coordinate', 'y': 'Y Coordinate', 'Visitation_Count_y': 'Visitation Count'},
                         nbinsx=grid_size, nbinsy=grid_size)

# Reverse the y-axis to display 0 to 5 from top to bottom
fig.update_yaxes(autorange="reversed") 

# Display the heatmap using Streamlit
st.title('Goal Position Visitation Counts Heatmap Visualization')
st.plotly_chart(fig)

# Clean the State_2D column to remove any extra characters or spaces
df['State_2D'] = df['State_2D'].str.replace(r'[^\d,]', '', regex=True)

# Split the cleaned State_2D into separate columns
df[['x', 'y']] = df['State_2D'].str.split(',', expand=True).astype(int)

# 1. Compare State_2D and Extrinsic Reward
fig1 = px.bar(df, x='State_2D', y='Extrinsic Reward', 
              title='Comparison of State_2D and Extrinsic Reward',
              labels={'State_2D': 'State 2D', 'Extrinsic Reward': 'Extrinsic Reward'})

# 2. Compare Intrinsic Reward | Extrinsic Reward | State_2D
fig2 = px.scatter(df, x='Intrinsic Reward', y='Extrinsic Reward', color='State_2D',
                  title='Comparison of Intrinsic Reward, Extrinsic Reward, and State_2D',
                  labels={'Intrinsic Reward': 'Intrinsic Reward', 'Extrinsic Reward': 'Extrinsic Reward', 'State_2D': 'State 2D'})

# 3. Compare Total Reward | Intrinsic Reward | Extrinsic Reward | State_2D
fig3 = px.scatter_3d(df, x='Total Reward', y='Intrinsic Reward', z='Extrinsic Reward', color='State_2D',
                     title='Comparison of Total Reward, Intrinsic Reward, Extrinsic Reward, and State_2D',
                     labels={'Total Reward': 'Total Reward', 'Intrinsic Reward': 'Intrinsic Reward', 'Extrinsic Reward': 'Extrinsic Reward', 'State_2D': 'State 2D'})

# 4. Compare State_2D and Total Reward
fig4 = px.bar(df, x='State_2D', y='Total Reward', 
              title='Comparison of State_2D and Total Reward',
              labels={'State_2D': 'State 2D', 'Total Reward': 'Total Reward'})

# Display the visualizations using Streamlit
st.title('Intrinsic Analysis Visualizations')
st.plotly_chart(fig1)
st.plotly_chart(fig2)
st.plotly_chart(fig3)
st.plotly_chart(fig4)

# Display the image with a caption
st.image("homer_plotly_visualizations_2.webp", caption="Homer Simpson Meta-Learning HRL Lecture Plotly 2", use_column_width=True)