algo2.txt

!pip install gym

# Import necessary libraries
import torch
import torch.nn as nn
import pandas as pd
import torch.optim as optim
import numpy as np
import random
import warnings
import gym
from gym import spaces
import matplotlib.pyplot as plt

# Suppress possible warnings for cleaner output
warnings.filterwarnings("ignore")

# Set random seeds for reproducibility
def set_seed(seed):
  np.random.seed(seed)
  torch.manual_seed(seed)
  random.seed(seed)
  if torch.cuda.is_available():
    torch.cuda.manual_seed_all(seed)

set_seed(42)

# Define the device for computation (use CUDA if available)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"Using device: {device}")

# Custom grid environment with fixed size and traps
class CustomGridEnv:
  def __init__(self, size=6, num_traps=3):
    # Fixed complexity: grid 6x6 with 3 traps

    self.size = size
    self.num_traps = num_traps
    self.obversation_space = spaces.Discrete(self.size * self.size)
    self.action_space = spaces.Discrete(4) # 4 actions: Up, Down, Left, Right
    self.reset()

  def reset(self):
    self.agent_pos = [0, 0]
    self.goal_pos = [self.size - 1, self.size - 1]
    self._generate_traps()
    return self._get_obs()

  def _generate_traps(self):
    self.traps = []
    while len(self.traps) < self.num_traps:
      trap = [np.random.randint(self.size), np.random.randint(self.size)]
      if trap != self.agent_pos and trap != self.goal_pos:
        self.traps.append(trap)

  def step(self, action):
    if action == 0 and self.agent_pos[0] > 0:

      # Up
      self.agent_pos[0] -= 1

    elif action == 1 and self.agent_pos[0] < self.size - 1:

      # Down
      self.agent_pos[0] += 1

    elif action == 2 and self.agent_pos[1] > 0:

      # Left
      self.agent_pos[1] -= 1

    elif action == 3 and self.agent_pos[1] < self.size - 1:

      # Right
      self.agent_pos[1] += 1

    done = False

    if self.agent_pos == self.goal_pos:

      reward = 10
      done = True

    elif self.agent_pos in self.traps:

      reward = -5
      done = True

    else:

      reward = -1

    return self._get_obs(), reward, done, {}

  def _get_obs(self):
    return self.agent_pos[0] * self.size + self.agent_pos[1]

# Neural network for the policy
class PolicyNet(nn.Module):
  def __init__(self, input_size, num_actions):
    super(PolicyNet, self).__init__()
    self.fc1 = nn.Linear(input_size, 64) # Fully-connected layer 1 [Input]
    self.fc2 = nn.Linear(64, num_actions) # Fully-connected layer 2 [State-action pair selections based on fc1 input]

  def forward(self, state):
    x = torch.relu(self.fc1(state))
    return self.fc2(x)

# Helper function to convert a state to one-hot representation
def state_to_one_hot(state, num_states):
  one_hot = np.zeros(num_states)
  one_hot[state] = 1
  return torch.FloatTensor([one_hot]).to(device)


# Epsilon-greedy action selection
def select_action(network, state, epsilon, num_actions):
  if np.random.uniform(0, 1) < epsilon: # Coin flip selection then epsilon comparison
    return np.random.choice(num_actions)
  else: # If less than epsilon then select largest q-values out of the q-values obtained
    with torch.no_grad():
      q_values = network(state)
      return torch.argmax(q_values).item() #e.g., q_values: Q[Q1(100), Q2(277.7123), Q3(69.1234567)] selects Q2

# Version 5
# Validate the df_goal_rows that appear in the visualization
# Compare the success rate based on goal visits with the success rate calculated during the training process


# Single complexity meta-training process with success rate tracking
# The Intrinsic Reward Analysis derives from within the anatomy of the function with the name meta_train_fixed_complexity
def meta_train_fixed_complexity(meta_learning_rate, epsilon_start, epsilon_decay, num_iterations, num_inner_steps, eta=0.1, epsilon=1e-5):
    num_states = 6 * 6  # Fixed grid size of 6x6
    num_actions = 4  # Up, Down, Left, Right
    discount_factor = 0.99  # Gamma

    # Initialize policy network
    policy_net = PolicyNet(input_size=num_states, num_actions=num_actions).to(device)
    optimizer = optim.Adam(policy_net.parameters(), lr=meta_learning_rate)

    epsilon_greedy = epsilon_start
    meta_losses, meta_rewards, success_rates = [], [], []

    env = CustomGridEnv(size=6, num_traps=3)  # Fixed complexity level: grid 6x6 with 3 traps

    # Intrinsic Reward Analysis data capture encapsulated by 3 variables:
    # 1. state_visitation_counts
    # 2. intrinsic_reward
    # 3. total_reward

    # 1. state_visitation_counts
    # State visitation counts
    state_visitation_counts = np.zeros(num_states)

    # Initialize intrinsic analysis list
    intrinsic_analysis = []

    for iteration in range(num_iterations):
        print(f"Iteration {iteration + 1}/{num_iterations}")

        total_loss = 0
        total_reward = 0
        successes = 0

        for task in range(10):  # Fixed number of tasks for each iteration
            state = env.reset()
            state = state_to_one_hot(state, num_states)
            optimizer.zero_grad()

            for step in range(num_inner_steps):
                action = select_action(policy_net, state, epsilon_greedy, num_actions)
                next_state, reward_ext, done, _ = env.step(action)
                next_state = state_to_one_hot(next_state, num_states)

                # 1. state_visitation_counts
                # Update state visitation count
                state_visitation_counts[state.argmax().item()] += 1

                # 2. intrinsic_reward
                # Calculate intrinsic reward
                intrinsic_reward = eta * (1 / np.sqrt(state_visitation_counts[state.argmax().item()] + epsilon))

                # 3. total_reward
                # Calculate total reward
                total_reward = reward_ext + intrinsic_reward

                # Convert state index to 2D grid representation
                state_2d = (state.argmax().item() // 6, state.argmax().item() % 6)

                # Append intrinsic analysis data
                intrinsic_analysis.append({
                    'State_2D': state_2d,
                    'Intrinsic Reward': intrinsic_reward,
                    'Total Reward': total_reward,
                    'Extrinsic Reward': reward_ext
                })

                with torch.no_grad():
                    target = total_reward + discount_factor * torch.max(policy_net(next_state))

                prediction = policy_net(state)[0][action]
                loss = nn.functional.smooth_l1_loss(prediction, target)
                loss.backward()
                total_loss += loss.item()

                optimizer.step()
                state = next_state
                total_reward += reward_ext
                if done:
                    if reward_ext == 10:  # Success is defined as reaching the goal
                        successes += 1
                    break

        meta_losses.append(total_loss / 10)
        meta_rewards.append(total_reward / 10)
        success_rates.append(successes / 10)
        epsilon_greedy = max(0.1, epsilon_greedy * epsilon_decay)

    # Convert intrinsic analysis list to DataFrame and save to CSV
    df_intrinsic_analysis = pd.DataFrame(intrinsic_analysis)
    df_intrinsic_analysis.to_csv('intrinsic_analysis.csv', index=False)

    # Find all rows associated with the goal position (Extrinsic Reward == 10)
    df_goal_rows = df_intrinsic_analysis[df_intrinsic_analysis['Extrinsic Reward'] == 10]
    print(f"Shape of df_goal_rows: {df_goal_rows.shape}")
    print("Rows associated with the goal position:")
    print(df_goal_rows)

    # Filter for positions (4,5) and (5,4)
    df_goal_rows_4_5 = df_goal_rows[df_goal_rows['State_2D'] == (4, 5)]
    df_goal_rows_5_4 = df_goal_rows[df_goal_rows['State_2D'] == (5, 4)]
    print("Rows associated with position (4,5):")
    print(df_goal_rows_4_5)
    print("Rows associated with position (5,4):")
    print(df_goal_rows_5_4)

    # Filter for any other positions
    df_goal_rows_other = df_goal_rows[~df_goal_rows['State_2D'].isin([(4, 5), (5, 4)])]
    print("Rows associated with any other positions:")
    print(df_goal_rows_other)

    # Calculate success rate based on goal visits
    total_tasks = num_iterations * 10
    success_rate_goal_visits = len(df_goal_rows) / total_tasks * 100
    print(f"Success Rate based on Goal Visits: {success_rate_goal_visits:.2f}%")

    # Visualize state visitation counts
    plt.figure(figsize=(15, 5))
    plt.subplot(1, 3, 1)
    plt.imshow(state_visitation_counts.reshape(6, 6), cmap='hot', interpolation='nearest')
    plt.title('State Visitation Counts')
    plt.colorbar()

    # Visualize intrinsic rewards
    intrinsic_rewards = df_intrinsic_analysis['Intrinsic Reward'].values
    plt.subplot(1, 3, 2)
    plt.hist(intrinsic_rewards, bins=50, color='blue', alpha=0.7)
    plt.title('Intrinsic Reward Distribution')
    plt.xlabel('Intrinsic Reward')
    plt.ylabel('Frequency')

    # Visualize goal position visits
    goal_positions = df_goal_rows['State_2D'].apply(lambda x: x[0] * 6 + x[1]).values
    goal_counts = np.zeros(num_states)
    for pos in goal_positions:
        goal_counts[pos] += 1
    plt.subplot(1, 3, 3)
    plt.imshow(goal_counts.reshape(6, 6), cmap='hot', interpolation='nearest')
    plt.title('Goal Position Visitation Counts')
    plt.colorbar()

    plt.tight_layout()
    plt.show()

    return meta_losses, meta_rewards, success_rates

# Plot function for meta-loss, average reward, and success rate with white background and markers
def plot_meta_losses_rewards_success(meta_losses, meta_rewards, success_rates, window_size=10):
    smoothed_losses = moving_average(meta_losses, window_size)
    smoothed_rewards = moving_average(meta_rewards, window_size)
    smoothed_success_rates = moving_average(success_rates, window_size)

    # Create the figure and axes with white background
    fig, ax1 = plt.subplots(figsize=(14, 7), facecolor='white')

    # Set axes background color to white
    ax1.set_facecolor('white')

    color = 'tab:red'
    ax1.set_xlabel('Meta-Iteration')
    ax1.set_ylabel('Meta-Loss', color=color)
    ax1.plot(meta_losses, color=color, alpha=0.1, label='Meta-Loss', marker='o', markersize=5)
    ax1.plot(range(window_size - 1, len(meta_losses)), smoothed_losses, color=color, label=f'Smoothed Meta-Loss (window={window_size})', marker='o', markersize=3)
    ax1.tick_params(axis='y', labelcolor=color)

    # Twin x-axis for Average Reward
    ax2 = ax1.twinx()
    ax2.set_facecolor('white')  # Set the background color of the second axis to white
    color = 'tab:blue'
    ax2.set_ylabel('Average Reward', color=color)
    ax2.plot(meta_rewards, color=color, alpha=0.1, label='Average Reward', marker='s', markersize=5)
    ax2.plot(range(window_size - 1, len(meta_rewards)), smoothed_rewards, color=color, label=f'Smoothed Average Reward (window={window_size})', marker='s', markersize=3)
    ax2.tick_params(axis='y', labelcolor=color)

    # Third axis for Success Rate
    ax3 = ax1.twinx()
    ax3.spines['right'].set_position(('outward', 60))
    ax3.set_facecolor('white')  # Set the background color of the third axis to white
    color = 'tab:green'
    ax3.set_ylabel('Success Rate', color=color)
    ax3.plot(success_rates, color=color, alpha=0.1, label='Success Rate', marker='^', markersize=5)
    ax3.plot(range(window_size - 1, len(success_rates)), smoothed_success_rates, color=color, label=f'Smoothed Success Rate (window={window_size})', marker='^', markersize=3)
    ax3.tick_params(axis='y', labelcolor=color)

    # Title and grid
    plt.title("Meta-Loss, Average Reward, and Success Rate Progress")
    fig.tight_layout()  # Adjust layout to prevent label clipping
    plt.grid(True)

    # Show the plot
    plt.show()

# Function to calculate moving average
def moving_average(data, window_size=30):
    return np.convolve(data, np.ones(window_size) / window_size, mode='valid')

# Simplified run with single complexity and success rate tracking
if __name__ == "__main__":
    meta_learning_rate = 1e-3
    epsilon_start = 0.9
    epsilon_decay = 0.99
    num_iterations = 500
    num_inner_steps = 50
    eta = 0.1
    epsilon = 1e-5

    meta_losses, meta_rewards, success_rates = meta_train_fixed_complexity(
        meta_learning_rate=meta_learning_rate,
        epsilon_start=epsilon_start,
        epsilon_decay=epsilon_decay,
        num_iterations=num_iterations,
        num_inner_steps=num_inner_steps,
        eta=eta,
        epsilon=epsilon
    )

    # Plot results
    plot_meta_losses_rewards_success(meta_losses, meta_rewards, success_rates)