removing all irrelevant files

DDQN_FNN.py

@@ -0,0 +1,103 @@
+import numpy as np
+from collections import deque
+from tensorflow import keras
+import random
+
+
+class DoubleDeepQNetwork:
+    def __init__(self, states, actions, history, alpha, gamma, epsilon, epsilon_min, epsilon_decay):
+        self.nS = states
+        self.nA = actions
+        self.history = history
+        self.memory = deque([], maxlen=2500)
+        self.alpha = alpha
+        self.gamma = gamma
+        # Explore/Exploit
+        self.epsilon = epsilon
+        self.epsilon_min = epsilon_min
+        self.epsilon_decay = epsilon_decay
+        self.model = self.build_model()
+        self.model_target = self.build_model()  # Second (target) neural network
+        self.update_target_from_model()  # Update weights
+        self.loss = []
+
+    def build_model(self):
+        model = keras.Sequential()  # linear stack of layers https://keras.io/models/sequential/
+        model.add(keras.layers.Dense(24, input_dim=self.history * self.nS, activation='relu'))  # [Input] -> Layer 1
+        #   Dense: Densely connected layer https://keras.io/layers/core/
+        #   24: Number of neurons
+        #   input_dim: Number of input variables
+        #   activation: Rectified Linear Unit (relu) ranges >= 0
+        model.add(keras.layers.Dense(24, activation='relu'))  # Layer 2 -> 3
+        model.add(keras.layers.Dense(self.nA, activation='linear'))  # Layer 3 -> [output]
+        #   Size has to match the output (different actions)
+        #   Linear activation on the last layer
+        model.compile(loss='mean_squared_error',  # Loss function: Mean Squared Error
+                      optimizer=keras.optimizers.Adam(
+                          lr=self.alpha))  # Optimizer: Adam (Feel free to check other options)
+        return model
+
+    def update_target_from_model(self):
+        # Update the target model from the base model
+        self.model_target.set_weights(self.model.get_weights())
+
+    def action(self, state):
+        if np.random.rand() <= self.epsilon:
+            return random.randrange(self.nA)  # Explore
+        action_vals = self.model.predict(state)  # Exploit: Use the NN to predict the correct action from this state
+        return np.argmax(action_vals[0])
+
+    def test_action(self, state):  # Exploit
+        action_vals = self.model.predict(state)
+        return np.argmax(action_vals[0])
+
+    def store(self, state, action, reward, next_state, done):
+        # Store the experience in memory
+        self.memory.append((state, action, reward, next_state, done))
+
+    def save_model(self, agentName):
+        # Save the agent model weights in a file
+        self.model.save(agentName)
+
+    def experience_replay(self, batch_size):
+        # Execute the experience replay
+        minibatch = random.sample(self.memory, batch_size)  # Randomly sample from memory
+
+        # Convert to numpy for speed by vectorization
+        x = []
+        y = []
+        np_array = np.array(minibatch)
+        st = np.zeros((0, self.history*self.nS))  # States
+        nst = np.zeros((0, self.history*self.nS))  # Next States
+        for i in range(len(np_array)):  # Creating the state and next state np arrays
+            st = np.append(st, np_array[i, 0], axis=0)
+            nst = np.append(nst, np_array[i, 3], axis=0)
+        st_predict = self.model.predict(st)  # Here is the speedup! I can predict on the ENTIRE batch
+        nst_predict = self.model.predict(nst)
+        nst_predict_target = self.model_target.predict(nst)  # Predict from the TARGET
+        index = 0
+        for state, action, reward, next_state, done in minibatch:
+            x.append(state)
+            # Predict from state
+            nst_action_predict_target = nst_predict_target[index]
+            nst_action_predict_model = nst_predict[index]
+            if done:  # Terminal: Just assign reward much like {* (not done) - QB[state][action]}
+                target = reward
+            else:  # Non-terminal
+                target = reward + self.gamma * nst_action_predict_target[
+                    np.argmax(nst_action_predict_model)]  # Using Q to get T is Double DQN
+            target_f = st_predict[index]
+            target_f[action] = target
+            y.append(target_f)
+            index += 1
+        # Reshape for Keras Fit
+        x_reshape = np.array(x).reshape(batch_size, self.history * self.nS)
+        y_reshape = np.array(y)
+        epoch_count = 1
+        hist = self.model.fit(x_reshape, y_reshape, epochs=epoch_count, verbose=0)
+        # Graph Losses
+        for i in range(epoch_count):
+            self.loss.append(hist.history['loss'][i])
+        # Decay Epsilon
+        if self.epsilon > self.epsilon_min:
+            self.epsilon *= self.epsilon_decay

agents/Base_Agent.py

@@ -1,394 +0,0 @@
-import logging
-import os
-import sys
-import gym
-import random
-import numpy as np
-import torch
-import time
-# import tensorflow as tf
-from nn_builder.pytorch.NN import NN
-# from tensorboardX import SummaryWriter
-from torch.optim import optimizer
-
-
-class Base_Agent(object):
-
-    def __init__(self, config):
-        self.logger = self.setup_logger()
-        self.debug_mode = config.debug_mode
-        # if self.debug_mode: self.tensorboard = SummaryWriter()
-        self.config = config
-        self.set_random_seeds(config.seed)
-        self.environment = config.environment
-        self.environment_title = self.get_environment_title()
-        self.action_types = "DISCRETE" if self.environment.action_space.dtype == np.int64 else "CONTINUOUS"
-        self.action_size = int(self.get_action_size())
-        self.config.action_size = self.action_size
-
-        self.lowest_possible_episode_score = self.get_lowest_possible_episode_score()
-
-        self.state_size = int(self.get_state_size())
-        self.hyperparameters = config.hyperparameters
-        self.average_score_required_to_win = self.get_score_required_to_win()
-        self.rolling_score_window = self.get_trials()
-        # self.max_steps_per_episode = self.environment.spec.max_episode_steps
-        self.total_episode_score_so_far = 0
-        self.game_full_episode_scores = []
-        self.game_full_episode_signals = []
-        self.rolling_results = []
-        self.max_rolling_score_seen = float("-inf")
-        self.max_episode_score_seen = float("-inf")
-        self.episode_number = 0
-        self.device = "cuda:0" if config.use_GPU else "cpu"
-        self.visualise_results_boolean = config.visualise_individual_results
-        self.global_step_number = 0
-        self.turn_off_exploration = False if config.training else True
-        gym.logger.set_level(40)  # stops it from printing an unnecessary warning
-        self.log_game_info()
-
-    def step(self):
-        """Takes a step in the game. This method must be overriden by any agent"""
-        raise ValueError("Step needs to be implemented by the agent")
-
-    def get_environment_title(self):
-        """Extracts name of environment from it"""
-        try:
-            name = self.environment.unwrapped.id
-        except AttributeError:
-            try:
-                if str(self.environment.unwrapped)[1:11] == "FetchReach":
-                    return "FetchReach"
-                elif str(self.environment.unwrapped)[1:8] == "AntMaze":
-                    return "AntMaze"
-                elif str(self.environment.unwrapped)[1:7] == "Hopper":
-                    return "Hopper"
-                elif str(self.environment.unwrapped)[1:9] == "Walker2d":
-                    return "Walker2d"
-                else:
-                    name = self.environment.spec.id.split("-")[0]
-            except AttributeError:
-                name = str(self.environment.env)
-                if name[0:10] == "TimeLimit<": name = name[10:]
-                name = name.split(" ")[0]
-                if name[0] == "<": name = name[1:]
-                if name[-3:] == "Env": name = name[:-3]
-        return name
-
-    def get_lowest_possible_episode_score(self):
-        """Returns the lowest possible episode score you can get in an environment"""
-        if self.environment_title == "Taxi": return -800
-        return None
-
-    def get_action_size(self):
-        """Gets the action_size for the gym env into the correct shape for a neural network"""
-        if "overwrite_action_size" in self.config.__dict__: return self.config.overwrite_action_size
-        if "action_size" in self.environment.__dict__: return self.environment.action_size
-        if self.action_types == "DISCRETE":
-            return self.environment.action_space.n
-        else:
-            return self.environment.action_space.shape[0]
-
-    def get_state_size(self):
-        """Gets the state_size for the gym env into the correct shape for a neural network"""
-        random_state = self.environment.reset()
-        if isinstance(random_state, dict):
-            state_size = random_state["observation"].shape[0] + random_state["desired_goal"].shape[0]
-            return state_size
-        else:
-            return random_state.size
-
-    def get_score_required_to_win(self):
-        """Gets average score required to win game"""
-        print("TITLE ", self.environment_title)
-        if self.environment_title == "FetchReach": return -5
-        if self.environment_title in ["AntMaze", "Hopper", "Walker2d"]:
-            print("Score required to win set to infinity therefore no learning rate annealing will happen")
-            return float("inf")
-        try:
-            return self.environment.unwrapped.reward_threshold
-        except AttributeError:
-            try:
-                return self.environment.spec.reward_threshold
-            except AttributeError:
-                return self.environment.unwrapped.spec.reward_threshold
-
-    def get_trials(self):
-        """Gets the number of trials to average a score over"""
-        if self.environment_title in ["AntMaze", "FetchReach", "Hopper", "Walker2d", "CartPole"]: return 100
-        try:
-            return self.environment.unwrapped.trials
-        except AttributeError:
-            return self.environment.spec.trials
-
-    def setup_logger(self):
-        """Sets up the logger"""
-        filename = "Training.log"
-        try:
-            if os.path.isfile(filename):
-                os.remove(filename)
-        except:
-            pass
-
-        logger = logging.getLogger(__name__)
-        logger.setLevel(logging.INFO)
-        # create a file handler
-        handler = logging.FileHandler(filename)
-        handler.setLevel(logging.INFO)
-        # create a logging format
-        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-        handler.setFormatter(formatter)
-        # add the handlers to the logger
-        logger.addHandler(handler)
-        return logger
-
-    def log_game_info(self):
-        """Logs info relating to the game"""
-        for ix, param in enumerate(
-                [self.environment_title, self.action_types, self.action_size, self.lowest_possible_episode_score,
-                 self.state_size, self.hyperparameters, self.average_score_required_to_win, self.rolling_score_window,
-                 self.device]):
-            self.logger.info("{} -- {}".format(ix, param))
-
-    def set_random_seeds(self, random_seed):
-        """Sets all possible random seeds so results can be reproduced"""
-        os.environ['PYTHONHASHSEED'] = str(random_seed)
-        torch.backends.cudnn.deterministic = True
-        torch.backends.cudnn.benchmark = False
-        torch.manual_seed(random_seed)
-        # tf.set_random_seed(random_seed)
-        random.seed(random_seed)
-        np.random.seed(random_seed)
-        if torch.cuda.is_available():
-            torch.cuda.manual_seed_all(random_seed)
-            torch.cuda.manual_seed(random_seed)
-        if hasattr(gym.spaces, 'prng'):
-            gym.spaces.prng.seed(random_seed)
-
-    def reset_game(self):
-        """Resets the game information so we are ready to play a new episode"""
-        self.environment.seed(self.config.seed)
-        self.state = self.environment.reset()
-        self.next_state = None
-        self.action = None
-        self.reward = None
-        self.signal = None
-        self.done = False
-        self.total_episode_score_so_far = 0
-        self.total_episode_signal_so_far = 0
-        self.episode_states = []
-        self.episode_rewards = []
-        self.episode_signals = []
-        self.episode_actions = []
-        self.episode_next_states = []
-        self.episode_dones = []
-        self.episode_desired_goals = []
-        self.episode_achieved_goals = []
-        self.episode_observations = []
-        if "exploration_strategy" in self.__dict__.keys(): self.exploration_strategy.reset()
-        self.logger.info("Reseting game -- New start state {}".format(self.state))
-
-    def track_episodes_data(self):
-        """Saves the data from the recent episodes"""
-        self.episode_states.append(self.state)
-        self.episode_actions.append(self.action)
-        self.episode_rewards.append(self.reward)
-        self.episode_signals.append(self.signal)
-        self.episode_next_states.append(self.next_state)
-        self.episode_dones.append(self.done)
-
-    def run_n_episodes(self, num_episodes=None, show_whether_achieved_goal=True, save_and_print_results=True):
-        """Runs game to completion n times and then summarises results and saves model (if asked to)"""
-        if num_episodes is None: num_episodes = self.config.num_episodes_to_run
-        start = time.time()
-        while self.episode_number < num_episodes:
-            self.reset_game()
-            self.step()
-            if save_and_print_results: self.save_and_print_result()
-        time_taken = time.time() - start
-        if show_whether_achieved_goal: self.show_whether_achieved_goal()
-        if self.config.save_model: self.locally_save_policy()
-        return self.game_full_episode_scores, self.rolling_results, time_taken, self.game_full_episode_signals
-
-    def conduct_action(self, action):
-        """Conducts an action in the environment"""
-        self.next_state, self.reward, self.done, self.signal = self.environment.step(action)
-        self.total_episode_score_so_far += self.reward
-        self.total_episode_signal_so_far += self.signal
-        if self.hyperparameters["clip_rewards"]: self.reward = max(min(self.reward, 1.0), -1.0)
-
-    def save_and_print_result(self):
-        """Saves and prints results of the game"""
-        self.save_result()
-        self.print_rolling_result()
-
-    def save_result(self):
-        """Saves the result of an episode of the game"""
-        self.game_full_episode_scores.append(self.total_episode_score_so_far)
-        self.game_full_episode_signals.append(self.total_episode_signal_so_far)
-        self.rolling_results.append(np.mean(self.game_full_episode_scores[-1 * self.rolling_score_window:]))
-        self.save_max_result_seen()
-
-    def save_max_result_seen(self):
-        """Updates the best episode result seen so far"""
-        if self.game_full_episode_scores[-1] > self.max_episode_score_seen:
-            self.max_episode_score_seen = self.game_full_episode_scores[-1]
-
-        if self.rolling_results[-1] > self.max_rolling_score_seen:
-            if len(self.rolling_results) > self.rolling_score_window:
-                self.max_rolling_score_seen = self.rolling_results[-1]
-
-    def print_rolling_result(self):
-        """Prints out the latest episode results"""
-        text = """"\r Episode {0}, Score: {3: .2f}, Max score seen: {4: .2f}, Rolling score: {1: .2f}, Max rolling score seen: {2: .2f}"""
-        sys.stdout.write(
-            text.format(len(self.game_full_episode_scores), self.rolling_results[-1], self.max_rolling_score_seen,
-                        self.game_full_episode_scores[-1], self.max_episode_score_seen))
-        sys.stdout.flush()
-
-    def show_whether_achieved_goal(self):
-        """Prints out whether the agent achieved the environment target goal"""
-        index_achieved_goal = self.achieved_required_score_at_index()
-        print(" ")
-        if index_achieved_goal == -1:  # this means agent never achieved goal
-            print("\033[91m" + "\033[1m" +
-                  "{} did not achieve required score \n".format(self.agent_name) +
-                  "\033[0m" + "\033[0m")
-        else:
-            print("\033[92m" + "\033[1m" +
-                  "{} achieved required score at episode {} \n".format(self.agent_name, index_achieved_goal) +
-                  "\033[0m" + "\033[0m")
-
-    def achieved_required_score_at_index(self):
-        """Returns the episode at which agent achieved goal or -1 if it never achieved it"""
-        for ix, score in enumerate(self.rolling_results):
-            if score > self.average_score_required_to_win:
-                return ix
-        return -1
-
-    def update_learning_rate(self, starting_lr, optimizer):
-        """Lowers the learning rate according to how close we are to the solution"""
-        if len(self.rolling_results) > 0:
-            last_rolling_score = self.rolling_results[-1]
-            if last_rolling_score > 0.75 * self.average_score_required_to_win:
-                new_lr = starting_lr / 100.0
-            elif last_rolling_score > 0.6 * self.average_score_required_to_win:
-                new_lr = starting_lr / 20.0
-            elif last_rolling_score > 0.5 * self.average_score_required_to_win:
-                new_lr = starting_lr / 10.0
-            elif last_rolling_score > 0.25 * self.average_score_required_to_win:
-                new_lr = starting_lr / 2.0
-            else:
-                new_lr = starting_lr
-            for g in optimizer.param_groups:
-                g['lr'] = new_lr
-        if random.random() < 0.001: self.logger.info("Learning rate {}".format(new_lr))
-
-    def enough_experiences_to_learn_from(self):
-        """Boolean indicated whether there are enough experiences in the memory buffer to learn from"""
-        return len(self.memory) > self.hyperparameters["batch_size"]
-
-    def save_experience(self, memory=None, experience=None):
-        """Saves the recent experience to the memory buffer"""
-        if memory is None: memory = self.memory
-        if experience is None: experience = self.state, self.action, self.reward, self.next_state, self.done
-        memory.add_experience(*experience)
-
-    def take_optimisation_step(self, optimizer, network, loss, clipping_norm=None, retain_graph=False):
-        """Takes an optimisation step by calculating gradients given the loss and then updating the parameters"""
-        if not isinstance(network, list): network = [network]
-        optimizer.zero_grad()  # reset gradients to 0
-        loss.backward(retain_graph=retain_graph)  # this calculates the gradients
-        self.logger.info("Loss -- {}".format(loss.item()))
-        if self.debug_mode: self.log_gradient_and_weight_information(network, optimizer)
-        if clipping_norm is not None:
-            for net in network:
-                torch.nn.utils.clip_grad_norm_(net.parameters(),
-                                               clipping_norm)  # clip gradients to help stabilise training
-        optimizer.step()  # this applies the gradients
-
-    def log_gradient_and_weight_information(self, network, optimizer):
-
-        # log weight information
-        total_norm = 0
-        for name, param in network.named_parameters():
-            param_norm = param.grad.data.norm(2)
-            total_norm += param_norm.item() ** 2
-        total_norm = total_norm ** (1. / 2)
-        self.logger.info("Gradient Norm {}".format(total_norm))
-
-        for g in optimizer.param_groups:
-            learning_rate = g['lr']
-            break
-        self.logger.info("Learning Rate {}".format(learning_rate))
-
-    def soft_update_of_target_network(self, local_model, target_model, tau):
-        """Updates the target network in the direction of the local network but by taking a step size
-        less than one so the target network's parameter values trail the local networks. This helps stabilise training"""
-        for target_param, local_param in zip(target_model.parameters(), local_model.parameters()):
-            target_param.data.copy_(tau * local_param.data + (1.0 - tau) * target_param.data)
-
-    def create_NN(self, input_dim, output_dim, key_to_use=None, override_seed=None, hyperparameters=None):
-        """Creates a neural network for the agents to use"""
-        if hyperparameters is None: hyperparameters = self.hyperparameters
-        if key_to_use: hyperparameters = hyperparameters[key_to_use]
-        if override_seed:
-            seed = override_seed
-        else:
-            seed = self.config.seed
-
-        default_hyperparameter_choices = {"output_activation": None, "hidden_activations": "relu", "dropout": 0.0,
-                                          "initialiser": "default", "batch_norm": False,
-                                          "columns_of_data_to_be_embedded": [],
-                                          "embedding_dimensions": [], "y_range": ()}
-
-        for key in default_hyperparameter_choices:
-            if key not in hyperparameters.keys():
-                hyperparameters[key] = default_hyperparameter_choices[key]
-
-        return NN(input_dim=input_dim, layers_info=hyperparameters["linear_hidden_units"] + [output_dim],
-                  output_activation=hyperparameters["final_layer_activation"],
-                  batch_norm=hyperparameters["batch_norm"], dropout=hyperparameters["dropout"],
-                  hidden_activations=hyperparameters["hidden_activations"], initialiser=hyperparameters["initialiser"],
-                  columns_of_data_to_be_embedded=hyperparameters["columns_of_data_to_be_embedded"],
-                  embedding_dimensions=hyperparameters["embedding_dimensions"], y_range=hyperparameters["y_range"],
-                  random_seed=seed).to(self.device)
-
-    def turn_on_any_epsilon_greedy_exploration(self):
-        """Turns off all exploration with respect to the epsilon greedy exploration strategy"""
-        print("Turning on epsilon greedy exploration")
-        self.turn_off_exploration = False
-
-    def turn_off_any_epsilon_greedy_exploration(self):
-        """Turns off all exploration with respect to the epsilon greedy exploration strategy"""
-        print("Turning off epsilon greedy exploration")
-        self.turn_off_exploration = True
-
-    def freeze_all_but_output_layers(self, network):
-        """Freezes all layers except the output layer of a network"""
-        print("Freezing hidden layers")
-        for param in network.named_parameters():
-            param_name = param[0]
-            assert "hidden" in param_name or "output" in param_name or "embedding" in param_name, "Name {} of network layers not understood".format(
-                param_name)
-            if "output" not in param_name:
-                param[1].requires_grad = False
-
-    def unfreeze_all_layers(self, network):
-        """Unfreezes all layers of a network"""
-        print("Unfreezing all layers")
-        for param in network.parameters():
-            param.requires_grad = True
-
-    @staticmethod
-    def move_gradients_one_model_to_another(from_model, to_model, set_from_gradients_to_zero=False):
-        """Copies gradients from from_model to to_model"""
-        for from_model, to_model in zip(from_model.parameters(), to_model.parameters()):
-            to_model._grad = from_model.grad.clone()
-            if set_from_gradients_to_zero: from_model._grad = None
-
-    @staticmethod
-    def copy_model_over(from_model, to_model):
-        """Copies model parameters from from_model to to_model"""
-        for to_model, from_model in zip(to_model.parameters(), from_model.parameters()):
-            to_model.data.copy_(from_model.data.clone())

agents/DQN_agents/DDQN.py

@@ -1,18 +0,0 @@
-from agents.DQN_agents.DQN_With_Fixed_Q_Targets import DQN_With_Fixed_Q_Targets
-
-class DDQN(DQN_With_Fixed_Q_Targets):
-    """A double DQN agent"""
-    agent_name = "DDQN"
-
-    def __init__(self, config):
-        DQN_With_Fixed_Q_Targets.__init__(self, config)
-
-    def compute_q_values_for_next_states(self, next_states):
-        """Computes the q_values for next state we will use to create the loss to train the Q network. Double DQN
-        uses the local index to pick the maximum q_value action and then the target network to calculate the q_value.
-        The reasoning behind this is that it will help stop the network from overestimating q values"""
-        max_action_indexes = self.q_network_local(next_states).detach().argmax(1)
-        Q_targets_next = self.q_network_target(next_states).gather(1, max_action_indexes.unsqueeze(1))
-        return Q_targets_next 
-            
-

agents/DQN_agents/DDQN_With_Prioritised_Experience_Replay.py

@@ -1,37 +0,0 @@
-import torch
-import torch.nn.functional as F
-from agents.DQN_agents.DDQN import DDQN
-from utilities.data_structures.Prioritised_Replay_Buffer import Prioritised_Replay_Buffer
-
-class DDQN_With_Prioritised_Experience_Replay(DDQN):
-    """A DQN agent with prioritised experience replay"""
-    agent_name = "DDQN with Prioritised Replay"
-
-    def __init__(self, config):
-        DDQN.__init__(self, config)
-        self.memory = Prioritised_Replay_Buffer(self.hyperparameters, config.seed)
-
-    def learn(self):
-        """Runs a learning iteration for the Q network after sampling from the replay buffer in a prioritised way"""
-        sampled_experiences, importance_sampling_weights = self.memory.sample()
-        states, actions, rewards, next_states, dones = sampled_experiences
-        loss, td_errors = self.compute_loss_and_td_errors(states, next_states, rewards, actions, dones, importance_sampling_weights)
-        self.take_optimisation_step(self.q_network_optimizer, self.q_network_local, loss, self.hyperparameters["gradient_clipping_norm"])
-        self.soft_update_of_target_network(self.q_network_local, self.q_network_target, self.hyperparameters["tau"])
-        self.memory.update_td_errors(td_errors.squeeze(1))
-
-    def save_experience(self):
-        """Saves the latest experience including the td_error"""
-        max_td_error_in_experiences = self.memory.give_max_td_error() + 1e-9
-        self.memory.add_experience(max_td_error_in_experiences, self.state, self.action, self.reward, self.next_state, self.done)
-
-    def compute_loss_and_td_errors(self, states, next_states, rewards, actions, dones, importance_sampling_weights):
-        """Calculates the loss for the local Q network. It weighs each observations loss according to the importance
-        sampling weights which come from the prioritised replay buffer"""
-        Q_targets = self.compute_q_targets(next_states, rewards, dones)
-        Q_expected = self.compute_expected_q_values(states, actions)
-        loss = F.mse_loss(Q_expected, Q_targets)
-        loss = loss * importance_sampling_weights
-        loss = torch.mean(loss)
-        td_errors = Q_targets.data.cpu().numpy() - Q_expected.data.cpu().numpy()
-        return loss, td_errors

agents/DQN_agents/DQN.py

@@ -1,135 +0,0 @@
-from collections import Counter
-
-import torch
-import random
-import torch.optim as optim
-import torch.nn.functional as F
-import numpy as np
-from agents.Base_Agent import Base_Agent
-from exploration_strategies.Epsilon_Greedy_Exploration import Epsilon_Greedy_Exploration
-from utilities.data_structures.Replay_Buffer import Replay_Buffer
-
-
-class DQN(Base_Agent):
-    """A deep Q learning agent"""
-    agent_name = "DQN"
-
-    def __init__(self, config):
-        Base_Agent.__init__(self, config)
-        self.memory = Replay_Buffer(self.hyperparameters["buffer_size"], self.hyperparameters["batch_size"],
-                                    config.seed, self.device)
-        self.q_network_local = self.create_NN(input_dim=self.state_size, output_dim=self.action_size)
-        self.q_network_optimizer = optim.Adam(self.q_network_local.parameters(),
-                                              lr=self.hyperparameters["learning_rate"], eps=1e-4)
-        self.exploration_strategy = Epsilon_Greedy_Exploration(config)
-
-    def reset_game(self):
-        super(DQN, self).reset_game()
-        self.update_learning_rate(self.hyperparameters["learning_rate"], self.q_network_optimizer)
-
-    def step(self):
-        """Runs a step within a game including a learning step if required"""
-        while not self.done:
-            self.action = self.pick_action()
-            self.conduct_action(self.action)
-            # If we are in training mode
-            if self.config.training:
-                if self.time_for_q_network_to_learn():
-                    for _ in range(self.hyperparameters["learning_iterations"]):
-                        self.learn()
-                self.save_experience()
-            self.state = self.next_state  # this is to set the state for the next iteration
-            self.global_step_number += 1
-        self.episode_number += 1
-
-    def pick_action(self, state=None):
-        """Uses the local Q network and an epsilon greedy policy to pick an action"""
-        # PyTorch only accepts mini-batches and not single observations so we have to use unsqueeze to add
-        # a "fake" dimension to make it a mini-batch rather than a single observation
-        if state is None: state = self.state
-        if isinstance(state, np.int64) or isinstance(state, int): state = np.array([state])
-        state = torch.from_numpy(state).float().unsqueeze(0).to(self.device)
-        if len(state.shape) < 2: state = state.unsqueeze(0)
-        if not self.config.training:
-            self.q_network_local = self.locally_load_policy()
-        self.q_network_local.eval()  # puts network in evaluation mode
-        with torch.no_grad():
-            action_values = self.q_network_local(state)
-        if self.config.training:
-            self.q_network_local.train()  # puts network back in training mode
-        action = self.exploration_strategy.perturb_action_for_exploration_purposes({"action_values": action_values,
-                                                                                        "turn_off_exploration": self.turn_off_exploration,
-                                                                                        "episode_number": self.episode_number})
-        self.logger.info("Q values {} -- Action chosen {}".format(action_values, action))
-        return action
-
-    def learn(self, experiences=None):
-        """Runs a learning iteration for the Q network"""
-        if experiences is None:
-            states, actions, rewards, next_states, dones = self.sample_experiences()  # Sample experiences
-        else:
-            states, actions, rewards, next_states, dones = experiences
-        loss = self.compute_loss(states, next_states, rewards, actions, dones)
-
-        actions_list = [action_X.item() for action_X in actions]
-
-        self.logger.info("Action counts {}".format(Counter(actions_list)))
-        self.take_optimisation_step(self.q_network_optimizer, self.q_network_local, loss,
-                                    self.hyperparameters["gradient_clipping_norm"])
-
-    def compute_loss(self, states, next_states, rewards, actions, dones):
-        """Computes the loss required to train the Q network"""
-        with torch.no_grad():
-            Q_targets = self.compute_q_targets(next_states, rewards, dones)
-        Q_expected = self.compute_expected_q_values(states, actions)
-        loss = F.mse_loss(Q_expected, Q_targets)
-        return loss
-
-    def compute_q_targets(self, next_states, rewards, dones):
-        """Computes the q_targets we will compare to predicted q values to create the loss to train the Q network"""
-        Q_targets_next = self.compute_q_values_for_next_states(next_states)
-        Q_targets = self.compute_q_values_for_current_states(rewards, Q_targets_next, dones)
-        return Q_targets
-
-    def compute_q_values_for_next_states(self, next_states):
-        """Computes the q_values for next state we will use to create the loss to train the Q network"""
-        Q_targets_next = self.q_network_local(next_states).detach().max(1)[0].unsqueeze(1)
-        return Q_targets_next
-
-    def compute_q_values_for_current_states(self, rewards, Q_targets_next, dones):
-        """Computes the q_values for current state we will use to create the loss to train the Q network"""
-        Q_targets_current = rewards + (self.hyperparameters["discount_rate"] * Q_targets_next * (1 - dones))
-        return Q_targets_current
-
-    def compute_expected_q_values(self, states, actions):
-        """Computes the expected q_values we will use to create the loss to train the Q network"""
-        Q_expected = self.q_network_local(states).gather(1,
-                                                         actions.long())  # must convert actions to long so can be used as index
-        return Q_expected
-
-    def locally_save_policy(self):
-        """Saves the policy"""
-        torch.save(self.q_network_local.state_dict(),
-                   "{}/{}_network.pt".format(self.config.models_dir, self.agent_name))
-
-    def locally_load_policy(self):
-        """loads the policy"""
-        filename = f'{self.config.models_dir}/{self.agent_name}_network.pt'
-        saved_q_network_local = self.q_network_local
-        saved_q_network_local.load_state_dict(torch.load(filename))
-        return saved_q_network_local
-
-    def time_for_q_network_to_learn(self):
-        """Returns boolean indicating whether enough steps have been taken for learning to begin and there are
-        enough experiences in the replay buffer to learn from"""
-        return self.right_amount_of_steps_taken() and self.enough_experiences_to_learn_from()
-
-    def right_amount_of_steps_taken(self):
-        """Returns boolean indicating whether enough steps have been taken for learning to begin"""
-        return self.global_step_number % self.hyperparameters["update_every_n_steps"] == 0
-
-    def sample_experiences(self):
-        """Draws a random sample of experience from the memory buffer"""
-        experiences = self.memory.sample()
-        states, actions, rewards, next_states, dones = experiences
-        return states, actions, rewards, next_states, dones

agents/DQN_agents/DQN_HER.py

@@ -1,30 +0,0 @@
-from agents.DQN_agents.DQN import DQN
-from agents.HER_Base import HER_Base
-
-class DQN_HER(HER_Base, DQN):
-    """DQN algorithm with hindsight experience replay"""
-    agent_name = "DQN-HER"
-    def __init__(self, config):
-        DQN.__init__(self, config)
-        HER_Base.__init__(self, self.hyperparameters["buffer_size"], self.hyperparameters["batch_size"],
-                          self.hyperparameters["HER_sample_proportion"])
-
-    def step(self):
-        """Runs a step within a game including a learning step if required"""
-        while not self.done:
-            self.action = self.pick_action()
-            self.conduct_action_in_changeable_goal_envs(self.action)
-            if self.time_for_q_network_to_learn():
-                for _ in range(self.hyperparameters["learning_iterations"]):
-                    self.learn(experiences=self.sample_from_HER_and_Ordinary_Buffer())
-            self.track_changeable_goal_episodes_data()
-            self.save_experience()
-            if self.done: self.save_alternative_experience()
-            self.state_dict = self.next_state_dict  # this is to set the state for the next iteration
-            self.state = self.next_state
-            self.global_step_number += 1
-        self.episode_number += 1
-
-    def enough_experiences_to_learn_from(self):
-        """Returns booleans indicating whether there are enough experiences in the two replay buffers to learn from"""
-        return len(self.memory) > self.ordinary_buffer_batch_size and len(self.HER_memory) > self.HER_buffer_batch_size

agents/DQN_agents/DQN_With_Fixed_Q_Targets.py

@@ -1,23 +0,0 @@
-import copy
-
-from agents.Base_Agent import Base_Agent
-from agents.DQN_agents.DQN import DQN
-
-class DQN_With_Fixed_Q_Targets(DQN):
-    """A DQN agent that uses an older version of the q_network as the target network"""
-    agent_name = "DQN with Fixed Q Targets"
-    def __init__(self, config):
-        DQN.__init__(self, config)
-        self.q_network_target = self.create_NN(input_dim=self.state_size, output_dim=self.action_size)
-        Base_Agent.copy_model_over(from_model=self.q_network_local, to_model=self.q_network_target)
-
-    def learn(self, experiences=None):
-        """Runs a learning iteration for the Q network"""
-        super(DQN_With_Fixed_Q_Targets, self).learn(experiences=experiences)
-        self.soft_update_of_target_network(self.q_network_local, self.q_network_target,
-                                           self.hyperparameters["tau"])  # Update the target network
-
-    def compute_q_values_for_next_states(self, next_states):
-        """Computes the q_values for next state we will use to create the loss to train the Q network"""
-        Q_targets_next = self.q_network_target(next_states).detach().max(1)[0].unsqueeze(1)
-        return Q_targets_next

agents/DQN_agents/Dueling_DDQN.py

@@ -1,64 +0,0 @@
-import torch
-from torch import optim
-from agents.Base_Agent import Base_Agent
-from agents.DQN_agents.DDQN import DDQN
-
-class Dueling_DDQN(DDQN):
-    """A dueling double DQN agent as described in the paper http://proceedings.mlr.press/v48/wangf16.pdf"""
-    agent_name = "Dueling DDQN"
-
-    def __init__(self, config):
-        DDQN.__init__(self, config)
-        self.q_network_local = self.create_NN(input_dim=self.state_size, output_dim=self.action_size + 1)
-        self.q_network_optimizer = optim.Adam(self.q_network_local.parameters(), lr=self.hyperparameters["learning_rate"], eps=1e-4)
-        self.q_network_target = self.create_NN(input_dim=self.state_size, output_dim=self.action_size + 1)
-        Base_Agent.copy_model_over(from_model=self.q_network_local, to_model=self.q_network_target)
-
-    def pick_action(self, state=None):
-        """Uses the local Q network and an epsilon greedy policy to pick an action"""
-        # PyTorch only accepts mini-batches and not single observations so we have to use unsqueeze to add
-        # a "fake" dimension to make it a mini-batch rather than a single observation
-        if state is None: state = self.state
-        state = torch.from_numpy(state).float().unsqueeze(0).to(self.device)
-        if len(state.shape) < 2: state = state.unsqueeze(0)
-        self.q_network_local.eval()
-        with torch.no_grad():
-            action_values = self.q_network_local(state)
-            action_values = action_values[:, :-1] #because we treat the last output element as state-value and rest as advantages
-        self.q_network_local.train()
-        action = self.exploration_strategy.perturb_action_for_exploration_purposes({"action_values": action_values,
-                                                                                    "turn_off_exploration": self.turn_off_exploration,
-                                                                                    "episode_number": self.episode_number})
-        return action
-
-    def compute_q_values_for_next_states(self, next_states):
-        """Computes the q_values for next state we will use to create the loss to train the Q network. Double DQN
-        uses the local index to pick the maximum q_value action and then the target network to calculate the q_value.
-        The reasoning behind this is that it will help stop the network from overestimating q values"""
-        max_action_indexes = self.q_network_local(next_states)[:, :-1].detach().argmax(1)
-        duelling_network_output = self.q_network_target(next_states)
-        q_values = self.calculate_duelling_q_values(duelling_network_output)
-        Q_targets_next = q_values.gather(1, max_action_indexes.unsqueeze(1))
-        return Q_targets_next
-
-    def calculate_duelling_q_values(self, duelling_q_network_output):
-        """Calculates the q_values using the duelling network architecture. This is equation (9) in the paper
-        referenced at the top of the class"""
-        state_value = duelling_q_network_output[:, -1]
-        avg_advantage = torch.mean(duelling_q_network_output[:, :-1], dim=1)
-        q_values = state_value.unsqueeze(1) + (duelling_q_network_output[:, :-1] - avg_advantage.unsqueeze(1))
-        return q_values
-
-    def compute_expected_q_values(self, states, actions):
-        """Computes the expected q_values we will use to create the loss to train the Q network"""
-        duelling_network_output = self.q_network_local(states)
-        q_values = self.calculate_duelling_q_values(duelling_network_output)
-        Q_expected = q_values.gather(1, actions.long())
-        return Q_expected
-
-
-
-
-
-
-

agents/DQN_agents/__init__.py

@@ -1 +0,0 @@
-import os, sys; sys.path.append(os.path.dirname(os.path.realpath(__file__)))

agents/HER_Base.py

@@ -1,100 +0,0 @@
-import torch
-import numpy as np
-from utilities.data_structures.Replay_Buffer import Replay_Buffer
-from utilities.Utility_Functions import abstract
-
-@abstract
-class HER_Base(object):
-    """Contains methods needed to turn an algorithm into a hindsight experience replay (HER) algorithm"""
-    def __init__(self, buffer_size, batch_size, HER_sample_proportion):
-        self.HER_memory = Replay_Buffer(buffer_size, batch_size, self.config.seed)
-        self.ordinary_buffer_batch_size = int(batch_size * (1.0 - HER_sample_proportion))
-        self.HER_buffer_batch_size = batch_size - self.ordinary_buffer_batch_size
-
-    def reset_game(self):
-        """Resets the game information so we are ready to play a new episode"""
-        self.state_dict = self.environment.reset()
-        self.observation = self.state_dict["observation"]
-        self.desired_goal = self.state_dict["desired_goal"]
-        self.achieved_goal = self.state_dict["achieved_goal"]
-
-        self.state = self.create_state_from_observation_and_desired_goal(self.observation, self.desired_goal)
-        self.next_state = None
-        self.action = None
-        self.reward = None
-        self.done = False
-
-        self.episode_states = []
-        self.episode_rewards = []
-        self.episode_actions = []
-        self.episode_next_states = []
-        self.episode_dones = []
-
-        self.episode_desired_goals = []
-        self.episode_achieved_goals = []
-        self.episode_observations = []
-
-        self.episode_next_desired_goals = []
-        self.episode_next_achieved_goals = []
-        self.episode_next_observations = []
-
-        self.total_episode_score_so_far = 0
-
-    def track_changeable_goal_episodes_data(self):
-        """Saves the data from the recent episodes in a way compatible with changeable goal environments"""
-        self.episode_rewards.append(self.reward)
-        self.episode_actions.append(self.action)
-        self.episode_dones.append(self.done)
-
-        self.episode_states.append(self.state)
-        self.episode_next_states.append(self.next_state)
-
-        self.episode_desired_goals.append(self.state_dict["desired_goal"])
-        self.episode_achieved_goals.append(self.state_dict["achieved_goal"])
-        self.episode_observations.append(self.state_dict["observation"])
-
-        self.episode_next_desired_goals.append(self.next_state_dict["desired_goal"])
-        self.episode_next_achieved_goals.append(self.next_state_dict["achieved_goal"])
-        self.episode_next_observations.append(self.next_state_dict["observation"])
-
-    def conduct_action_in_changeable_goal_envs(self, action):
-        """Adapts conduct_action from base agent so that can handle changeable goal environments"""
-        self.next_state_dict, self.reward, self.done, _ = self.environment.step(action)
-        self.total_episode_score_so_far += self.reward
-        if self.hyperparameters["clip_rewards"]:
-            self.reward = max(min(self.reward, 1.0), -1.0)
-        self.observation = self.next_state_dict["observation"]
-        self.desired_goal = self.next_state_dict["desired_goal"]
-        self.achieved_goal = self.next_state_dict["achieved_goal"]
-        self.next_state =  self.create_state_from_observation_and_desired_goal(self.observation, self.desired_goal)
-
-
-    def create_state_from_observation_and_desired_goal(self, observation, desired_goal):
-        return np.concatenate((observation, desired_goal))
-
-    def save_alternative_experience(self):
-        """Saves the experiences as if the final state visited in the episode was the goal state"""
-        new_goal = self.achieved_goal
-        new_states = [self.create_state_from_observation_and_desired_goal(observation, new_goal) for observation in self.episode_observations]
-        new_next_states = [self.create_state_from_observation_and_desired_goal(observation, new_goal) for observation in
-                      self.episode_next_observations]
-        new_rewards = [self.environment.compute_reward(next_achieved_goal, new_goal, None) for next_achieved_goal in  self.episode_next_achieved_goals]
-
-        if self.hyperparameters["clip_rewards"]:
-            new_rewards = [max(min(reward, 1.0), -1.0) for reward in new_rewards]
-
-        self.HER_memory.add_experience(new_states, self.episode_actions, new_rewards, new_next_states, self.episode_dones)
-
-    def sample_from_HER_and_Ordinary_Buffer(self):
-        """Samples from the ordinary replay buffer and HER replay buffer according to a proportion specified in config"""
-        states, actions, rewards, next_states, dones = self.memory.sample(self.ordinary_buffer_batch_size)
-        HER_states, HER_actions, HER_rewards, HER_next_states, HER_dones = self.HER_memory.sample(self.HER_buffer_batch_size)
-
-        states = torch.cat((states, HER_states))
-        actions = torch.cat((actions, HER_actions))
-        rewards = torch.cat((rewards, HER_rewards))
-        next_states = torch.cat((next_states, HER_next_states))
-        dones = torch.cat((dones, HER_dones))
-        return states, actions, rewards, next_states, dones
-
-

agents/Trainer.py

@@ -1,304 +0,0 @@
-import copy
-import random
-import pickle
-import os
-import gym
-from gym import wrappers
-import numpy as np
-import matplotlib.pyplot as plt
-
-class Trainer(object):
-    """Runs games for given agents. Optionally will visualise and save the results"""
-    def __init__(self, config, agents):
-        self.config = config
-        self.agents = agents
-        self.agent_to_agent_group = self.create_agent_to_agent_group_dictionary()
-        self.agent_to_color_group = self.create_agent_to_color_dictionary()
-        self.results = None
-        self.signals_result = None
-        self.colors = ["red", "blue", "green", "orange", "yellow", "purple"]
-        self.colour_ix = 0
-        self.y_limits = None
-
-    def create_agent_to_agent_group_dictionary(self):
-        """Creates a dictionary that maps an agent to their wider agent group"""
-        agent_to_agent_group_dictionary = {
-            "DQN": "DQN_Agents",
-            "DQN-HER": "DQN_Agents",
-            "DDQN": "DQN_Agents",
-            "DDQN with Prioritised Replay": "DQN_Agents",
-            "DQN with Fixed Q Targets": "DQN_Agents",
-            "Duelling DQN": "DQN_Agents",
-            "PPO": "Policy_Gradient_Agents",
-            "REINFORCE": "Policy_Gradient_Agents",
-            "Genetic_Agent": "Stochastic_Policy_Search_Agents",
-            "Hill Climbing": "Stochastic_Policy_Search_Agents",
-            "DDPG": "Actor_Critic_Agents",
-            "DDPG-HER": "Actor_Critic_Agents",
-            "TD3": "Actor_Critic_Agents",
-            "A2C": "Actor_Critic_Agents",
-            "A3C": "Actor_Critic_Agents",
-            "h-DQN": "h_DQN",
-            "SNN-HRL": "SNN_HRL",
-            "HIRO": "HIRO",
-            "SAC": "Actor_Critic_Agents",
-            "HRL": "HRL",
-            "Model_HRL": "HRL",
-            "DIAYN": "DIAYN",
-            "Dueling DDQN": "DQN_Agents"
-        }
-        return agent_to_agent_group_dictionary
-
-    def create_agent_to_color_dictionary(self):
-        """Creates a dictionary that maps an agent to a hex color (for plotting purposes)
-        See https://en.wikipedia.org/wiki/Web_colors and https://htmlcolorcodes.com/ for hex colors"""
-        agent_to_color_dictionary = {
-            "DQN": "#0000FF",
-            "DQN with Fixed Q Targets": "#1F618D",
-            "DDQN": "#2980B9",
-            "DDQN with Prioritised Replay": "#7FB3D5",
-            "Dueling DDQN": "#22DAF3",
-            "PPO": "#5B2C6F",
-            "DDPG": "#800000",
-            "DQN-HER": "#008000",
-            "DDPG-HER": "#008000",
-            "TD3": "#E74C3C",
-            "h-DQN": "#D35400",
-            "SNN-HRL": "#800000",
-            "A3C": "#E74C3C",
-            "A2C": "#F1948A",
-            "SAC": "#1C2833",
-            "DIAYN": "#F322CD",
-            "HRL": "#0E0F0F"
-        }
-        return agent_to_color_dictionary
-
-    def run_games_for_agents(self):
-        """Run a set of games for each agent. Optionally visualising and/or saving the results"""
-        self.results = self.create_object_to_store_results()
-        self.signals_result = self.create_object_to_store_results()
-        for agent_number, agent_class in enumerate(self.agents):
-            agent_name = agent_class.agent_name
-            self.run_games_for_agent(agent_number + 1, agent_class)
-            if self.config.visualise_overall_agent_results:
-                agent_rolling_score_results = [results[1] for results in  self.results[agent_name]]
-                self.visualise_overall_agent_results(agent_rolling_score_results, agent_name, show_mean_and_std_range=True, y_limits=self.y_limits)
-        if self.config.file_to_save_data_results: self.save_obj(self.results, self.config.file_to_save_data_results)
-        if self.config.file_to_save_results_graph: plt.savefig(self.config.file_to_save_results_graph, bbox_inches="tight")
-        plt.show()
-        return self.results
-
-    def create_object_to_store_results(self):
-        """Creates a dictionary that we will store the results in if it doesn't exist, otherwise it loads it up"""
-        if self.config.overwrite_existing_results_file or not self.config.file_to_save_data_results or not os.path.isfile(self.config.file_to_save_data_results):
-            results = {}
-        else: results = self.load_obj(self.config.file_to_save_data_results)
-        return results
-
-    def run_games_for_agent(self, agent_number, agent_class):
-        """Runs a set of games for a given agent, saving the results in self.results"""
-        agent_results = []
-        agent_name = agent_class.agent_name
-        agent_group = self.agent_to_agent_group[agent_name]
-        agent_round = 1
-        for run in range(self.config.runs_per_agent):
-            agent_config = copy.deepcopy(self.config)
-
-            if self.environment_has_changeable_goals(agent_config.environment) and self.agent_cant_handle_changeable_goals_without_flattening(agent_name):
-                print("Flattening changeable-goal environment for agent {}".format(agent_name))
-                agent_config.environment = gym.wrappers.FlattenDictWrapper(agent_config.environment,
-                                                                           dict_keys=["observation", "desired_goal"])
-
-            if self.config.randomise_random_seed: agent_config.seed = random.randint(0, 2**32 - 2)
-            agent_config.hyperparameters = agent_config.hyperparameters[agent_group]
-            print("AGENT NAME: {}".format(agent_name))
-            print("\033[1m" + "{}.{}: {}".format(agent_number, agent_round, agent_name) + "\033[0m", flush=True)
-            agent = agent_class(agent_config)
-            self.environment_name = agent.environment_title
-            print(agent.hyperparameters)
-            print("RANDOM SEED " , agent_config.seed)
-            game_scores, rolling_scores, time_taken, game_signals = agent.run_n_episodes()
-            print("Time taken: {}".format(time_taken), flush=True)
-            self.print_two_empty_lines()
-            agent_results.append([game_scores, rolling_scores, len(rolling_scores), -1 * max(rolling_scores), time_taken, game_signals])
-            if self.config.visualise_individual_results:
-                self.visualise_overall_agent_results([rolling_scores], agent_name, show_each_run=True, y_limits=self.y_limits)
-                plt.show()
-            agent_round += 1
-        self.results[agent_name] = agent_results
-
-    def environment_has_changeable_goals(self, env):
-        """Determines whether environment is such that for each episode there is a different goal or not"""
-        return isinstance(env.reset(), dict)
-
-    def agent_cant_handle_changeable_goals_without_flattening(self, agent_name):
-        """Boolean indicating whether the agent is set up to handle changeable goals"""
-        return "HER" not in agent_name
-
-    def visualise_overall_agent_results(self, agent_results, agent_name, show_mean_and_std_range=False, show_each_run=False,
-                                        color=None, ax=None, title=None, y_limits=None):
-        """Visualises the results for one agent"""
-        assert isinstance(agent_results, list), "agent_results must be a list of lists, 1 set of results per list"
-        assert isinstance(agent_results[0], list), "agent_results must be a list of lists, 1 set of results per list"
-        assert bool(show_mean_and_std_range) ^ bool(show_each_run), "either show_mean_and_std_range or show_each_run must be true"
-        if not ax: ax = plt.gca()
-        if not color: color =  self.agent_to_color_group[agent_name]
-        if show_mean_and_std_range:
-            mean_minus_x_std, mean_results, mean_plus_x_std = self.get_mean_and_standard_deviation_difference_results(agent_results)
-            x_vals = list(range(len(mean_results)))
-            ax.plot(x_vals, mean_results, label=agent_name, color=color)
-            ax.plot(x_vals, mean_plus_x_std, color=color, alpha=0.1)
-            ax.plot(x_vals, mean_minus_x_std, color=color, alpha=0.1)
-            ax.fill_between(x_vals, y1=mean_minus_x_std, y2=mean_plus_x_std, alpha=0.1, color=color)
-        else:
-            for ix, result in enumerate(agent_results):
-                x_vals = list(range(len(agent_results[0])))
-                plt.plot(x_vals, result, label=agent_name + "_{}".format(ix+1), color=color)
-                color = self.get_next_color()
-
-        ax.set_facecolor('xkcd:white')
-
-        # Shrink current axis's height by 10% on the bottom
-        box = ax.get_position()
-        ax.set_position([box.x0, box.y0 + box.height * 0.05,
-                         box.width, box.height * 0.95])
-
-        # Put a legend below current axis
-        ax.legend(loc='upper center', bbox_to_anchor=(0.5, -0.15),
-                  fancybox=True, shadow=True, ncol=3)
-
-        if not title: title = self.environment_name
-
-        ax.set_title(title, fontsize=15, fontweight='bold')
-        ax.set_ylabel('Rolling Episode Scores')
-        ax.set_xlabel('Episode Number')
-        self.hide_spines(ax, ['right', 'top'])
-        ax.set_xlim([0, x_vals[-1]])
-
-        if y_limits is None: y_min, y_max = self.get_y_limits(agent_results)
-        else: y_min, y_max = y_limits
-
-        ax.set_ylim([y_min, y_max])
-
-        if self.config.show_solution_score:
-            self.draw_horizontal_line_with_label(ax, y_value=self.config.environment.get_score_to_win(), x_min=0,
-                                        x_max=self.config.num_episodes_to_run * 1.02, label="Target \n score")
-
-    def get_y_limits(self, results):
-        """Extracts the minimum and maximum seen y_values from a set of results"""
-        min_result = float("inf")
-        max_result = float("-inf")
-        for result in results:
-            temp_max = np.max(result)
-            temp_min = np.min(result)
-            if temp_max > max_result:
-                max_result = temp_max
-            if temp_min < min_result:
-                min_result = temp_min
-        return min_result, max_result
-
-    def get_next_color(self):
-        """Gets the next color in list self.colors. If it gets to the end then it starts from beginning"""
-        self.colour_ix += 1
-        if self.colour_ix >= len(self.colors): self.colour_ix = 0
-        color = self.colors[self.colour_ix]
-        return color
-
-    def get_mean_and_standard_deviation_difference_results(self, results):
-        """From a list of lists of agent results it extracts the mean results and the mean results plus or minus
-         some multiple of the standard deviation"""
-        def get_results_at_a_time_step(results, timestep):
-            results_at_a_time_step = [result[timestep] for result in results]
-            return results_at_a_time_step
-        def get_standard_deviation_at_time_step(results, timestep):
-            results_at_a_time_step = [result[timestep] for result in results]
-            return np.std(results_at_a_time_step)
-        mean_results = [np.mean(get_results_at_a_time_step(results, timestep)) for timestep in range(len(results[0]))]
-        mean_minus_x_std = [mean_val - self.config.standard_deviation_results * get_standard_deviation_at_time_step(results, timestep) for
-                            timestep, mean_val in enumerate(mean_results)]
-        mean_plus_x_std = [mean_val + self.config.standard_deviation_results * get_standard_deviation_at_time_step(results, timestep) for
-                           timestep, mean_val in enumerate(mean_results)]
-        return mean_minus_x_std, mean_results, mean_plus_x_std
-
-    def hide_spines(self, ax, spines_to_hide):
-        """Hides splines on a matplotlib image"""
-        for spine in spines_to_hide:
-            ax.spines[spine].set_visible(False)
-
-    def ignore_points_after_game_solved(self, mean_minus_x_std, mean_results, mean_plus_x_std):
-        """Removes the datapoints after the mean result achieves the score required to solve the game"""
-        for ix in range(len(mean_results)):
-            if mean_results[ix] >= self.config.environment.get_score_to_win():
-                break
-        return mean_minus_x_std[:ix], mean_results[:ix], mean_plus_x_std[:ix]
-
-    def draw_horizontal_line_with_label(self, ax, y_value, x_min, x_max, label):
-        """Draws a dotted horizontal line on the given image at the given point and with the given label"""
-        ax.hlines(y=y_value, xmin=x_min, xmax=x_max,
-                  linewidth=2, color='k', linestyles='dotted', alpha=0.5)
-        ax.text(x_max, y_value * 0.965, label)
-
-    def print_two_empty_lines(self):
-        print("-----------------------------------------------------------------------------------")
-        print("-----------------------------------------------------------------------------------")
-        print(" ")
-
-    def save_obj(self, obj, name):
-        """Saves given object as a pickle file"""
-        if name[-4:] != ".pkl":
-            name += ".pkl"
-        with open(name, 'wb') as f:
-            pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)
-
-    def load_obj(self, name):
-        """Loads a pickle file object"""
-        with open(name, 'rb') as f:
-            return pickle.load(f)
-
-    def visualise_preexisting_results(self, save_image_path=None, data_path=None, colors=None, show_image=True, ax=None,
-                                      title=None, y_limits=None):
-        """Visualises saved data results and then optionally saves the image"""
-        if not data_path: preexisting_results = self.create_object_to_store_results()
-        else: preexisting_results = self.load_obj(data_path)
-        for ix, agent in enumerate(list(preexisting_results.keys())):
-            agent_rolling_score_results = [results[1] for results in preexisting_results[agent]]
-            if colors: color = colors[ix]
-            else: color = None
-            self.visualise_overall_agent_results(agent_rolling_score_results, agent, show_mean_and_std_range=True,
-                                                 color=color, ax=ax, title=title, y_limits=y_limits)
-        if save_image_path: plt.savefig(save_image_path, bbox_inches="tight")
-        if show_image: plt.show()
-
-    def visualise_set_of_preexisting_results(self, results_data_paths, save_image_path=None, show_image=True, plot_titles=None,
-                                             y_limits=[None,None]):
-        """Visualises a set of preexisting results on 1 plot by making subplots"""
-        assert isinstance(results_data_paths, list), "all_results must be a list of data paths"
-
-        num_figures = len(results_data_paths)
-        col_width = 15
-        row_height = 6
-
-        if num_figures <= 2:
-            fig, axes = plt.subplots(1, num_figures, figsize=(col_width, row_height ))
-        elif num_figures <= 4:
-            fig, axes = plt.subplots(2, num_figures, figsize=(row_height, col_width))
-        else:
-            raise ValueError("Need to tell this method how to deal with more than 4 plots")
-        for ax_ix in range(len(results_data_paths)):
-            self.visualise_preexisting_results(show_image=False, data_path=results_data_paths[ax_ix], ax=axes[ax_ix],
-                                               title=plot_titles[ax_ix], y_limits=y_limits[ax_ix])
-        fig.tight_layout()
-        fig.subplots_adjust(bottom=0.25)
-
-        if save_image_path: plt.savefig(save_image_path) #, bbox_inches="tight")
-        if show_image: plt.show()
-
-        # ax.imshow(z, aspect="auto")
-
-
-
-
-
-
-
-

agents/__init__.py

@@ -1 +0,0 @@
-import os, sys; sys.path.append(os.path.dirname(os.path.realpath(__file__)))

agents/actor_critic_agents/A2C.py

@@ -1,25 +0,0 @@
-from agents.actor_critic_agents.A3C import A3C
-
-class A2C(A3C):
-    """Synchronous version of A2C algorithm from deepmind paper https://arxiv.org/pdf/1602.01783.pdf. The only
-    difference between this and the A3C is that gradient updates get done in a batch rather than 1 by 1 as the gradients
-    come in"""
-    agent_name = "A2C"
-    def __init__(self, config):
-        super(A2C, self).__init__(config)
-
-    def update_shared_model(self, gradient_updates_queue):
-        """Worker that updates the shared model with gradients as they get put into the queue"""
-        while True:
-            gradients_seen = 0
-            while gradients_seen < self.worker_processes:
-                if gradients_seen == 0:
-                    gradients = gradient_updates_queue.get()
-                else:
-                    new_grads = gradient_updates_queue.get()
-                    gradients = [grad + new_grad for grad, new_grad in zip(gradients, new_grads)]
-                gradients_seen += 1
-            self.actor_critic_optimizer.zero_grad()
-            for grads, params in zip(gradients, self.actor_critic.parameters()):
-                params._grad = grads
-            self.actor_critic_optimizer.step()

agents/actor_critic_agents/A3C.py

@@ -1,229 +0,0 @@
-import copy
-import random
-import time
-import numpy as np
-import torch
-from torch import multiprocessing
-from torch.multiprocessing import Queue
-from torch.optim import Adam
-from agents.Base_Agent import Base_Agent
-from utilities.Utility_Functions import create_actor_distribution, SharedAdam
-
-class A3C(Base_Agent):
-    """Actor critic A3C algorithm from deepmind paper https://arxiv.org/pdf/1602.01783.pdf"""
-    agent_name = "A3C"
-    def __init__(self, config):
-        super(A3C, self).__init__(config)
-        self.num_processes = multiprocessing.cpu_count()
-        self.worker_processes = max(1, self.num_processes - 2)
-        self.actor_critic = self.create_NN(input_dim=self.state_size, output_dim=[self.action_size, 1])
-        self.actor_critic_optimizer = SharedAdam(self.actor_critic.parameters(), lr=self.hyperparameters["learning_rate"], eps=1e-4)
-
-    def run_n_episodes(self):
-        """Runs game to completion n times and then summarises results and saves model (if asked to)"""
-        start = time.time()
-        results_queue = Queue()
-        gradient_updates_queue = Queue()
-        episode_number = multiprocessing.Value('i', 0)
-        self.optimizer_lock = multiprocessing.Lock()
-        episodes_per_process = int(self.config.num_episodes_to_run / self.worker_processes) + 1
-        processes = []
-        self.actor_critic.share_memory()
-        self.actor_critic_optimizer.share_memory()
-
-        optimizer_worker = multiprocessing.Process(target=self.update_shared_model, args=(gradient_updates_queue,))
-        optimizer_worker.start(,
-
-        for process_num in range(self.worker_processes):
-            worker = Actor_Critic_Worker(process_num, copy.deepcopy(self.environment), self.actor_critic, episode_number, self.optimizer_lock,
-                                    self.actor_critic_optimizer, self.config, episodes_per_process,
-                                    self.hyperparameters["epsilon_decay_rate_denominator"],
-                                    self.action_size, self.action_types,
-                                    results_queue, copy.deepcopy(self.actor_critic), gradient_updates_queue)
-            worker.start()
-            processes.append(worker)
-        self.print_results(episode_number, results_queue)
-        for worker in processes:
-            worker.join()
-        optimizer_worker.kill()
-
-        time_taken = time.time() - start
-        return self.game_full_episode_scores, self.rolling_results, time_taken
-
-    def print_results(self, episode_number, results_queue):
-        """Worker that prints out results as they get put into a queue"""
-        while True:
-            with episode_number.get_lock():
-                carry_on = episode_number.value < self.config.num_episodes_to_run
-            if carry_on:
-                if not results_queue.empty():
-                    self.total_episode_score_so_far = results_queue.get()
-                    self.save_and_print_result()
-            else: break
-
-    def update_shared_model(self, gradient_updates_queue):
-        """Worker that updates the shared model with gradients as they get put into the queue"""
-        while True:
-            gradients = gradient_updates_queue.get()
-            with self.optimizer_lock:
-                self.actor_critic_optimizer.zero_grad()
-                for grads, params in zip(gradients, self.actor_critic.parameters()):
-                    params._grad = grads  # maybe need to do grads.clone()
-                self.actor_critic_optimizer.step()
-
-class Actor_Critic_Worker(torch.multiprocessing.Process):
-    """Actor critic worker that will play the game for the designated number of episodes """
-    def __init__(self, worker_num, environment, shared_model, counter, optimizer_lock, shared_optimizer,
-                 config, episodes_to_run, epsilon_decay_denominator, action_size, action_types, results_queue,
-                 local_model, gradient_updates_queue):
-        super(Actor_Critic_Worker, self).__init__()
-        self.environment = environment
-        self.config = config
-        self.worker_num = worker_num
-
-        self.gradient_clipping_norm = self.config.hyperparameters["gradient_clipping_norm"]
-        self.discount_rate = self.config.hyperparameters["discount_rate"]
-        self.normalise_rewards = self.config.hyperparameters["normalise_rewards"]
-
-        self.action_size = action_size
-        self.set_seeds(self.worker_num)
-        self.shared_model = shared_model
-        self.local_model = local_model
-        self.local_optimizer = Adam(self.local_model.parameters(), lr=0.0, eps=1e-4)
-        self.counter = counter
-        self.optimizer_lock = optimizer_lock
-        self.shared_optimizer = shared_optimizer
-        self.episodes_to_run = episodes_to_run
-        self.epsilon_decay_denominator = epsilon_decay_denominator
-        self.exploration_worker_difference = self.config.hyperparameters["exploration_worker_difference"]
-        self.action_types = action_types
-        self.results_queue = results_queue
-        self.episode_number = 0
-
-        self.gradient_updates_queue = gradient_updates_queue
-
-    def set_seeds(self, worker_num):
-        """Sets random seeds for this worker"""
-        torch.manual_seed(self.config.seed + worker_num)
-        self.environment.seed(self.config.seed + worker_num)
-
-    def run(self):
-        """Starts the worker"""
-        torch.set_num_threads(1)
-        for ep_ix in range(self.episodes_to_run):
-            with self.optimizer_lock:
-                Base_Agent.copy_model_over(self.shared_model, self.local_model)
-            epsilon_exploration = self.calculate_new_exploration()
-            state = self.reset_game_for_worker()
-            done = False
-            self.episode_states = []
-            self.episode_actions = []
-            self.episode_rewards = []
-            self.episode_log_action_probabilities = []
-            self.critic_outputs = []
-
-            while not done:
-                action, action_log_prob, critic_outputs = self.pick_action_and_get_critic_values(self.local_model, state, epsilon_exploration)
-                next_state, reward, done, _ =  self.environment.step(action)
-                self.episode_states.append(state)
-                self.episode_actions.append(action)
-                self.episode_rewards.append(reward)
-                self.episode_log_action_probabilities.append(action_log_prob)
-                self.critic_outputs.append(critic_outputs)
-                state = next_state
-
-            total_loss = self.calculate_total_loss()
-            self.put_gradients_in_queue(total_loss)
-            self.episode_number += 1
-            with self.counter.get_lock():
-                self.counter.value += 1
-                self.results_queue.put(np.sum(self.episode_rewards))
-
-    def calculate_new_exploration(self):
-        """Calculates the new exploration parameter epsilon. It picks a random point within 3X above and below the
-        current epsilon"""
-        with self.counter.get_lock():
-            epsilon = 1.0 / (1.0 + (self.counter.value / self.epsilon_decay_denominator))
-        epsilon = max(0.0, random.uniform(epsilon / self.exploration_worker_difference, epsilon * self.exploration_worker_difference))
-        return epsilon
-
-    def reset_game_for_worker(self):
-        """Resets the game environment so it is ready to play a new episode"""
-        state = self.environment.reset()
-        if self.action_types == "CONTINUOUS": self.noise.reset()
-        return state
-
-    def pick_action_and_get_critic_values(self, policy, state, epsilon_exploration=None):
-        """Picks an action using the policy"""
-        state = torch.from_numpy(state).float().unsqueeze(0)
-        model_output = policy.forward(state)
-        actor_output = model_output[:, list(range(self.action_size))] #we only use first set of columns to decide action, last column is state-value
-        critic_output = model_output[:, -1]
-        action_distribution = create_actor_distribution(self.action_types, actor_output, self.action_size)
-        action = action_distribution.sample().cpu().numpy()
-        if self.action_types == "CONTINUOUS": action += self.noise.sample()
-        if self.action_types == "DISCRETE":
-            if random.random() <= epsilon_exploration:
-                action = random.randint(0, self.action_size - 1)
-            else:
-                action = action[0]
-        action_log_prob = self.calculate_log_action_probability(action, action_distribution)
-        return action, action_log_prob, critic_output
-
-    def calculate_log_action_probability(self, actions, action_distribution):
-        """Calculates the log probability of the chosen action"""
-        policy_distribution_log_prob = action_distribution.log_prob(torch.Tensor([actions]))
-        return policy_distribution_log_prob
-
-    def calculate_total_loss(self):
-        """Calculates the actor loss + critic loss"""
-        discounted_returns = self.calculate_discounted_returns()
-        if self.normalise_rewards:
-            discounted_returns = self.normalise_discounted_returns(discounted_returns)
-        critic_loss, advantages = self.calculate_critic_loss_and_advantages(discounted_returns)
-        actor_loss = self.calculate_actor_loss(advantages)
-        total_loss = actor_loss + critic_loss
-        return total_loss
-
-    def calculate_discounted_returns(self):
-        """Calculates the cumulative discounted return for an episode which we will then use in a learning iteration"""
-        discounted_returns = [0]
-        for ix in range(len(self.episode_states)):
-            return_value = self.episode_rewards[-(ix + 1)] + self.discount_rate*discounted_returns[-1]
-            discounted_returns.append(return_value)
-        discounted_returns = discounted_returns[1:]
-        discounted_returns = discounted_returns[::-1]
-        return discounted_returns
-
-    def normalise_discounted_returns(self, discounted_returns):
-        """Normalises the discounted returns by dividing by mean and std of returns that episode"""
-        mean = np.mean(discounted_returns)
-        std = np.std(discounted_returns)
-        discounted_returns -= mean
-        discounted_returns /= (std + 1e-5)
-        return discounted_returns
-
-    def calculate_critic_loss_and_advantages(self, all_discounted_returns):
-        """Calculates the critic's loss and the advantages"""
-        critic_values = torch.cat(self.critic_outputs)
-        advantages = torch.Tensor(all_discounted_returns) - critic_values
-        advantages = advantages.detach()
-        critic_loss =  (torch.Tensor(all_discounted_returns) - critic_values)**2
-        critic_loss = critic_loss.mean()
-        return critic_loss, advantages
-
-    def calculate_actor_loss(self, advantages):
-        """Calculates the loss for the actor"""
-        action_log_probabilities_for_all_episodes = torch.cat(self.episode_log_action_probabilities)
-        actor_loss = -1.0 * action_log_probabilities_for_all_episodes * advantages
-        actor_loss = actor_loss.mean()
-        return actor_loss
-
-    def put_gradients_in_queue(self, total_loss):
-        """Puts gradients in a queue for the optimisation process to use to update the shared model"""
-        self.local_optimizer.zero_grad()
-        total_loss.backward()
-        torch.nn.utils.clip_grad_norm_(self.local_model.parameters(), self.gradient_clipping_norm)
-        gradients = [param.grad.clone() for param in self.local_model.parameters()]
-        self.gradient_updates_queue.put(gradients)
-

agents/actor_critic_agents/DDPG.py

@@ -1,115 +0,0 @@
-import torch
-import torch.nn.functional as functional
-from torch import optim
-from agents.Base_Agent import Base_Agent
-from utilities.data_structures.Replay_Buffer import Replay_Buffer
-from exploration_strategies.OU_Noise_Exploration import OU_Noise_Exploration
-
-class DDPG(Base_Agent):
-    """A DDPG Agent"""
-    agent_name = "DDPG"
-
-    def __init__(self, config):
-        Base_Agent.__init__(self, config)
-        self.hyperparameters = config.hyperparameters
-        self.critic_local = self.create_NN(input_dim=self.state_size + self.action_size, output_dim=1, key_to_use="Critic")
-        self.critic_target = self.create_NN(input_dim=self.state_size + self.action_size, output_dim=1, key_to_use="Critic")
-        Base_Agent.copy_model_over(self.critic_local, self.critic_target)
-
-        self.critic_optimizer = optim.Adam(self.critic_local.parameters(),
-                                           lr=self.hyperparameters["Critic"]["learning_rate"], eps=1e-4)
-        self.memory = Replay_Buffer(self.hyperparameters["Critic"]["buffer_size"], self.hyperparameters["batch_size"],
-                                    self.config.seed)
-        self.actor_local = self.create_NN(input_dim=self.state_size, output_dim=self.action_size, key_to_use="Actor")
-        self.actor_target = self.create_NN(input_dim=self.state_size, output_dim=self.action_size, key_to_use="Actor")
-        Base_Agent.copy_model_over(self.actor_local, self.actor_target)
-
-        self.actor_optimizer = optim.Adam(self.actor_local.parameters(),
-                                          lr=self.hyperparameters["Actor"]["learning_rate"], eps=1e-4)
-        self.exploration_strategy = OU_Noise_Exploration(self.config)
-
-    def step(self):
-        """Runs a step in the game"""
-        while not self.done:
-            # print("State ", self.state.shape)
-            self.action = self.pick_action()
-            self.conduct_action(self.action)
-            if self.time_for_critic_and_actor_to_learn():
-                for _ in range(self.hyperparameters["learning_updates_per_learning_session"]):
-                    states, actions, rewards, next_states, dones = self.sample_experiences()
-                    self.critic_learn(states, actions, rewards, next_states, dones)
-                    self.actor_learn(states)
-            self.save_experience()
-            self.state = self.next_state #this is to set the state for the next iteration
-            self.global_step_number += 1
-        self.episode_number += 1
-
-    def sample_experiences(self):
-        return self.memory.sample()
-
-    def pick_action(self, state=None):
-        """Picks an action using the actor network and then adds some noise to it to ensure exploration"""
-        if state is None: state = torch.from_numpy(self.state).float().unsqueeze(0).to(self.device)
-        self.actor_local.eval()
-        with torch.no_grad():
-            action = self.actor_local(state).cpu().data.numpy()
-        self.actor_local.train()
-        action = self.exploration_strategy.perturb_action_for_exploration_purposes({"action": action})
-        return action.squeeze(0)
-
-    def critic_learn(self, states, actions, rewards, next_states, dones):
-        """Runs a learning iteration for the critic"""
-        loss = self.compute_loss(states, next_states, rewards, actions, dones)
-        self.take_optimisation_step(self.critic_optimizer, self.critic_local, loss, self.hyperparameters["Critic"]["gradient_clipping_norm"])
-        self.soft_update_of_target_network(self.critic_local, self.critic_target, self.hyperparameters["Critic"]["tau"])
-
-    def compute_loss(self, states, next_states, rewards, actions, dones):
-        """Computes the loss for the critic"""
-        with torch.no_grad():
-            critic_targets = self.compute_critic_targets(next_states, rewards, dones)
-        critic_expected = self.compute_expected_critic_values(states, actions)
-        loss = functional.mse_loss(critic_expected, critic_targets)
-        return loss
-
-    def compute_critic_targets(self, next_states, rewards, dones):
-        """Computes the critic target values to be used in the loss for the critic"""
-        critic_targets_next = self.compute_critic_values_for_next_states(next_states)
-        critic_targets = self.compute_critic_values_for_current_states(rewards, critic_targets_next, dones)
-        return critic_targets
-
-    def compute_critic_values_for_next_states(self, next_states):
-        """Computes the critic values for next states to be used in the loss for the critic"""
-        with torch.no_grad():
-            actions_next = self.actor_target(next_states)
-            critic_targets_next = self.critic_target(torch.cat((next_states, actions_next), 1))
-        return critic_targets_next
-
-    def compute_critic_values_for_current_states(self, rewards, critic_targets_next, dones):
-        """Computes the critic values for current states to be used in the loss for the critic"""
-        critic_targets_current = rewards + (self.hyperparameters["discount_rate"] * critic_targets_next * (1.0 - dones))
-        return critic_targets_current
-
-    def compute_expected_critic_values(self, states, actions):
-        """Computes the expected critic values to be used in the loss for the critic"""
-        critic_expected = self.critic_local(torch.cat((states, actions), 1))
-        return critic_expected
-
-    def time_for_critic_and_actor_to_learn(self):
-        """Returns boolean indicating whether there are enough experiences to learn from and it is time to learn for the
-        actor and critic"""
-        return self.enough_experiences_to_learn_from() and self.global_step_number % self.hyperparameters["update_every_n_steps"] == 0
-
-    def actor_learn(self, states):
-        """Runs a learning iteration for the actor"""
-        if self.done: #we only update the learning rate at end of each episode
-            self.update_learning_rate(self.hyperparameters["Actor"]["learning_rate"], self.actor_optimizer)
-        actor_loss = self.calculate_actor_loss(states)
-        self.take_optimisation_step(self.actor_optimizer, self.actor_local, actor_loss,
-                                    self.hyperparameters["Actor"]["gradient_clipping_norm"])
-        self.soft_update_of_target_network(self.actor_local, self.actor_target, self.hyperparameters["Actor"]["tau"])
-
-    def calculate_actor_loss(self, states):
-        """Calculates the loss for the actor"""
-        actions_pred = self.actor_local(states)
-        actor_loss = -self.critic_local(torch.cat((states, actions_pred), 1)).mean()
-        return actor_loss

agents/actor_critic_agents/DDPG_HER.py

@@ -1,38 +0,0 @@
-from agents.actor_critic_agents.DDPG import DDPG
-from agents.HER_Base import HER_Base
-
-class DDPG_HER(HER_Base, DDPG):
-    """DDPG algorithm with hindsight experience replay"""
-    agent_name = "DDPG-HER"
-
-    def __init__(self, config):
-        DDPG.__init__(self, config)
-        HER_Base.__init__(self, self.hyperparameters["Critic"]["buffer_size"], self.hyperparameters["batch_size"],
-                          self.hyperparameters["HER_sample_proportion"])
-
-    def step(self):
-        """Runs a step within a game including a learning step if required"""
-        while not self.done:
-            self.action = self.pick_action()
-            self.conduct_action_in_changeable_goal_envs(self.action)
-            if self.time_for_critic_and_actor_to_learn():
-                for _ in range(self.hyperparameters["learning_updates_per_learning_session"]):
-                    states, actions, rewards, next_states, dones = self.sample_from_HER_and_Ordinary_Buffer()  # Samples experiences from buffer
-                    self.critic_learn(states, actions, rewards, next_states, dones)
-                    self.actor_learn(states)
-            self.track_changeable_goal_episodes_data()
-            self.save_experience()
-            if self.done: self.save_alternative_experience()
-            self.state_dict = self.next_state_dict  # this is to set the state for the next iteration
-            self.state = self.next_state
-            self.global_step_number += 1
-        self.episode_number += 1
-
-    def enough_experiences_to_learn_from(self):
-        """Returns boolean indicating whether there are enough experiences to learn from and it is time to learn"""
-        return len(self.memory) > self.ordinary_buffer_batch_size and len(self.HER_memory) > self.HER_buffer_batch_size
-
-
-
-
-

agents/actor_critic_agents/SAC.py

@@ -1,211 +0,0 @@
-from agents.Base_Agent import Base_Agent
-from utilities.OU_Noise import OU_Noise
-from utilities.data_structures.Replay_Buffer import Replay_Buffer
-from torch.optim import Adam
-import torch
-import torch.nn.functional as F
-from torch.distributions import Normal
-import numpy as np
-
-LOG_SIG_MAX = 2
-LOG_SIG_MIN = -20
-TRAINING_EPISODES_PER_EVAL_EPISODE = 10
-EPSILON = 1e-6
-
-class SAC(Base_Agent):
-    """Soft Actor-Critic model based on the 2018 paper https://arxiv.org/abs/1812.05905 and on this github implementation
-      https://github.com/pranz24/pytorch-soft-actor-critic. It is an actor-critic algorithm where the agent is also trained
-      to maximise the entropy of their actions as well as their cumulative reward"""
-    agent_name = "SAC"
-    def __init__(self, config):
-        Base_Agent.__init__(self, config)
-        assert self.action_types == "CONTINUOUS", "Action types must be continuous. Use SAC Discrete instead for discrete actions"
-        assert self.config.hyperparameters["Actor"]["final_layer_activation"] != "Softmax", "Final actor layer must not be softmax"
-        self.hyperparameters = config.hyperparameters
-        self.critic_local = self.create_NN(input_dim=self.state_size + self.action_size, output_dim=1, key_to_use="Critic")
-        self.critic_local_2 = self.create_NN(input_dim=self.state_size + self.action_size, output_dim=1,
-                                           key_to_use="Critic", override_seed=self.config.seed + 1)
-        self.critic_optimizer = torch.optim.Adam(self.critic_local.parameters(),
-                                                 lr=self.hyperparameters["Critic"]["learning_rate"], eps=1e-4)
-        self.critic_optimizer_2 = torch.optim.Adam(self.critic_local_2.parameters(),
-                                                   lr=self.hyperparameters["Critic"]["learning_rate"], eps=1e-4)
-        self.critic_target = self.create_NN(input_dim=self.state_size + self.action_size, output_dim=1,
-                                           key_to_use="Critic")
-        self.critic_target_2 = self.create_NN(input_dim=self.state_size + self.action_size, output_dim=1,
-                                            key_to_use="Critic")
-        Base_Agent.copy_model_over(self.critic_local, self.critic_target)
-        Base_Agent.copy_model_over(self.critic_local_2, self.critic_target_2)
-        self.memory = Replay_Buffer(self.hyperparameters["Critic"]["buffer_size"], self.hyperparameters["batch_size"],
-                                    self.config.seed)
-        self.actor_local = self.create_NN(input_dim=self.state_size, output_dim=self.action_size * 2, key_to_use="Actor")
-        self.actor_optimizer = torch.optim.Adam(self.actor_local.parameters(),
-                                          lr=self.hyperparameters["Actor"]["learning_rate"], eps=1e-4)
-        self.automatic_entropy_tuning = self.hyperparameters["automatically_tune_entropy_hyperparameter"]
-        if self.automatic_entropy_tuning:
-            self.target_entropy = -torch.prod(torch.Tensor(self.environment.action_space.shape).to(self.device)).item() # heuristic value from the paper
-            self.log_alpha = torch.zeros(1, requires_grad=True, device=self.device)
-            self.alpha = self.log_alpha.exp()
-            self.alpha_optim = Adam([self.log_alpha], lr=self.hyperparameters["Actor"]["learning_rate"], eps=1e-4)
-        else:
-            self.alpha = self.hyperparameters["entropy_term_weight"]
-
-        self.add_extra_noise = self.hyperparameters["add_extra_noise"]
-        if self.add_extra_noise:
-            self.noise = OU_Noise(self.action_size, self.config.seed, self.hyperparameters["mu"],
-                                  self.hyperparameters["theta"], self.hyperparameters["sigma"])
-
-        self.do_evaluation_iterations = self.hyperparameters["do_evaluation_iterations"]
-
-    def save_result(self):
-        """Saves the result of an episode of the game. Overriding the method in Base Agent that does this because we only
-        want to keep track of the results during the evaluation episodes"""
-        if self.episode_number == 1 or not self.do_evaluation_iterations:
-            self.game_full_episode_scores.extend([self.total_episode_score_so_far])
-            self.rolling_results.append(np.mean(self.game_full_episode_scores[-1 * self.rolling_score_window:]))
-            self.save_max_result_seen()
-
-        elif (self.episode_number - 1) % TRAINING_EPISODES_PER_EVAL_EPISODE == 0:
-            self.game_full_episode_scores.extend([self.total_episode_score_so_far for _ in range(TRAINING_EPISODES_PER_EVAL_EPISODE)])
-            self.rolling_results.extend([np.mean(self.game_full_episode_scores[-1 * self.rolling_score_window:]) for _ in range(TRAINING_EPISODES_PER_EVAL_EPISODE)])
-            self.save_max_result_seen()
-
-    def reset_game(self):
-        """Resets the game information so we are ready to play a new episode"""
-        Base_Agent.reset_game(self)
-        if self.add_extra_noise: self.noise.reset()
-
-    def step(self):
-        """Runs an episode on the game, saving the experience and running a learning step if appropriate"""
-        eval_ep = self.episode_number % TRAINING_EPISODES_PER_EVAL_EPISODE == 0 and self.do_evaluation_iterations
-        self.episode_step_number_val = 0
-        while not self.done:
-            self.episode_step_number_val += 1
-            self.action = self.pick_action(eval_ep)
-            self.conduct_action(self.action)
-            if self.time_for_critic_and_actor_to_learn():
-                for _ in range(self.hyperparameters["learning_updates_per_learning_session"]):
-                    self.learn()
-            mask = False if self.episode_step_number_val >= self.environment._max_episode_steps else self.done
-            if not eval_ep: self.save_experience(experience=(self.state, self.action, self.reward, self.next_state, mask))
-            self.state = self.next_state
-            self.global_step_number += 1
-        print(self.total_episode_score_so_far)
-        if eval_ep: self.print_summary_of_latest_evaluation_episode()
-        self.episode_number += 1
-
-    def pick_action(self, eval_ep, state=None):
-        """Picks an action using one of three methods: 1) Randomly if we haven't passed a certain number of steps,
-         2) Using the actor in evaluation mode if eval_ep is True  3) Using the actor in training mode if eval_ep is False.
-         The difference between evaluation and training mode is that training mode does more exploration"""
-        if state is None: state = self.state
-        if eval_ep: action = self.actor_pick_action(state=state, eval=True)
-        elif self.global_step_number < self.hyperparameters["min_steps_before_learning"]:
-            action = self.environment.action_space.sample()
-            print("Picking random action ", action)
-        else: action = self.actor_pick_action(state=state)
-        if self.add_extra_noise:
-            action += self.noise.sample()
-        return action
-
-    def actor_pick_action(self, state=None, eval=False):
-        """Uses actor to pick an action in one of two ways: 1) If eval = False and we aren't in eval mode then it picks
-        an action that has partly been randomly sampled 2) If eval = True then we pick the action that comes directly
-        from the network and so did not involve any random sampling"""
-        if state is None: state = self.state
-        state = torch.FloatTensor([state]).to(self.device)
-        if len(state.shape) == 1: state = state.unsqueeze(0)
-        if eval == False: action, _, _ = self.produce_action_and_action_info(state)
-        else:
-            with torch.no_grad():
-                _, z, action = self.produce_action_and_action_info(state)
-        action = action.detach().cpu().numpy()
-        return action[0]
-
-    def produce_action_and_action_info(self, state):
-        """Given the state, produces an action, the log probability of the action, and the tanh of the mean action"""
-        actor_output = self.actor_local(state)
-        mean, log_std = actor_output[:, :self.action_size], actor_output[:, self.action_size:]
-        std = log_std.exp()
-        normal = Normal(mean, std)
-        x_t = normal.rsample()  #rsample means it is sampled using reparameterisation trick
-        action = torch.tanh(x_t)
-        log_prob = normal.log_prob(x_t)
-        log_prob -= torch.log(1 - action.pow(2) + EPSILON)
-        log_prob = log_prob.sum(1, keepdim=True)
-        return action, log_prob, torch.tanh(mean)
-
-    def time_for_critic_and_actor_to_learn(self):
-        """Returns boolean indicating whether there are enough experiences to learn from and it is time to learn for the
-        actor and critic"""
-        return self.global_step_number > self.hyperparameters["min_steps_before_learning"] and \
-               self.enough_experiences_to_learn_from() and self.global_step_number % self.hyperparameters["update_every_n_steps"] == 0
-
-    def learn(self):
-        """Runs a learning iteration for the actor, both critics and (if specified) the temperature parameter"""
-        state_batch, action_batch, reward_batch, next_state_batch, mask_batch = self.sample_experiences()
-        qf1_loss, qf2_loss = self.calculate_critic_losses(state_batch, action_batch, reward_batch, next_state_batch, mask_batch)
-        self.update_critic_parameters(qf1_loss, qf2_loss)
-
-        policy_loss, log_pi = self.calculate_actor_loss(state_batch)
-        if self.automatic_entropy_tuning: alpha_loss = self.calculate_entropy_tuning_loss(log_pi)
-        else: alpha_loss = None
-        self.update_actor_parameters(policy_loss, alpha_loss)
-
-    def sample_experiences(self):
-        return  self.memory.sample()
-
-    def calculate_critic_losses(self, state_batch, action_batch, reward_batch, next_state_batch, mask_batch):
-        """Calculates the losses for the two critics. This is the ordinary Q-learning loss except the additional entropy
-         term is taken into account"""
-        with torch.no_grad():
-            next_state_action, next_state_log_pi, _ = self.produce_action_and_action_info(next_state_batch)
-            qf1_next_target = self.critic_target(torch.cat((next_state_batch, next_state_action), 1))
-            qf2_next_target = self.critic_target_2(torch.cat((next_state_batch, next_state_action), 1))
-            min_qf_next_target = torch.min(qf1_next_target, qf2_next_target) - self.alpha * next_state_log_pi
-            next_q_value = reward_batch + (1.0 - mask_batch) * self.hyperparameters["discount_rate"] * (min_qf_next_target)
-        qf1 = self.critic_local(torch.cat((state_batch, action_batch), 1))
-        qf2 = self.critic_local_2(torch.cat((state_batch, action_batch), 1))
-        qf1_loss = F.mse_loss(qf1, next_q_value)
-        qf2_loss = F.mse_loss(qf2, next_q_value)
-        return qf1_loss, qf2_loss
-
-    def calculate_actor_loss(self, state_batch):
-        """Calculates the loss for the actor. This loss includes the additional entropy term"""
-        action, log_pi, _ = self.produce_action_and_action_info(state_batch)
-        qf1_pi = self.critic_local(torch.cat((state_batch, action), 1))
-        qf2_pi = self.critic_local_2(torch.cat((state_batch, action), 1))
-        min_qf_pi = torch.min(qf1_pi, qf2_pi)
-        policy_loss = ((self.alpha * log_pi) - min_qf_pi).mean()
-        return policy_loss, log_pi
-
-    def calculate_entropy_tuning_loss(self, log_pi):
-        """Calculates the loss for the entropy temperature parameter. This is only relevant if self.automatic_entropy_tuning
-        is True."""
-        alpha_loss = -(self.log_alpha * (log_pi + self.target_entropy).detach()).mean()
-        return alpha_loss
-
-    def update_critic_parameters(self, critic_loss_1, critic_loss_2):
-        """Updates the parameters for both critics"""
-        self.take_optimisation_step(self.critic_optimizer, self.critic_local, critic_loss_1,
-                                    self.hyperparameters["Critic"]["gradient_clipping_norm"])
-        self.take_optimisation_step(self.critic_optimizer_2, self.critic_local_2, critic_loss_2,
-                                    self.hyperparameters["Critic"]["gradient_clipping_norm"])
-        self.soft_update_of_target_network(self.critic_local, self.critic_target,
-                                           self.hyperparameters["Critic"]["tau"])
-        self.soft_update_of_target_network(self.critic_local_2, self.critic_target_2,
-                                           self.hyperparameters["Critic"]["tau"])
-
-    def update_actor_parameters(self, actor_loss, alpha_loss):
-        """Updates the parameters for the actor and (if specified) the temperature parameter"""
-        self.take_optimisation_step(self.actor_optimizer, self.actor_local, actor_loss,
-                                    self.hyperparameters["Actor"]["gradient_clipping_norm"])
-        if alpha_loss is not None:
-            self.take_optimisation_step(self.alpha_optim, None, alpha_loss, None)
-            self.alpha = self.log_alpha.exp()
-
-    def print_summary_of_latest_evaluation_episode(self):
-        """Prints a summary of the latest episode"""
-        print(" ")
-        print("----------------------------")
-        print("Episode score {} ".format(self.total_episode_score_so_far))
-        print("----------------------------")

agents/actor_critic_agents/SAC_Discrete.py

@@ -1,94 +0,0 @@
-import torch
-from torch.optim import Adam
-import torch.nn.functional as F
-import numpy as np
-from agents.Base_Agent import Base_Agent
-from utilities.data_structures.Replay_Buffer import Replay_Buffer
-from agents.actor_critic_agents.SAC import SAC
-from utilities.Utility_Functions import create_actor_distribution
-
-class SAC_Discrete(SAC):
-    """The Soft Actor Critic for discrete actions. It inherits from SAC for continuous actions and only changes a few
-    methods."""
-    agent_name = "SAC"
-    def __init__(self, config):
-        Base_Agent.__init__(self, config)
-        assert self.action_types == "DISCRETE", "Action types must be discrete. Use SAC instead for continuous actions"
-        assert self.config.hyperparameters["Actor"]["final_layer_activation"] == "Softmax", "Final actor layer must be softmax"
-        self.hyperparameters = config.hyperparameters
-        self.critic_local = self.create_NN(input_dim=self.state_size, output_dim=self.action_size, key_to_use="Critic")
-        self.critic_local_2 = self.create_NN(input_dim=self.state_size, output_dim=self.action_size,
-                                           key_to_use="Critic", override_seed=self.config.seed + 1)
-        self.critic_optimizer = torch.optim.Adam(self.critic_local.parameters(),
-                                                 lr=self.hyperparameters["Critic"]["learning_rate"], eps=1e-4)
-        self.critic_optimizer_2 = torch.optim.Adam(self.critic_local_2.parameters(),
-                                                   lr=self.hyperparameters["Critic"]["learning_rate"], eps=1e-4)
-        self.critic_target = self.create_NN(input_dim=self.state_size, output_dim=self.action_size,
-                                           key_to_use="Critic")
-        self.critic_target_2 = self.create_NN(input_dim=self.state_size, output_dim=self.action_size,
-                                            key_to_use="Critic")
-        Base_Agent.copy_model_over(self.critic_local, self.critic_target)
-        Base_Agent.copy_model_over(self.critic_local_2, self.critic_target_2)
-        self.memory = Replay_Buffer(self.hyperparameters["Critic"]["buffer_size"], self.hyperparameters["batch_size"],
-                                    self.config.seed, device=self.device)
-
-        self.actor_local = self.create_NN(input_dim=self.state_size, output_dim=self.action_size, key_to_use="Actor")
-        self.actor_optimizer = torch.optim.Adam(self.actor_local.parameters(),
-                                          lr=self.hyperparameters["Actor"]["learning_rate"], eps=1e-4)
-        self.automatic_entropy_tuning = self.hyperparameters["automatically_tune_entropy_hyperparameter"]
-        if self.automatic_entropy_tuning:
-            # we set the max possible entropy as the target entropy
-            self.target_entropy = -np.log((1.0 / self.action_size)) * 0.98
-            self.log_alpha = torch.zeros(1, requires_grad=True, device=self.device)
-            self.alpha = self.log_alpha.exp()
-            self.alpha_optim = Adam([self.log_alpha], lr=self.hyperparameters["Actor"]["learning_rate"], eps=1e-4)
-        else:
-            self.alpha = self.hyperparameters["entropy_term_weight"]
-        assert not self.hyperparameters["add_extra_noise"], "There is no add extra noise option for the discrete version of SAC at moment"
-        self.add_extra_noise = False
-        self.do_evaluation_iterations = self.hyperparameters["do_evaluation_iterations"]
-
-    def produce_action_and_action_info(self, state):
-        """Given the state, produces an action, the probability of the action, the log probability of the action, and
-        the argmax action"""
-        action_probabilities = self.actor_local(state)
-        max_probability_action = torch.argmax(action_probabilities, dim=-1)
-        action_distribution = create_actor_distribution(self.action_types, action_probabilities, self.action_size)
-        action = action_distribution.sample().cpu()
-        # Have to deal with situation of 0.0 probabilities because we can't do log 0
-        z = action_probabilities == 0.0
-        z = z.float() * 1e-8
-        log_action_probabilities = torch.log(action_probabilities + z)
-        return action, (action_probabilities, log_action_probabilities), max_probability_action
-
-    def calculate_critic_losses(self, state_batch, action_batch, reward_batch, next_state_batch, mask_batch):
-        """Calculates the losses for the two critics. This is the ordinary Q-learning loss except the additional entropy
-         term is taken into account"""
-        with torch.no_grad():
-            next_state_action, (action_probabilities, log_action_probabilities), _ = self.produce_action_and_action_info(next_state_batch)
-            qf1_next_target = self.critic_target(next_state_batch)
-            qf2_next_target = self.critic_target_2(next_state_batch)
-            min_qf_next_target = action_probabilities * (torch.min(qf1_next_target, qf2_next_target) - self.alpha * log_action_probabilities)
-            min_qf_next_target = min_qf_next_target.sum(dim=1).unsqueeze(-1)
-            next_q_value = reward_batch + (1.0 - mask_batch) * self.hyperparameters["discount_rate"] * (min_qf_next_target)
-
-        qf1 = self.critic_local(state_batch).gather(1, action_batch.long())
-        qf2 = self.critic_local_2(state_batch).gather(1, action_batch.long())
-        qf1_loss = F.mse_loss(qf1, next_q_value)
-        qf2_loss = F.mse_loss(qf2, next_q_value)
-        return qf1_loss, qf2_loss
-
-    def calculate_actor_loss(self, state_batch):
-        """Calculates the loss for the actor. This loss includes the additional entropy term"""
-        action, (action_probabilities, log_action_probabilities), _ = self.produce_action_and_action_info(state_batch)
-        qf1_pi = self.critic_local(state_batch)
-        qf2_pi = self.critic_local_2(state_batch)
-        min_qf_pi = torch.min(qf1_pi, qf2_pi)
-        inside_term = self.alpha * log_action_probabilities - min_qf_pi
-        policy_loss = (action_probabilities * inside_term).sum(dim=1).mean()
-        log_action_probabilities = torch.sum(log_action_probabilities * action_probabilities, dim=1)
-        return policy_loss, log_action_probabilities
-
-    def locally_save_policy(self):
-        """Saves the policy"""
-        torch.save(self.actor_local.state_dict(), "{}/{}_network.pt".format(self.config.models_dir, self.agent_name))

agents/actor_critic_agents/TD3.py

@@ -1,54 +0,0 @@
-import torch
-import torch.nn.functional as functional
-from torch import optim
-from agents.Base_Agent import Base_Agent
-from .DDPG import DDPG
-from exploration_strategies.Gaussian_Exploration import Gaussian_Exploration
-
-class TD3(DDPG):
-    """A TD3 Agent from the paper Addressing Function Approximation Error in Actor-Critic Methods (Fujimoto et al. 2018)
-    https://arxiv.org/abs/1802.09477"""
-    agent_name = "TD3"
-
-    def __init__(self, config):
-        DDPG.__init__(self, config)
-        self.critic_local_2 = self.create_NN(input_dim=self.state_size + self.action_size, output_dim=1,
-                                           key_to_use="Critic", override_seed=self.config.seed + 1)
-        self.critic_target_2 = self.create_NN(input_dim=self.state_size + self.action_size, output_dim=1,
-                                            key_to_use="Critic")
-        Base_Agent.copy_model_over(self.critic_local_2, self.critic_target_2)
-        self.critic_optimizer_2 = optim.Adam(self.critic_local_2.parameters(),
-                                           lr=self.hyperparameters["Critic"]["learning_rate"], eps=1e-4)
-        self.exploration_strategy_critic = Gaussian_Exploration(self.config)
-
-    def compute_critic_values_for_next_states(self, next_states):
-        """Computes the critic values for next states to be used in the loss for the critic"""
-        with torch.no_grad():
-            actions_next = self.actor_target(next_states)
-            actions_next_with_noise =  self.exploration_strategy_critic.perturb_action_for_exploration_purposes({"action": actions_next})
-            critic_targets_next_1 = self.critic_target(torch.cat((next_states, actions_next_with_noise), 1))
-            critic_targets_next_2 = self.critic_target_2(torch.cat((next_states, actions_next_with_noise), 1))
-            critic_targets_next = torch.min(torch.cat((critic_targets_next_1, critic_targets_next_2),1), dim=1)[0].unsqueeze(-1)
-        return critic_targets_next
-
-    def critic_learn(self, states, actions, rewards, next_states, dones):
-        """Runs a learning iteration for both the critics"""
-        critic_targets_next =  self.compute_critic_values_for_next_states(next_states)
-        critic_targets = self.compute_critic_values_for_current_states(rewards, critic_targets_next, dones)
-
-        critic_expected_1 = self.critic_local(torch.cat((states, actions), 1))
-        critic_expected_2 = self.critic_local_2(torch.cat((states, actions), 1))
-
-        critic_loss_1 = functional.mse_loss(critic_expected_1, critic_targets)
-        critic_loss_2 = functional.mse_loss(critic_expected_2, critic_targets)
-
-        self.take_optimisation_step(self.critic_optimizer, self.critic_local, critic_loss_1, self.hyperparameters["Critic"]["gradient_clipping_norm"])
-        self.take_optimisation_step(self.critic_optimizer_2, self.critic_local_2, critic_loss_2,
-                                    self.hyperparameters["Critic"]["gradient_clipping_norm"])
-
-        self.soft_update_of_target_network(self.critic_local, self.critic_target, self.hyperparameters["Critic"]["tau"])
-        self.soft_update_of_target_network(self.critic_local_2, self.critic_target_2, self.hyperparameters["Critic"]["tau"])
-
-
-
-