Spaces:
Sleeping
Sleeping
File size: 4,005 Bytes
6fa23b0 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 |
import random
import torch
import sys
from contextlib import closing
#
# from pathos.multiprocessing import ProcessingPool as Pool
from torch.multiprocessing import Pool
from random import randint
from utilities.OU_Noise import OU_Noise
from utilities.Utility_Functions import create_actor_distribution
class Parallel_Experience_Generator(object):
""" Plays n episode in parallel using a fixed agent. Only works for PPO or DDPG type agents at the moment, not Q-learning agents"""
def __init__(self, environment, policy, seed, hyperparameters, action_size, use_GPU=False, action_choice_output_columns=None):
self.use_GPU = use_GPU
self.environment = environment
self.action_types = "DISCRETE" if self.environment.action_space.dtype in [int, 'int64'] else "CONTINUOUS"
self.action_size = action_size
self.policy = policy
self.action_choice_output_columns = action_choice_output_columns
self.hyperparameters = hyperparameters
if self.action_types == "CONTINUOUS": self.noise = OU_Noise(self.action_size, seed, self.hyperparameters["mu"],
self.hyperparameters["theta"], self.hyperparameters["sigma"])
def play_n_episodes(self, n, exploration_epsilon=None):
"""Plays n episodes in parallel using the fixed policy and returns the data"""
self.exploration_epsilon = exploration_epsilon
with closing(Pool(processes=n)) as pool:
results = pool.map(self, range(n))
pool.terminate()
states_for_all_episodes = [episode[0] for episode in results]
actions_for_all_episodes = [episode[1] for episode in results]
rewards_for_all_episodes = [episode[2] for episode in results]
return states_for_all_episodes, actions_for_all_episodes, rewards_for_all_episodes
def __call__(self, n):
exploration = max(0.0, random.uniform(self.exploration_epsilon / 3.0, self.exploration_epsilon * 3.0))
return self.play_1_episode(exploration)
def play_1_episode(self, epsilon_exploration):
"""Plays 1 episode using the fixed policy and returns the data"""
state = self.reset_game()
done = False
episode_states = []
episode_actions = []
episode_rewards = []
while not done:
action = self.pick_action(self.policy, state, epsilon_exploration)
next_state, reward, done, _ = self.environment.step(action)
if self.hyperparameters["clip_rewards"]: reward = max(min(reward, 1.0), -1.0)
episode_states.append(state)
episode_actions.append(action)
episode_rewards.append(reward)
state = next_state
return episode_states, episode_actions, episode_rewards
def reset_game(self):
"""Resets the game environment so it is ready to play a new episode"""
seed = randint(0, sys.maxsize)
torch.manual_seed(seed) # Need to do this otherwise each worker generates same experience
state = self.environment.reset()
if self.action_types == "CONTINUOUS": self.noise.reset()
return state
def pick_action(self, policy, state, epsilon_exploration=None):
"""Picks an action using the policy"""
if self.action_types == "DISCRETE":
if random.random() <= epsilon_exploration:
action = random.randint(0, self.action_size - 1)
return action
state = torch.from_numpy(state).float().unsqueeze(0)
actor_output = policy.forward(state)
if self.action_choice_output_columns is not None:
actor_output = actor_output[:, self.action_choice_output_columns]
action_distribution = create_actor_distribution(self.action_types, actor_output, self.action_size)
action = action_distribution.sample().cpu()
if self.action_types == "CONTINUOUS": action += torch.Tensor(self.noise.sample())
else: action = action.item()
return action |