File size: 2,332 Bytes
9b19c29 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 |
from abc import ABC, abstractmethod
from collections.abc import Sequence
import numpy as np
class BaseNoise(ABC):
"""The action noise base class."""
@abstractmethod
def reset(self) -> None:
"""Reset to the initial state."""
@abstractmethod
def __call__(self, size: Sequence[int]) -> np.ndarray:
"""Generate new noise."""
raise NotImplementedError
class GaussianNoise(BaseNoise):
"""The vanilla Gaussian process, for exploration in DDPG by default."""
def __init__(self, mu: float = 0.0, sigma: float = 1.0) -> None:
self._mu = mu
assert sigma >= 0, "Noise std should not be negative."
self._sigma = sigma
def __call__(self, size: Sequence[int]) -> np.ndarray:
return np.random.normal(self._mu, self._sigma, size)
def reset(self) -> None:
pass
class OUNoise(BaseNoise):
"""Class for Ornstein-Uhlenbeck process, as used for exploration in DDPG.
Usage:
::
# init
self.noise = OUNoise()
# generate noise
noise = self.noise(logits.shape, eps)
For required parameters, you can refer to the stackoverflow page. However,
our experiment result shows that (similar to OpenAI SpinningUp) using
vanilla Gaussian process has little difference from using the
Ornstein-Uhlenbeck process.
"""
def __init__(
self,
mu: float = 0.0,
sigma: float = 0.3,
theta: float = 0.15,
dt: float = 1e-2,
x0: float | np.ndarray | None = None,
) -> None:
super().__init__()
self._mu = mu
self._alpha = theta * dt
self._beta = sigma * np.sqrt(dt)
self._x0 = x0
self.reset()
def reset(self) -> None:
"""Reset to the initial state."""
self._x = self._x0
def __call__(self, size: Sequence[int], mu: float | None = None) -> np.ndarray:
"""Generate new noise.
Return an numpy array which size is equal to ``size``.
"""
if self._x is None or isinstance(self._x, np.ndarray) and self._x.shape != size:
self._x = 0.0
if mu is None:
mu = self._mu
r = self._beta * np.random.normal(size=size)
self._x = self._x + self._alpha * (mu - self._x) + r
return self._x # type: ignore
|