Spaces:
Runtime error
Runtime error
File size: 1,425 Bytes
8044721 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 |
#!/usr/bin/env python3
# coding=utf-8
import torch
import torch.nn as nn
class AutoClip:
def __init__(self, parameters, initial_clipping=0.1, percentile=50, history_len=1000):
self.parameters = list(parameters)
self.grad_history = [torch.full([history_len], initial_clipping) for _ in self.parameters]
self.index = 0
self.history_len = history_len
self.percentile = percentile
@torch.no_grad()
def __call__(self):
self._add_to_history(self.parameters)
grad_norms = []
for parameter, history in zip(self.parameters, self.grad_history):
if parameter.grad is None or not parameter.grad.abs().sum().is_nonzero():
continue
clip_value = self._get_percentile(history, self.percentile)
grad_norms.append(nn.utils.clip_grad_norm_(parameter, clip_value).item())
return sum(grad_norms) / len(grad_norms)
def _add_to_history(self, parameters):
for i, param in enumerate(parameters):
if param.grad is None or not param.grad.abs().sum().is_nonzero():
continue
self.grad_history[i][self.index] = param.grad.data.norm(2)
self.index = (self.index + 1) % self.history_len
def _get_percentile(self, tensor, percentile):
k = 1 + round(0.01 * percentile * (tensor.numel() - 1))
return tensor.kthvalue(k).values.item()
|