File size: 2,478 Bytes
a164e13 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 |
# Copyright (C) 2024 Charles O. Goddard
#
# This software is free software: you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This software is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see http://www.gnu.org/licenses/.
from enum import Enum
import torch
class SparsificationMethod(str, Enum):
magnitude = "magnitude"
random = "random"
rescaled_random = "rescaled_random"
def magnitude(tensor: torch.Tensor, density: float) -> torch.Tensor:
"""Masks out the smallest values, retaining a proportion of `density`."""
if density >= 1:
return tensor
k = int(density * tensor.view(-1).shape[0])
assert k > 0, "not gonna zero out the whole tensor buddy"
mask = torch.zeros_like(tensor)
w = tensor.abs().view(-1)
if w.device.type == "cpu":
w = w.float()
topk = torch.topk(w, k=k, largest=True)
mask.view(-1)[topk.indices] = 1
return tensor * mask
def bernoulli(
tensor: torch.Tensor, density: float, rescale: bool = True
) -> torch.Tensor:
if density >= 1:
return tensor
if (tensor.device.type != "cpu") or tensor.dtype == torch.bfloat16:
work_dtype = tensor.dtype
else:
# torch.bernoulli not implemented for float16 on CPU, upcast to float32
work_dtype = torch.float32
mask = torch.bernoulli(
torch.full_like(input=tensor, fill_value=density, dtype=work_dtype)
)
res = tensor.to(work_dtype) * mask
if rescale:
res /= density
return res.to(tensor.dtype)
def sparsify(
tensor: torch.Tensor, density: float, method: SparsificationMethod
) -> torch.Tensor:
if method == SparsificationMethod.magnitude:
return magnitude(tensor, density=density)
elif method == SparsificationMethod.random:
return bernoulli(tensor, density=density, rescale=False)
elif method == SparsificationMethod.rescaled_random:
return bernoulli(tensor, density=density, rescale=True)
else:
raise NotImplementedError(method)
|