Spaces:
Running
Running
File size: 6,215 Bytes
9d61c9b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 |
from typing import List, Tuple
import numpy as np
import torch
import torch.nn.functional as F
def get_device() -> torch.device:
r"""Function returns the device where the model and tensors should be placed.
Returns
torch.device: The device where the model and tensors should be placed.
"""
return torch.device("cuda" if torch.cuda.is_available() else "cpu")
def pad(input_ele: List[torch.Tensor], max_len: int) -> torch.Tensor:
r"""Takes a list of 1D or 2D tensors and pads them to match the maximum length.
Args:
input_ele (List[torch.Tensor]): The list of tensors to be padded.
max_len (int): The length to which the tensors should be padded.
Returns:
torch.Tensor: A tensor containing all the padded input tensors.
"""
# Create an empty list to store the padded tensors
out_list = torch.jit.annotate(List[torch.Tensor], [])
for batch in input_ele:
if len(batch.shape) == 1:
# Perform padding for 1D tensor
one_batch_padded = F.pad(
batch,
(0, max_len - batch.size(0)),
"constant",
0.0,
)
else:
# Perform padding for 2D tensor
one_batch_padded = F.pad(
batch,
(0, 0, 0, max_len - batch.size(0)),
"constant",
0.0,
)
# Append the padded tensor to the list
out_list.append(one_batch_padded)
# Stack all the tensors in the list into a single tensor
return torch.stack(out_list)
def get_mask_from_lengths(lengths: torch.Tensor) -> torch.Tensor:
r"""Generate a mask tensor from a tensor of sequence lengths.
Args:
lengths (torch.Tensor): A tensor of sequence lengths of shape: (batch_size, )
Returns:
torch.Tensor: A mask tensor of shape: (batch_size, max_len) where max_len is the
maximum sequence length in the provided tensor. The mask tensor has a value of
True at each position that is more than the length of the sequence (padding positions).
Example:
lengths: `torch.tensor([2, 3, 1, 4])`
Mask tensor will be: `torch.tensor([
[False, False, True, True],
[False, False, False, True],
[False, True, True, True],
[False, False, False, False]
])`
"""
# Get batch size
batch_size = lengths.shape[0]
# Get maximum sequence length in the batch
max_len = int(torch.max(lengths).item())
# Generate a tensor of shape (batch_size, max_len)
# where each row contains values from 0 to max_len
ids = (
torch.arange(0, max_len, device=lengths.device)
.unsqueeze(0)
.expand(batch_size, -1)
)
# Compare each value in the ids tensor with
# corresponding sequence length to generate a mask.
# The mask will have True at positions where id >= sequence length,
# indicating padding positions in the original sequences
return ids >= lengths.unsqueeze(1).type(torch.int64).expand(-1, max_len)
def stride_lens_downsampling(lens: torch.Tensor, stride: int = 2) -> torch.Tensor:
r"""Function computes the lengths of 1D tensor when applying a stride for downsampling.
Args:
lens (torch.Tensor): Tensor containing the lengths to be downsampled.
stride (int, optional): The stride to be used for downsampling. Defaults to 2.
Returns:
torch.Tensor: A tensor of the same shape as the input containing the downsampled lengths.
"""
# The torch.ceil function is used to handle cases where the length is not evenly divisible
# by the stride. The torch.ceil function rounds up to the nearest integer, ensuring that
# each item is present at least once in the downsampled lengths.
# Finally, the .int() is used to convert the resulting float32 tensor to an integer tensor.
return torch.ceil(lens / stride).int()
def calc_same_padding(kernel_size: int) -> Tuple[int, int]:
r"""Calculates the necessary padding for 'same' padding in convolutional operations.
For 'same' padding, the output size is the same as the input size for `stride=1`. This function returns
two integers, representing the padding to be added on either side of the input to achieve 'same' padding.
Args:
kernel_size (int): Size of the convolving kernel.
Returns:
Tuple[int, int]: A tuple of two integers representing the number of padding elements to be applied on
left and right (or top and bottom for 2D) of the input tensor respectively.
"""
# Check if kernel_size is an integer greater than zero
if not isinstance(kernel_size, int) or kernel_size <= 0:
raise ValueError("kernel_size must be an integer greater than zero")
# Determine base padding amount (equal to half the kernel size, truncated down)
pad = kernel_size // 2
# Return padding for each side of the kernel. If kernel size is odd, padding is (pad, pad).
# If kernel size is even, padding is (pad, pad - 1) because we can't pad equally on both sides.
return (pad, pad - (kernel_size + 1) % 2)
def initialize_embeddings(shape: Tuple[int, ...]) -> torch.Tensor:
r"""Initialize embeddings using Kaiming initialization (He initialization).
This method is specifically designed for 2D matrices and helps to avoid
the vanishing/exploding gradient problem in deep neural networks.
This is achieved by keeping the variance of the outputs of a layer to be
the same as the variance of its inputs.
Args:
shape (Tuple[int, ...]): The shape of the embedding matrix to create, denoted as a tuple of integers.
The shape should comprise 2 dimensions, i.e., (embedding_dim, num_embeddings).
Raises:
AssertionError: if the provided shape is not 2D.
Returns:
torch.Tensor: the created embedding matrix.
"""
# Check if the input shape is 2D
assert len(shape) == 2, "Can only initialize 2-D embedding matrices ..."
# Initialize the embedding matrix using Kaiming initialization
return torch.randn(shape) * np.sqrt(2 / shape[1])
|