Spaces:
Sleeping
Sleeping
File size: 2,686 Bytes
ad52a00 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 |
import os
import torch
from datetime import datetime
# hyperparameters
BATCH_SIZE = 64 # how many independent sequences will we process in parallel?
BLOCK_SIZE = 128 # what is the maximum context length for predictions?
MAX_ITER = 2 # number of training iterations
EVAL_INTER = 1
LEARNING_RATE = 1e-5
EPS = 1e-5
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
NUM_HEAD = 6
NUM_EMBED = NUM_HEAD * 128
NUM_LAYER = 8
DROPOUT = 0.3
MAX_SEQ_LEN = 2048
def encode(text_seq: str, tokenizer: any) -> torch.Tensor:
"""
Function to encode input text using a pre-trained tokenizer and vectorized lookups
"""
# tokenize the input text
tokens = tokenizer.tokenize(text_seq)
# convert the tokens to their corresponding ids
token_indices = tokenizer.convert_tokens_to_ids(tokens)
token_indices = torch.tensor(token_indices, dtype=torch.long)
return token_indices
def decode(enc_sec: torch.Tensor, tokenizer: any) -> str:
"""
Function to decode a sequence of token indices back to a string
"""
# convert the indices to a list
enc_sec = enc_sec.tolist()
# decode the indices to a string
text = tokenizer.decode(enc_sec)
return text
def get_batch(data: list[str], block_size: int, batch_size: int):
"""
This is a simple function to create batches of data.
GPUs allow for parallel processing we can feed multiple chunks at once
so that's why we would need batches - how many independant sequences
will we process in parallel.
Parameters:
data: list[str]: data to take batch from
block_size (int): size of the text that is proccessed at once
batch_size (int): number of sequences to process in parallel
Returns:
x, y: a tuple with token sequence and token target
"""
ix = torch.randint(len(data) - block_size, (batch_size, ))
# we stack batch_size rows of sentences
# so x and y are the matrices with rows_num=batch_size
# and col_num=block_size
x = torch.stack([data[i : i + block_size] for i in ix])
# y is x shifted one position right - because we predict
# word in y having all the previous words as context
y = torch.stack([data[i + 1 : i + block_size + 1] for i in ix])
x, y = x.to(DEVICE), y.to(DEVICE)
return x, y
@torch.no_grad()
def estimate_loss(
val_loader,
model: torch.nn.Module,
eval_iters: int = 10
):
out = {}
model.eval()
losses = torch.zeros(eval_iters)
k = 0
for x, y in val_loader:
if k >= eval_iters:
break
logits, loss = model.forward(x, y)
losses[k] = loss.item()
k += 1
out = losses.mean()
model.train()
return out
|