Upload NanoGPT

model.py

@@ -9,13 +9,11 @@ https://github.com/huggingface/transformers/blob/main/src/transformers/models/gp
 
 import math
 import inspect
-# from dataclasses import dataclass
 
 import torch
 import torch.nn as nn
 from torch.nn import functional as F
 
-# from huggingface_hub import PyTorchModelHubMixin
 from transformers import AutoConfig, AutoModel, PretrainedConfig, PreTrainedModel
 
 
@@ -109,17 +107,6 @@ class Block(nn.Module):
         x = x + self.mlp(self.ln_2(x))
         return x
 
-# @dataclass
-# class GPTConfig:
-#     block_size: int = 1024
-#     vocab_size: int = 50304 # GPT-2 vocab_size of 50257, padded up to nearest multiple of 64 for efficiency
-#     n_layer: int = 12
-#     n_head: int = 12
-#     n_embd: int = 768
-#     dropout: float = 0.0
-#     bias: bool = True # True: bias in Linears and LayerNorms, like GPT-2. False: a bit better and faster
-#     outbedding_weight_tying: bool = True
-
 class NanoGPTConfig(PretrainedConfig):
     model_type = "nanoGPT"
 
@@ -148,16 +135,6 @@ class NanoGPTConfig(PretrainedConfig):
 AutoConfig.register("nanoGPT", NanoGPTConfig)
 
 
-# class NanoGPT(PreTrainedModel):
-#     config_class = NanoGPTConfig
-
-#     def __init__(self, config):
-#         super().__init__(config)
-#         self.model = GPT(config)
-
-#     def forward(self, *args, **kwargs):
-#         return self.model.forward(*args, **kwargs)
-
 class NanoGPT(PreTrainedModel):
     config_class = NanoGPTConfig