Upload BabyLangModel with model card and config

README.md

@@ -1,3 +1,106 @@
+# 🍼 BabyLangModel
+
+A tiny GPT-style language model trained from scratch on the [TinyStories](https://huggingface.co/datasets/roneneldan/TinyStories) dataset. Built using PyTorch and a custom architecture inspired by [nanoGPT](https://github.com/karpathy/nanoGPT). This model was trained for 200k iterations on a consumer GPU (RTX 4060) using custom code from scratch.
+
 ---
-license: mit
+
+## 🧠 Model Details
+
+- **Architecture**: GPT (custom implementation)
+- **Parameters**: ~10–15M
+- **Layers**: 6
+- **Heads**: 6
+- **Embedding Size**: 384
+- **Block Size**: 128
+- **Tokenizer**: GPT-2 (`tiktoken`)
+- **Training Steps**: 200,000
+- **Training Loss**: ~1.80
+
 ---
+
+## 📚 Training Data
+
+We trained on the open-source **[TinyStories](https://huggingface.co/datasets/roneneldan/TinyStories)** dataset by Microsoft Research. It's a dataset of short, simple English stories written for young children (ages 2–4).
+
+- Clean, simple narratives
+- Ideal for small model generalization
+- 100% open and publicly available
+
+---
+
+## 🧰 Usage (with `transformers`)
+
+This model uses a **custom architecture**, so you need to use `trust_remote_code=True`:
+
+```python
+from transformers import AutoModel
+
+model = AutoModel.from_pretrained("Exquisique/BabyLangModel", trust_remote_code=True)
+```
+
+---
+
+## ✨ Sample Generation
+
+```text
+Prompt: Once upon a time there was a tiny robot who
+
+Output: ...lived in a far away home. One day, a little girl named Lily decided to go on a special trip in the forest. She walked and walked until she got there but suddenly she started to go. Her mom called her and said, "Don't worry, Lily. We will get you my special ride."
+```
+
+> 🗣️ Still improving, but quite readable and story-like after 200k iterations!
+
+---
+
+## 💻 Train It Yourself
+
+You can find the full training code on [GitHub](https://github.com/Exquisique/Babymodel) or use this structure:
+
+```bash
+python -m src.tokenizer      # Tokenize TinyStories
+python -m src.train          # Train model from scratch
+python -m src.generate       # Generate text
+```
+
+You’ll also find:
+- Checkpointing & resume support
+- Configurable hyperparams
+- Gradient accumulation & mixed precision
+
+---
+
+## 🔧 Config Used
+
+```json
+{
+  "vocab_size": 50257,
+  "block_size": 128,
+  "n_layer": 6,
+  "n_head": 6,
+  "n_embd": 384,
+  "model_type": "gpt"
+}
+```
+
+---
+
+## 📦 Inference Notes
+
+To load the model, use:
+
+```python
+from transformers import AutoModel
+model = AutoModel.from_pretrained("Exquisique/BabyLangModel", trust_remote_code=True)
+```
+
+You can also upload a tokenizer later for full text input support (e.g. with `tiktoken`).
+
+---
+
+## 🧑‍💻 Author
+**Exquisique** — GenAI explorer, poetic dreamer, and neural model whisperer.
+
+---
+
+## 📜 License
+MIT — open source, fine-tune and remix freely. ✨

config.json

@@ -0,0 +1,12 @@
+{
+  "model_type": "gpt",
+  "architectures": ["GPT"],
+  "vocab_size": 50257,
+  "block_size": 128,
+  "n_layer": 6,
+  "n_head": 6,
+  "n_embd": 384,
+  "auto_map": {
+    "AutoModel": "model.GPT"
+  }
+}

model.py

@@ -0,0 +1,133 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import math
+
+class LayerNorm(nn.Module):
+    def __init__(self, ndim, bias):
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(ndim))
+        self.bias = nn.Parameter(torch.zeros(ndim)) if bias else None
+
+    def forward(self, x):
+        return F.layer_norm(x, self.weight.shape, self.weight, self.bias, 1e-5)
+
+class CausalSelfAttention(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        assert config.n_embd % config.n_head == 0
+        self.c_attn = nn.Linear(config.n_embd, 3 * config.n_embd, bias=config.bias)
+        self.c_proj = nn.Linear(config.n_embd, config.n_embd, bias=config.bias)
+        self.attn_dropout = nn.Dropout(config.dropout)
+        self.resid_dropout = nn.Dropout(config.dropout)
+        self.n_head = config.n_head
+        self.n_embd = config.n_embd
+        self.flash = hasattr(F, 'scaled_dot_product_attention')
+        if not self.flash:
+            self.register_buffer("bias", torch.tril(torch.ones(config.block_size, config.block_size)).view(1, 1, config.block_size, config.block_size))
+
+    def forward(self, x):
+        B, T, C = x.size()
+        q, k, v = self.c_attn(x).split(self.n_embd, dim=2)
+        k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
+        q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
+        v = v.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
+
+        if self.flash:
+            y = F.scaled_dot_product_attention(q, k, v, attn_mask=None, dropout_p=self.attn_dropout.p if self.training else 0.0, is_causal=True)
+        else:
+            att = (q @ k.transpose(-2, -1)) * (1.0 / math.sqrt(k.size(-1)))
+            att = att.masked_fill(self.bias[:, :, :T, :T] == 0, float('-inf'))
+            att = F.softmax(att, dim=-1)
+            att = self.attn_dropout(att)
+            y = att @ v
+
+        y = y.transpose(1, 2).contiguous().view(B, T, C)
+        y = self.resid_dropout(self.c_proj(y))
+        return y
+
+class MLP(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.c_fc = nn.Linear(config.n_embd, 4 * config.n_embd, bias=config.bias)
+        self.gelu = nn.GELU()
+        self.c_proj = nn.Linear(4 * config.n_embd, config.n_embd, bias=config.bias)
+        self.dropout = nn.Dropout(config.dropout)
+
+    def forward(self, x):
+        return self.dropout(self.c_proj(self.gelu(self.c_fc(x))))
+
+class Block(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.ln1 = LayerNorm(config.n_embd, config.bias)
+        self.attn = CausalSelfAttention(config)
+        self.ln2 = LayerNorm(config.n_embd, config.bias)
+        self.mlp = MLP(config)
+
+    def forward(self, x):
+        x = x + self.attn(self.ln1(x))
+        x = x + self.mlp(self.ln2(x))
+        return x
+
+class GPT(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.transformer = nn.ModuleDict(dict(
+            wte=nn.Embedding(config.vocab_size, config.n_embd),
+            wpe=nn.Embedding(config.block_size, config.n_embd),
+            drop=nn.Dropout(config.dropout),
+            h=nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
+            ln_f=LayerNorm(config.n_embd, config.bias),
+        ))
+        self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
+        self.transformer.wte.weight = self.lm_head.weight
+
+        self.apply(self._init_weights)
+        for pn, p in self.named_parameters():
+            if pn.endswith('c_proj.weight'):
+                nn.init.normal_(p, mean=0.0, std=0.02 / math.sqrt(2 * config.n_layer))
+
+    def _init_weights(self, module):
+        if isinstance(module, nn.Linear):
+            nn.init.normal_(module.weight, mean=0.0, std=0.02)
+            if module.bias is not None:
+                nn.init.zeros_(module.bias)
+        elif isinstance(module, nn.Embedding):
+            nn.init.normal_(module.weight, mean=0.0, std=0.02)
+
+    def forward(self, idx, targets=None):
+        device = idx.device
+        b, t = idx.size()
+        assert t <= self.config.block_size
+        pos = torch.arange(0, t, dtype=torch.long, device=device)
+
+        tok_emb = self.transformer.wte(idx)
+        pos_emb = self.transformer.wpe(pos)
+        x = self.transformer.drop(tok_emb + pos_emb)
+        for block in self.transformer.h:
+            x = block(x)
+        x = self.transformer.ln_f(x)
+
+        if targets is not None:
+            logits = self.lm_head(x)
+            loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=-1)
+            return logits, loss
+        else:
+            logits = self.lm_head(x[:, [-1], :])
+            return logits, None
+
+    @torch.no_grad()
+    def generate(self, idx, max_new_tokens, temperature=1.0, top_k=None):
+        for _ in range(max_new_tokens):
+            idx_cond = idx if idx.size(1) <= self.config.block_size else idx[:, -self.config.block_size:]
+            logits, _ = self(idx_cond)
+            logits = logits[:, -1, :] / temperature
+            if top_k is not None:
+                v, _ = torch.topk(logits, min(top_k, logits.size(-1)))
+                logits[logits < v[:, [-1]]] = -float('Inf')
+            probs = F.softmax(logits, dim=-1)
+            idx_next = torch.multinomial(probs, num_samples=1)
+            idx = torch.cat((idx, idx_next), dim=1)
+        return idx

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04062f6b9a95597ed396310517437052b03f9de17bf352e405851bea2553bd70
+size 360042363