Delete 1

1/GPT2-small_finetune/.gitignore

@@ -1,162 +0,0 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-cover/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-.pybuilder/
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# pdm
-#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#pdm.lock
-#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
-#   in version control.
-#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
-.pdm.toml
-.pdm-python
-.pdm-build/
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-# pytype static type analyzer
-.pytype/
-
-# Cython debug symbols
-cython_debug/
-
-# PyCharm
-#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-#  and can be added to the global gitignore or merged into this file.  For a more nuclear
-#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/

1/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2024 Sathish Kumar R
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

1/README.md

@@ -1,2 +0,0 @@
-# GPT2-small_finetune
-FineTuning GPT2-small

1/checkpoint.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:537fb7db96118d1cc2cd241a035b9e5ca1f991f119d3b5bad80c853e1e43737b
-size 746821134

1/config/gpt2-small.gin

@@ -1,16 +0,0 @@
-GPTConfig.block_size = 1024            
-GPTConfig.vocab_size = 50257           
-GPTConfig.n_layer = 12                 
-GPTConfig.n_head = 12                  
-GPTConfig.n_embd = 768                 
-GPTConfig.batch_size = 8              
-GPTConfig.learning_rate = 6e-5    
-GPTConfig.seed = 42                    
-GPTConfig.epochs = 2
-GPTConfig.weight_decay = 0.001
-GPTConfig.eps = 1e-8
-GPTConfig.betas = (0.9, 0.95)
-GPTConfig.training_backend = "nccl"
-GPTConfig.device = "cuda"
-GPTConfig.model_name = "gpt2"
-GPTConfig.clip_grad_norm_val = 1.0

1/conversation_tokens.npy

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d6e45a35a853f6c564953289864aa042b155f6c7017bc2ec7f4b25511b521286
-size 10116720

1/dataset.py

@@ -1,56 +0,0 @@
-from __future__ import annotations
-import torch
-from torch.utils.data import Dataset
-from model import GPTConfig
-from typing import Tuple, List
-
-
-
-
-class TokenDataset(Dataset):
-    def __init__(self, model_args: GPTConfig, input_ids: List) -> None:
-        """
-        Initializes the TokenDataset.
-
-        Args:
-            model_args: An instance of ModelArgs containing model configuration
-                parameters, including the maximum sequence length.
-            input_ids: A tensor containing tokenized input data.
-
-        Attributes:
-            input_ids: Stores the tokenized input data.
-            block_size: The block size for dividing the input data, determined by
-                the maximum sequence length in model_args.
-        """
-        self.input_ids = input_ids
-        self.block_size = model_args.block_size
-
-    def __len__(self) -> int:
-        """
-        Returns the number of blocks in the dataset.
-
-        Since the input_ids are divided into blocks of size block_size, the number of
-        blocks is calculated as the length of the input_ids minus one, divided by the
-        block size.
-        """
-        return (len(self.input_ids) - 1) // self.block_size
-
-    def __getitem__(self, idx) -> Tuple[torch.Tensor, torch.Tensor]:     
-        """
-        Returns a tuple of two tensors, x and y, where x is the input tensor slice
-        and y is the output tensor slice. The slices are of size block_size, and are
-        taken from the input_ids tensor at the given index.
-
-        Args:
-            idx: The index of the block to retrieve.
-
-        Returns:
-            A tuple of two tensors, x and y.
-        """
-        start_idx = idx * self.block_size
-        end_idx = start_idx + self.block_size
-        
-        x = self.input_ids[start_idx:end_idx]
-        y = self.input_ids[start_idx+1:end_idx+1]
-        
-        return torch.LongTensor(x), torch.LongTensor(y)

1/gradient_norms.npy

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a6a0923184ec6bc884e6b29aa4c20d3034aa94614799d3a85340c51bd3edee1f
-size 2592

1/model.py

@@ -1,218 +0,0 @@
-from __future__ import annotations
-from dataclasses import dataclass
-import torch
-import torch.nn as nn
-from torch.nn import functional as F
-import gin
-from typing import Tuple
-
-
-@gin.configurable
-@dataclass
-class GPTConfig:
-    block_size: int
-    vocab_size: int
-    n_layer: int
-    n_head: int 
-    n_embd: int 
-    batch_size: int 
-    learning_rate: float
-    weight_decay: float
-    eps: float
-    betas: Tuple[float, float]
-    seed: int
-    epochs: int
-    training_backend: str
-    device: str
-    model_name: str
-    clip_grad_norm_val: float
-    dtype: torch.dtype = torch.bfloat16
-
-
-class CausalSelfAttention(nn.Module):
-
-    def __init__(self, config):
-        super().__init__()
-        assert config.n_embd % config.n_head == 0
-        # key, query, value projections for all heads, but in a batch
-        self.c_attn = nn.Linear(config.n_embd, 3 * config.n_embd)
-        # output projection
-        self.c_proj = nn.Linear(config.n_embd, config.n_embd)
-        self.c_proj.NANOGPT_SCALE_INIT = 1
-        # regularization
-        self.n_head = config.n_head
-        self.n_embd = config.n_embd
-
-    def forward(self, x):
-        B, T, C = x.size() # batch size, sequence length, embedding dimensionality (n_embd)
-        # calculate query, key, values for all heads in batch and move head forward to be the batch dim
-        # nh is "number of heads", hs is "head size", and C (number of channels) = nh * hs
-        # e.g. in GPT-2 (124M), n_head=12, hs=64, so nh*hs=C=768 channels in the Transformer
-        qkv = self.c_attn(x)
-        q, k, v = qkv.split(self.n_embd, dim=2)
-        k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)
-        q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)
-        v = v.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)
-        y = F.scaled_dot_product_attention(q, k, v, is_causal=True) # flash attention
-        y = y.transpose(1, 2).contiguous().view(B, T, C) # re-assemble all head outputs side by side
-        # output projection
-        y = self.c_proj(y)
-        return y
-
-class MLP(nn.Module):
-
-    def __init__(self, config):
-        super().__init__()
-        self.c_fc    = nn.Linear(config.n_embd, 4 * config.n_embd)
-        self.gelu    = nn.GELU(approximate='tanh')
-        self.c_proj  = nn.Linear(4 * config.n_embd, config.n_embd)
-        self.c_proj.NANOGPT_SCALE_INIT = 1
-
-    def forward(self, x):
-        x = self.c_fc(x)
-        x = self.gelu(x)
-        x = self.c_proj(x)
-        return x
-
-class Block(nn.Module):
-
-    def __init__(self, config):
-        super().__init__()
-        self.ln_1 = nn.LayerNorm(config.n_embd)
-        self.attn = CausalSelfAttention(config)
-        self.ln_2 = nn.LayerNorm(config.n_embd)
-        self.mlp = MLP(config)
-
-    def forward(self, x):
-        x = x + self.attn(self.ln_1(x))
-        x = x + self.mlp(self.ln_2(x))
-        return x
-
-class GPT(nn.Module):
-
-    def __init__(self, config):
-        super().__init__()
-        self.config = config
-
-        self.transformer = nn.ModuleDict(dict(
-            wte = nn.Embedding(config.vocab_size, config.n_embd),
-            wpe = nn.Embedding(config.block_size, config.n_embd),
-            h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
-            ln_f = nn.LayerNorm(config.n_embd),
-        ))
-        self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
-
-        # weight sharing scheme
-        self.transformer.wte.weight = self.lm_head.weight
-
-        # init params
-        self.apply(self._init_weights)
-
-    def _init_weights(self, module):
-        if isinstance(module, nn.Linear):
-            std = 0.02
-            if hasattr(module, 'NANOGPT_SCALE_INIT'):
-                std *= (2 * self.config.n_layer) ** -0.5
-            torch.nn.init.normal_(module.weight, mean=0.0, std=std)
-            if module.bias is not None:
-                torch.nn.init.zeros_(module.bias)
-        elif isinstance(module, nn.Embedding):
-            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
-
-    def forward(self, idx, targets=None):
-        # idx is of shape (B, T)
-        B, T = idx.size()
-        assert T <= self.config.block_size, f"Cannot forward sequence of length {T}, block size is only {self.config.block_size}"
-        # forward the token and posisition embeddings
-        pos = torch.arange(0, T, dtype=torch.long, device=idx.device) # shape (T)
-        pos_emb = self.transformer.wpe(pos) # position embeddings of shape (T, n_embd)
-        tok_emb = self.transformer.wte(idx) # token embeddings of shape (B, T, n_embd)
-        x = tok_emb + pos_emb
-        # forward the blocks of the transformer
-        for block in self.transformer.h:
-            x = block(x)
-        # forward the final layernorm and the classifier
-        x = self.transformer.ln_f(x)
-        logits = self.lm_head(x) # (B, T, vocab_size)
-        loss = None
-        if targets is not None:
-            loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1))
-        return logits, loss
-
-    @classmethod
-    def from_pretrained(cls, model_type):
-        """Loads pretrained GPT-2 model weights from huggingface"""
-        assert model_type in {'gpt2', 'gpt2-medium', 'gpt2-large', 'gpt2-xl'}
-        from transformers import GPT2LMHeadModel
-        print("loading weights from pretrained gpt: %s" % model_type)
-
-        # n_layer, n_head and n_embd are determined from model_type
-        config_args = {
-            'gpt2':         dict(n_layer=12, n_head=12, n_embd=768),  # 124M params
-            'gpt2-medium':  dict(n_layer=24, n_head=16, n_embd=1024), # 350M params
-            'gpt2-large':   dict(n_layer=36, n_head=20, n_embd=1280), # 774M params
-            'gpt2-xl':      dict(n_layer=48, n_head=25, n_embd=1600), # 1558M params
-        }[model_type]
-        config_args['vocab_size'] = 50257 # always 50257 for GPT model checkpoints
-        config_args['block_size'] = 1024 # always 1024 for GPT model checkpoints
-        # create a from-scratch initialized minGPT model
-        config = GPTConfig(**config_args)
-        model = GPT(config)
-        sd = model.state_dict()
-        sd_keys = sd.keys()
-        sd_keys = [k for k in sd_keys if not k.endswith('.attn.bias')] # discard this mask / buffer, not a param
-
-        # init a huggingface/transformers model
-        model_hf = GPT2LMHeadModel.from_pretrained(model_type)
-        sd_hf = model_hf.state_dict()
-
-        # copy while ensuring all of the parameters are aligned and match in names and shapes
-        sd_keys_hf = sd_hf.keys()
-        sd_keys_hf = [k for k in sd_keys_hf if not k.endswith('.attn.masked_bias')] # ignore these, just a buffer
-        sd_keys_hf = [k for k in sd_keys_hf if not k.endswith('.attn.bias')] # same, just the mask (buffer)
-        transposed = ['attn.c_attn.weight', 'attn.c_proj.weight', 'mlp.c_fc.weight', 'mlp.c_proj.weight']
-        # basically the openai checkpoints use a "Conv1D" module, but we only want to use a vanilla Linear
-        # this means that we have to transpose these weights when we import them
-        assert len(sd_keys_hf) == len(sd_keys), f"mismatched keys: {len(sd_keys_hf)} != {len(sd_keys)}"
-        for k in sd_keys_hf:
-            if any(k.endswith(w) for w in transposed):
-                # special treatment for the Conv1D weights we need to transpose
-                assert sd_hf[k].shape[::-1] == sd[k].shape
-                with torch.no_grad():
-                    sd[k].copy_(sd_hf[k].t())
-            else:
-                # vanilla copy over the other parameters
-                assert sd_hf[k].shape == sd[k].shape
-                with torch.no_grad():
-                    sd[k].copy_(sd_hf[k])
-
-        return model
-    
-
-# import lightning as L
-
-# class GPT2Wrapper(L.LightningModule):
-#     def __init__(self, config: GPTConfig, model: GPT):
-#         super().__init__()
-#         self.config = config
-#         self.model = model    
-#         self.optimizer = self.configure_optimizers()
-#         self.train_loss = []
-
-#     def forward(self, idx, targets=None):
-#         return self.model(idx, targets)
-
-#     def training_step(self, batch, batch_idx):
-#         self.model.train()
-#         optimizer = self.optimizers()
-#         optimizer.zero_grad()
-        
-#         batch, label = batch
-#         _, loss = self.model(batch, label)
-#         self.log("Train_Loss", loss, prog_bar=True)
-#         self.train_loss.append(loss.item())
-
-#         return loss
-    
-#     def configure_optimizers(self):
-#         return torch.optim.AdamW(self.model.parameters(), lr=self.config.learning_rate, betas=self.config.betas, eps=self.config.eps, weight_decay=self.config.weight_decay)

1/requirements.txt

@@ -1,4 +0,0 @@
-gin-config
-huggingface_hub
-# deepspeed
-# lightning

1/testing.ipynb

@@ -1,74 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "c:\\Users\\sathi\\miniconda3\\envs\\dl\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
-      "  from .autonotebook import tqdm as notebook_tqdm\n"
-     ]
-    }
-   ],
-   "source": [
-    "from model import *"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "GPTConfig(block_size=1024, vocab_size=50257, n_layer=12, n_head=12, n_embd=768, batch_size=16, learning_rate=0.0003, weight_decay=0.001, eps=1e-08, betas=(0.9, 0.95), seed=42, epochs=1)"
-      ]
-     },
-     "execution_count": 3,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "gin.parse_config_file(\"config/gpt2-small.gin\")\n",
-    "config = GPTConfig()\n",
-    "config"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "model = GPT.from_pretrained()"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "dl",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.9"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}

1/time_spent.npy

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8d4823bd1dd822c435a723dd8f7bc7fc060dcdf76ce0fc7e023a79a4c076b742
-size 2592

1/train.py

@@ -1,127 +0,0 @@
-from __future__ import annotations
-import os
-import gin
-import numpy as np
-import torch
-import torch.optim as optim
-import torch.distributed as dist
-import torch.multiprocessing as mp
-from torch.utils.data import DataLoader, DistributedSampler
-from torch.nn.parallel import DistributedDataParallel as DDP
-from huggingface_hub import hf_hub_download
-from model import GPTConfig, GPT
-from dataset import TokenDataset
-from tqdm import tqdm
-import time
-
-
-hf_hub_download(repo_id="pt-sk/chatgpt-dataset", filename="conversation_tokens.npy", repo_type="dataset", local_dir="/kaggle/working")
-tokens = np.load("/kaggle/working/conversation_tokens.npy")
-print(f"Number of tokens: {len(tokens)}")
-
-
-
-gin.parse_config_file("config/gpt2-small.gin")
-config = GPTConfig()
-
-
-np.random.seed(config.seed)
-torch.manual_seed(config.seed)
-torch.cuda.manual_seed(config.seed)
-torch.cuda.manual_seed_all(config.seed)
-
-
-
-
-def trainer(rank, world_size):
-    # Initialize the Process Group
-    dist.init_process_group(backend=config.training_backend, rank=rank, world_size=world_size)
-
-    # Set the Device for the Current Process
-    torch.cuda.set_device(rank)
-    device = torch.device(config.device, rank)
-
-    # Define Model and Optimizer
-    model = GPT.from_pretrained(config.model_name)
-    model.to(config.dtype).to(device)
-    model = DDP(model, device_ids=[rank])  # Wrap model in DDP
-
-    # Define Optimizer    
-    optimizer = optim.AdamW(model.parameters(), lr=config.learning_rate, betas=config.betas, eps=config.eps, weight_decay=config.weight_decay)
-
-    # Create DataLoader
-    dataset = TokenDataset(config, tokens)
-    # Use DistributedSampler to partition data among distributed processes
-    sampler = DistributedSampler(dataset, num_replicas=world_size, rank=rank)
-    # Use DataLoader to manage batches
-    dataloader = DataLoader(dataset, batch_size=config.batch_size, sampler=sampler, drop_last=True)
-
-
-    # Training Loop
-    model.train()
-    training_loss = []
-    gradient_norms = []
-    time_spent = []
-    for epoch in range(config.epochs) :  # Loop over the dataset multiple times
-        sampler.set_epoch(epoch)  # Shuffle data per epoch for 
-        
-        for batch, (inputs, labels) in enumerate(dataloader):
-            start_time = time.time()
-            # Move data to device
-            inputs, labels = inputs.to(device), labels.to(device)
-
-            # Forward pass
-            _, loss = model(inputs, labels)
-            
-            # Zero gradients before backward pass
-            optimizer.zero_grad()
-            
-            # Backward pass
-            loss.backward()
-
-            # Gradient clipping
-            grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), config.clip_grad_norm_val)
-            
-            # Update weights and biases
-            optimizer.step()
-
-            # Log training loss and gradient norms
-            training_loss.append(loss.item())
-            gradient_norms.append(grad_norm.item())
-            time_spent.append(time.time() - start_time)
-            if rank == 0:
-                print(f"Epoch: {epoch}, Batch: {batch}, Loss: {loss.item()}, Gradient Norm: {grad_norm.item()}, Time Spent: {round(time_spent[-1], 2)} seconds")
-
-
-    # Log training loss and gradient norms
-    if rank == 0:
-        np.save("training_loss.npy", np.array(training_loss))
-        np.save("gradient_norms.npy", np.array(gradient_norms))
-        np.save("time_spent.npy", np.array(time_spent))
-
-        # Save the model and optimizer states for checkpointing
-        torch.save(
-            {
-                "model_state_dict": model.state_dict(),
-                "optimizer_state_dict": optimizer.state_dict(),
-            },
-            "checkpoint.pth",
-        )
-
-    # Cleanup
-    dist.destroy_process_group()
-
-
-def run_ddp_training():
-    world_size = torch.cuda.device_count()  # Number of available GPUs
-    mp.spawn(trainer, args=(world_size,), nprocs=world_size, join=True)
-
-
-if __name__ == "__main__":
-    
-    os.environ['MASTER_ADDR'] = 'localhost'
-    os.environ['MASTER_PORT'] = '12355'
-    os.environ['WORLD_SIZE'] = str(torch.cuda.device_count())  # Total number of GPUs on this node
-    os.environ['RANK'] = '0'  # Rank 0 for a single-node setup
-
-    run_ddp_training()

1/training_loss.npy

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7ab3c5e3013452e42bb5095281767cf6780f1bf1fe1bd6fc0707190d5c6478dc
-size 2592