|
|
SLURM_JOB_ID = 1038303 |
|
|
SLURM_JOB_NAME = nvr_elm_llm:train/NVILA-Lite-8B-quantumn-qa-train |
|
|
RUN_NAME = NVILA-Lite-8B-quantumn-qa-train |
|
|
OUTPUT_DIR = runs/train/NVILA-Lite-8B-quantumn-qa-train |
|
|
NNODES = 8 |
|
|
NODES = pool0-01504 pool0-01683 pool0-01722 pool0-01867 pool0-01881 pool0-01893 pool0-01919 pool0-01939 |
|
|
NODE_RANK = 4 |
|
|
GPUS_PER_NODE = 8 |
|
|
MASTER_ADDR = pool0-01504 |
|
|
MASTER_PORT = 25001 |
|
|
GLOBAL_TRAIN_BATCH_SIZE = 1024 |
|
|
GRADIENT_ACCUMULATION_STEPS = 4 |
|
|
PER_DEVICE_TRAIN_BATCH_SIZE = 4 |
|
|
DEFAULT_LEARNING_RATE: 2e-5 |
|
|
SLURM_JOB_ID = 1038303 |
|
|
SLURM_JOB_NAME = nvr_elm_llm:train/NVILA-Lite-8B-quantumn-qa-train |
|
|
RUN_NAME = NVILA-Lite-8B-quantumn-qa-train |
|
|
OUTPUT_DIR = runs/train/NVILA-Lite-8B-quantumn-qa-train |
|
|
NNODES = 8 |
|
|
NODES = pool0-01504 pool0-01683 pool0-01722 pool0-01867 pool0-01881 pool0-01893 pool0-01919 pool0-01939 |
|
|
NODE_RANK = 5 |
|
|
GPUS_PER_NODE = 8 |
|
|
MASTER_ADDR = pool0-01504 |
|
|
MASTER_PORT = 25001 |
|
|
GLOBAL_TRAIN_BATCH_SIZE = 1024 |
|
|
GRADIENT_ACCUMULATION_STEPS = 4 |
|
|
PER_DEVICE_TRAIN_BATCH_SIZE = 4 |
|
|
DEFAULT_LEARNING_RATE: 2e-5 |
|
|
SLURM_JOB_ID = 1038303 |
|
|
SLURM_JOB_NAME = nvr_elm_llm:train/NVILA-Lite-8B-quantumn-qa-train |
|
|
RUN_NAME = NVILA-Lite-8B-quantumn-qa-train |
|
|
OUTPUT_DIR = runs/train/NVILA-Lite-8B-quantumn-qa-train |
|
|
NNODES = 8 |
|
|
NODES = pool0-01504 pool0-01683 pool0-01722 pool0-01867 pool0-01881 pool0-01893 pool0-01919 pool0-01939 |
|
|
NODE_RANK = 7 |
|
|
GPUS_PER_NODE = 8 |
|
|
MASTER_ADDR = pool0-01504 |
|
|
MASTER_PORT = 25001 |
|
|
GLOBAL_TRAIN_BATCH_SIZE = 1024 |
|
|
GRADIENT_ACCUMULATION_STEPS = 4 |
|
|
PER_DEVICE_TRAIN_BATCH_SIZE = 4 |
|
|
DEFAULT_LEARNING_RATE: 2e-5 |
|
|
SLURM_JOB_ID = 1038303 |
|
|
SLURM_JOB_NAME = nvr_elm_llm:train/NVILA-Lite-8B-quantumn-qa-train |
|
|
RUN_NAME = NVILA-Lite-8B-quantumn-qa-train |
|
|
OUTPUT_DIR = runs/train/NVILA-Lite-8B-quantumn-qa-train |
|
|
NNODES = 8 |
|
|
NODES = pool0-01504 pool0-01683 pool0-01722 pool0-01867 pool0-01881 pool0-01893 pool0-01919 pool0-01939 |
|
|
NODE_RANK = 6 |
|
|
GPUS_PER_NODE = 8 |
|
|
MASTER_ADDR = pool0-01504 |
|
|
MASTER_PORT = 25001 |
|
|
GLOBAL_TRAIN_BATCH_SIZE = 1024 |
|
|
GRADIENT_ACCUMULATION_STEPS = 4 |
|
|
PER_DEVICE_TRAIN_BATCH_SIZE = 4 |
|
|
DEFAULT_LEARNING_RATE: 2e-5 |
|
|
SLURM_JOB_ID = 1038303 |
|
|
SLURM_JOB_NAME = nvr_elm_llm:train/NVILA-Lite-8B-quantumn-qa-train |
|
|
RUN_NAME = NVILA-Lite-8B-quantumn-qa-train |
|
|
OUTPUT_DIR = runs/train/NVILA-Lite-8B-quantumn-qa-train |
|
|
NNODES = 8 |
|
|
NODES = pool0-01504 pool0-01683 pool0-01722 pool0-01867 pool0-01881 pool0-01893 pool0-01919 pool0-01939 |
|
|
NODE_RANK = 1 |
|
|
GPUS_PER_NODE = 8 |
|
|
MASTER_ADDR = pool0-01504 |
|
|
MASTER_PORT = 25001 |
|
|
GLOBAL_TRAIN_BATCH_SIZE = 1024 |
|
|
GRADIENT_ACCUMULATION_STEPS = 4 |
|
|
PER_DEVICE_TRAIN_BATCH_SIZE = 4 |
|
|
DEFAULT_LEARNING_RATE: 2e-5 |
|
|
SLURM_JOB_ID = 1038303 |
|
|
SLURM_JOB_NAME = nvr_elm_llm:train/NVILA-Lite-8B-quantumn-qa-train |
|
|
RUN_NAME = NVILA-Lite-8B-quantumn-qa-train |
|
|
OUTPUT_DIR = runs/train/NVILA-Lite-8B-quantumn-qa-train |
|
|
NNODES = 8 |
|
|
NODES = pool0-01504 pool0-01683 pool0-01722 pool0-01867 pool0-01881 pool0-01893 pool0-01919 pool0-01939 |
|
|
NODE_RANK = 3 |
|
|
GPUS_PER_NODE = 8 |
|
|
MASTER_ADDR = pool0-01504 |
|
|
MASTER_PORT = 25001 |
|
|
GLOBAL_TRAIN_BATCH_SIZE = 1024 |
|
|
GRADIENT_ACCUMULATION_STEPS = 4 |
|
|
PER_DEVICE_TRAIN_BATCH_SIZE = 4 |
|
|
DEFAULT_LEARNING_RATE: 2e-5 |
|
|
SLURM_JOB_ID = 1038303 |
|
|
SLURM_JOB_NAME = nvr_elm_llm:train/NVILA-Lite-8B-quantumn-qa-train |
|
|
RUN_NAME = NVILA-Lite-8B-quantumn-qa-train |
|
|
OUTPUT_DIR = runs/train/NVILA-Lite-8B-quantumn-qa-train |
|
|
NNODES = 8 |
|
|
NODES = pool0-01504 pool0-01683 pool0-01722 pool0-01867 pool0-01881 pool0-01893 pool0-01919 pool0-01939 |
|
|
NODE_RANK = 0 |
|
|
GPUS_PER_NODE = 8 |
|
|
MASTER_ADDR = pool0-01504 |
|
|
MASTER_PORT = 25001 |
|
|
GLOBAL_TRAIN_BATCH_SIZE = 1024 |
|
|
GRADIENT_ACCUMULATION_STEPS = 4 |
|
|
PER_DEVICE_TRAIN_BATCH_SIZE = 4 |
|
|
DEFAULT_LEARNING_RATE: 2e-5 |
|
|
SLURM_JOB_ID = 1038303 |
|
|
SLURM_JOB_NAME = nvr_elm_llm:train/NVILA-Lite-8B-quantumn-qa-train |
|
|
RUN_NAME = NVILA-Lite-8B-quantumn-qa-train |
|
|
OUTPUT_DIR = runs/train/NVILA-Lite-8B-quantumn-qa-train |
|
|
NNODES = 8 |
|
|
NODES = pool0-01504 pool0-01683 pool0-01722 pool0-01867 pool0-01881 pool0-01893 pool0-01919 pool0-01939 |
|
|
NODE_RANK = 2 |
|
|
GPUS_PER_NODE = 8 |
|
|
MASTER_ADDR = pool0-01504 |
|
|
MASTER_PORT = 25001 |
|
|
GLOBAL_TRAIN_BATCH_SIZE = 1024 |
|
|
GRADIENT_ACCUMULATION_STEPS = 4 |
|
|
PER_DEVICE_TRAIN_BATCH_SIZE = 4 |
|
|
DEFAULT_LEARNING_RATE: 2e-5 |
|
|
[2025-07-01 09:21:22,856] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:22,972] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:23,111] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:23,563] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:23,735] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:23,799] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:23,799] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:23,803] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:23,847] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:23,855] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:23,856] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:23,911] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,002] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,003] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,019] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,021] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,028] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,030] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,113] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,232] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,243] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,247] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,251] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,253] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,311] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,311] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,328] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,330] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,332] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,369] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,369] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,370] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,371] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,372] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,376] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,424] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,429] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,442] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,446] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,452] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,465] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,518] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,542] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,543] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,563] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,564] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,579] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:24,587] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:25,700] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:25,700] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:25,775] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:25,790] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:25,791] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:25,792] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:25,864] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:25,901] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:25,906] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:26,124] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:26,215] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:26,241] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:26,247] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:26,306] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:26,306] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:26,313] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect) |
|
|
[2025-07-01 09:21:26,339] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:26,339] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:26,459] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:26,459] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:26,560] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:26,561] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:26,944] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:26,944] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,281] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,281] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,322] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,322] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,339] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,339] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,459] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,459] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,513] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,513] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,516] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,516] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,549] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,549] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,571] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,571] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,582] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,583] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,583] [INFO] [comm.py:625:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl |
|
|
[2025-07-01 09:21:27,616] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,617] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,621] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,621] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,623] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,624] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,636] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,636] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,637] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,637] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,643] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,643] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,661] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,661] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,662] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,662] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,679] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,680] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,798] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,799] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,828] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,828] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,854] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,854] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,856] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,856] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,862] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,862] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,865] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,865] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,890] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,890] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,892] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,892] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,930] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,931] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,932] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,932] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,975] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,975] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,984] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,984] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,986] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,986] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,988] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,988] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:27,990] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:27,990] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:28,007] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:28,007] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:28,034] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:28,034] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:28,054] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:28,054] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:28,056] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:28,056] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:28,114] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:28,114] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:28,186] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:28,186] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:28,277] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:28,277] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:28,330] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:28,330] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:28,361] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:28,361] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:28,409] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:28,410] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:28,424] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:28,424] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:29,112] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:29,112] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:29,152] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:29,153] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:29,158] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:29,158] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:29,181] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:29,181] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:29,378] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:29,379] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:29,436] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:29,436] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:29,474] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:29,474] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:29,528] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:29,528] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:29,605] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:29,605] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:29,635] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:29,635] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:29,709] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:29,709] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:29,775] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:29,775] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:29,782] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:29,782] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:29,784] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:29,784] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:29,786] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:29,786] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:29,786] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented |
|
|
[2025-07-01 09:21:29,786] [INFO] [comm.py:594:init_distributed] cdb=None |
|
|
[2025-07-01 09:21:43,807] [INFO] [partition_parameters.py:453:__exit__] finished initializing model with 7.61B parameters |
|
|
[2025-07-01 09:21:52,746] [INFO] [partition_parameters.py:453:__exit__] finished initializing model with 8.03B parameters |
|
|
[2025-07-01 09:21:53,403] [INFO] [partition_parameters.py:453:__exit__] finished initializing model with 8.09B parameters |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
|
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
[dist-0-of-64] LlavaLlamaModel( |
|
|
(llm): Qwen2ForCausalLM( |
|
|
(model): Qwen2Model( |
|
|
(embed_tokens): Embedding(151648, 3584) |
|
|
(layers): ModuleList( |
|
|
(0-27): 28 x Qwen2DecoderLayer( |
|
|
(self_attn): Qwen2FlashAttention2( |
|
|
(q_proj): Linear(in_features=3584, out_features=3584, bias=True) |
|
|
(k_proj): Linear(in_features=3584, out_features=512, bias=True) |
|
|
(v_proj): Linear(in_features=3584, out_features=512, bias=True) |
|
|
(o_proj): Linear(in_features=3584, out_features=3584, bias=False) |
|
|
(rotary_emb): Qwen2RotaryEmbedding() |
|
|
) |
|
|
(mlp): Qwen2MLP( |
|
|
(gate_proj): Linear(in_features=3584, out_features=18944, bias=False) |
|
|
(up_proj): Linear(in_features=3584, out_features=18944, bias=False) |
|
|
(down_proj): Linear(in_features=18944, out_features=3584, bias=False) |
|
|
(act_fn): SiLU() |
|
|
) |
|
|
(input_layernorm): Qwen2RMSNorm((0,), eps=1e-06) |
|
|
(post_attention_layernorm): Qwen2RMSNorm((0,), eps=1e-06) |
|
|
) |
|
|
) |
|
|
(norm): Qwen2RMSNorm((0,), eps=1e-06) |
|
|
(rotary_emb): Qwen2RotaryEmbedding() |
|
|
) |
|
|
(lm_head): Linear(in_features=3584, out_features=151648, bias=False) |
|
|
) |
|
|
(vision_tower): SiglipVisionTower( |
|
|
(vision_tower): SiglipVisionModel( |
|
|
(vision_model): SiglipVisionTransformer( |
|
|
(embeddings): SiglipVisionEmbeddings( |
|
|
(patch_embedding): Conv2d(3, 1152, kernel_size=(14, 14), stride=(14, 14), padding=valid) |
|
|
(position_embedding): Embedding(1024, 1152) |
|
|
) |
|
|
(encoder): SiglipEncoder( |
|
|
(layers): ModuleList( |
|
|
(0-26): 27 x SiglipEncoderLayer( |
|
|
(self_attn): SiglipFlashAttention2( |
|
|
(k_proj): Linear(in_features=1152, out_features=1152, bias=True) |
|
|
(v_proj): Linear(in_features=1152, out_features=1152, bias=True) |
|
|
(q_proj): Linear(in_features=1152, out_features=1152, bias=True) |
|
|
(out_proj): Linear(in_features=1152, out_features=1152, bias=True) |
|
|
) |
|
|
(layer_norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True) |
|
|
(mlp): SiglipMLP( |
|
|
(activation_fn): PytorchGELUTanh() |
|
|
(fc1): Linear(in_features=1152, out_features=4304, bias=True) |
|
|
(fc2): Linear(in_features=4304, out_features=1152, bias=True) |
|
|
) |
|
|
(layer_norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True) |
|
|
) |
|
|
) |
|
|
) |
|
|
(post_layernorm): LayerNorm((1152,), eps=1e-06, elementwise_affine=True) |
|
|
) |
|
|
) |
|
|
) |
|
|
(mm_projector): MultimodalProjector( |
|
|
(layers): Sequential( |
|
|
(0): DownSample3x3BlockFix() |
|
|
(1): LayerNorm((10368,), eps=1e-05, elementwise_affine=True) |
|
|
(2): Linear(in_features=10368, out_features=3456, bias=True) |
|
|
(3): GELU(approximate='none') |
|
|
(4): LayerNorm((3456,), eps=1e-05, elementwise_affine=True) |
|
|
(5): Linear(in_features=3456, out_features=3584, bias=True) |
|
|
(6): GELU(approximate='none') |
|
|
(7): Linear(in_features=3584, out_features=3584, bias=True) |
|
|
) |
|
|
) |
|
|
) |
|
|
[dist-0-of-64] Tunable parameters: |
|
|
language model True |
|
|
[dist-0-of-64] vision tower True |
|
|
[dist-0-of-64] mm projector True |
|
|
trainable params: 8,087,063,152 || all params: 8,087,063,152 || trainable%: 100.0000 |
|
|
[2025-07-01 09:24:26] Rank 51: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 178.73019170761108s |
|
|
[2025-07-01 09:24:26] Rank 15: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 178.88804817199707s |
|
|
[2025-07-01 09:24:26] Rank 61: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 178.51190781593323s |
|
|
[2025-07-01 09:24:26] Rank 18: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.18334817886353s |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
[2025-07-01 09:24:26] Rank 35: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 177.17961239814758s |
|
|
[2025-07-01 09:24:26] Rank 30: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 177.50609588623047s |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
[2025-07-01 09:24:26] Rank 42: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.07143235206604s |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
[2025-07-01 09:24:26] Rank 50: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 178.9429168701172s |
|
|
[2025-07-01 09:24:26] Rank 16: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.6384243965149s |
|
|
[2025-07-01 09:24:26] Rank 0: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.61671090126038s |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
[2025-07-01 09:24:26] Rank 11: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.24142980575562s |
|
|
[2025-07-01 09:24:26] Rank 3: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.68241500854492s |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
[2025-07-01 09:24:27] Rank 59: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 178.93752551078796s |
|
|
[2025-07-01 09:24:27] Rank 36: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 177.47626900672913s |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
[2025-07-01 09:24:27] Rank 43: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.4129433631897s |
|
|
[2025-07-01 09:24:27] Rank 39: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 178.0158293247223s |
|
|
[2025-07-01 09:24:27] Rank 38: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 177.80202722549438s |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
[2025-07-01 09:24:27] Rank 10: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.43888783454895s |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
[2025-07-01 09:24:27] Rank 60: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.1183044910431s |
|
|
[2025-07-01 09:24:27] Rank 49: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.2389600276947s |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
[2025-07-01 09:24:27] Rank 57: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.06473851203918s |
|
|
[2025-07-01 09:24:27] Rank 63: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.11095666885376s |
|
|
[2025-07-01 09:24:27] Rank 41: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.51858401298523s |
|
|
[2025-07-01 09:24:27] Rank 52: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.30554270744324s |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
[2025-07-01 09:24:27] Rank 40: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.5245840549469s |
|
|
[2025-07-01 09:24:27] Rank 21: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 180.82316184043884s |
|
|
[2025-07-01 09:24:27] Rank 37: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 177.645690202713s |
|
|
[2025-07-01 09:24:27] Rank 56: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.1373429298401s |
|
|
[2025-07-01 09:24:27] Rank 7: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.96211647987366s |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
[2025-07-01 09:24:27] Rank 9: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.6913924217224s |
|
|
[2025-07-01 09:24:27] Rank 32: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 177.7236123085022s |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
[2025-07-01 09:24:27] Rank 26: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 178.0112874507904s |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
[2025-07-01 09:24:27] Rank 1: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 180.98196148872375s |
|
|
[2025-07-01 09:24:27] Rank 12: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 180.38428473472595s |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
[2025-07-01 09:24:27] Rank 23: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.83022689819336s |
|
|
[2025-07-01 09:24:27] Rank 62: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.28459119796753s |
|
|
[2025-07-01 09:24:27] Rank 33: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 177.73978686332703s |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
[2025-07-01 09:24:27] Rank 53: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.41727900505066s |
|
|
[2025-07-01 09:24:27] Rank 34: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 177.72356247901917s |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
[2025-07-01 09:24:27] Rank 44: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 180.81957077980042s |
|
|
[2025-07-01 09:24:27] Rank 13: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.6111810207367s |
|
|
[2025-07-01 09:24:27] Rank 20: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.8662896156311s |
|
|
[2025-07-01 09:24:27] Rank 58: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.25026488304138s |
|
|
[2025-07-01 09:24:27] Rank 25: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 178.25618314743042s |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
[2025-07-01 09:24:27] Rank 4: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 180.23809123039246s |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
[2025-07-01 09:24:27] Rank 27: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 178.11056566238403s |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
[2025-07-01 09:24:27] Rank 46: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.69872188568115s |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
[2025-07-01 09:24:27] Rank 14: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.87972497940063s |
|
|
[2025-07-01 09:24:27] Rank 8: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.7657687664032s |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
[2025-07-01 09:24:27] Rank 5: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 180.11463713645935s |
|
|
[2025-07-01 09:24:27] Rank 2: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 180.15511107444763s |
|
|
[2025-07-01 09:24:27] Rank 22: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.94539523124695s |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
[2025-07-01 09:24:27] Rank 48: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.58666896820068s |
|
|
[2025-07-01 09:24:27] Rank 17: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.94371223449707s |
|
|
[2025-07-01 09:24:27] Rank 54: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.51171231269836s |
|
|
[2025-07-01 09:24:27] Rank 6: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 180.14709281921387s |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
[2025-07-01 09:24:27] Rank 19: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.9675374031067s |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
[2025-07-01 09:24:27] Rank 28: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 178.3642818927765s |
|
|
[2025-07-01 09:24:27] Rank 31: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 178.2817325592041s |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
[2025-07-01 09:24:27] Rank 47: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.9938542842865s |
|
|
[2025-07-01 09:24:27] Rank 24: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 178.28741884231567s |
|
|
[2025-07-01 09:24:27] Rank 29: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 178.41006183624268s |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
[2025-07-01 09:24:27] Rank 55: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.64781546592712s |
|
|
[2025-07-01 09:24:27] Rank 45: Timer for terminate callback has been set. |
|
|
Total limit: 240min |
|
|
Pre terminate time: 10min elapsed_time: 179.78166794776917s |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
length of dataloader: 52 13312 |
|
|
[GPU memory] before trainer 2.292407512664795 |
|
|
Parameter Offload: Total persistent parameters: 771184 in 421 params |
|
|
{'loss': 1.2648, 'grad_norm': 130.75577214745246, 'learning_rate': 2e-05, 'epoch': 0.08} |
|
|
{'loss': 1.2283, 'grad_norm': 126.67616596972118, 'learning_rate': 1.9659258262890683e-05, 'epoch': 0.15} |
|
|
[2025-07-01 09:25:57,641] [WARNING] [stage3.py:1850:step] 1 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time |
|
|
{'loss': 0.4188, 'grad_norm': 56.84219933724937, 'learning_rate': 1.866025403784439e-05, 'epoch': 0.23} |
|
|
{'loss': 2.4789, 'grad_norm': 89.42666319016989, 'learning_rate': 1.7071067811865477e-05, 'epoch': 0.31} |
|
|
{'loss': 0.7853, 'grad_norm': 72.41844398977439, 'learning_rate': 1.5000000000000002e-05, 'epoch': 0.38} |
|
|
{'loss': 8.2197, 'grad_norm': 799.0148731599335, 'learning_rate': 1.2588190451025209e-05, 'epoch': 0.46} |
|
|
{'loss': 0.3008, 'grad_norm': 34.525610403243014, 'learning_rate': 1e-05, 'epoch': 0.54} |
|
|
{'loss': 0.3999, 'grad_norm': 64.98250527603693, 'learning_rate': 7.411809548974792e-06, 'epoch': 0.62} |
|
|
{'loss': 0.2575, 'grad_norm': 11.46902235575636, 'learning_rate': 5.000000000000003e-06, 'epoch': 0.69} |
|
|
[2025-07-01 09:28:31,764] [WARNING] [stage3.py:1850:step] 1 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time |
|
|
{'loss': 0.3174, 'grad_norm': 42.63293180170212, 'learning_rate': 2.9289321881345257e-06, 'epoch': 0.77} |
|
|
{'loss': 0.3054, 'grad_norm': 40.64988981794197, 'learning_rate': 1.339745962155613e-06, 'epoch': 0.85} |
|
|
{'loss': 0.2827, 'grad_norm': 27.588182133457394, 'learning_rate': 3.4074173710931804e-07, 'epoch': 0.92} |
|
|
{'loss': 0.2751, 'grad_norm': 17.48926557604337, 'learning_rate': 0.0, 'epoch': 1.0} |
|
|
saving llm to runs/train/NVILA-Lite-8B-quantumn-qa-train/model/tmp-checkpoint-13/llm |
|
|
saving vision_tower to runs/train/NVILA-Lite-8B-quantumn-qa-train/model/tmp-checkpoint-13/vision_tower |
|
|
saving mm_projector to runs/train/NVILA-Lite-8B-quantumn-qa-train/model/tmp-checkpoint-13/mm_projector |
|
|
{'train_runtime': 323.0635, 'train_samples_per_second': 41.206, 'train_steps_per_second': 0.04, 'train_loss': 1.2718768601234143, 'epoch': 1.0} |
|
|
saving llm to runs/train/NVILA-Lite-8B-quantumn-qa-train/model/llm |
|
|
saving vision_tower to runs/train/NVILA-Lite-8B-quantumn-qa-train/model/vision_tower |
|
|
saving mm_projector to runs/train/NVILA-Lite-8B-quantumn-qa-train/model/mm_projector |
|
|
[1;34mwandb[0m: |
|
|
[1;34mwandb[0m: 🚀 View run [33mNVILA-Lite-8B-quantumn-qa-train[0m at: [34mhttps://wandb.ai/ligeng-zhu/vila/runs/NVILA-Lite-8B-quantumn-qa-train[0m |
|
|
|
