| 04/16/2024 17:23:10 - WARNING - __main__ - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: True, 16-bits training: False | |
| 04/16/2024 17:23:10 - INFO - __main__ - Training/evaluation parameters DistillationTrainingArguments( | |
| _n_gpu=1, | |
| accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'gradient_accumulation_kwargs': None}, | |
| adafactor=False, | |
| adam_beta1=0.9, | |
| adam_beta2=0.999, | |
| adam_epsilon=1e-08, | |
| auto_find_batch_size=False, | |
| bf16=False, | |
| bf16_full_eval=False, | |
| data_seed=None, | |
| dataloader_drop_last=False, | |
| dataloader_num_workers=4, | |
| dataloader_persistent_workers=False, | |
| dataloader_pin_memory=True, | |
| dataloader_prefetch_factor=None, | |
| ddp_backend=None, | |
| ddp_broadcast_buffers=None, | |
| ddp_bucket_cap_mb=None, | |
| ddp_find_unused_parameters=None, | |
| ddp_timeout=7200, | |
| debug=[], | |
| deepspeed=None, | |
| disable_tqdm=False, | |
| dispatch_batches=None, | |
| do_eval=True, | |
| do_predict=False, | |
| do_train=True, | |
| dtype=bfloat16, | |
| eval_accumulation_steps=None, | |
| eval_delay=0, | |
| eval_steps=None, | |
| evaluation_strategy=IntervalStrategy.EPOCH, | |
| fp16=False, | |
| fp16_backend=auto, | |
| fp16_full_eval=False, | |
| fp16_opt_level=O1, | |
| freeze_lm_head=False, | |
| fsdp=[], | |
| fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, | |
| fsdp_min_num_params=0, | |
| fsdp_transformer_layer_cls_to_wrap=None, | |
| full_determinism=False, | |
| generation_config=None, | |
| generation_max_length=None, | |
| generation_num_beams=None, | |
| gradient_accumulation_steps=1, | |
| gradient_checkpointing=False, | |
| gradient_checkpointing_kwargs=None, | |
| greater_is_better=None, | |
| group_by_length=False, | |
| half_precision_backend=auto, | |
| hub_always_push=False, | |
| hub_model_id=None, | |
| hub_private_repo=False, | |
| hub_strategy=HubStrategy.EVERY_SAVE, | |
| hub_token=<HUB_TOKEN>, | |
| ignore_data_skip=False, | |
| include_inputs_for_metrics=False, | |
| include_num_input_tokens_seen=False, | |
| include_tokens_per_second=False, | |
| jit_mode_eval=False, | |
| kl_weight=1.0, | |
| label_names=None, | |
| label_smoothing_factor=0.0, | |
| learning_rate=0.0003, | |
| length_column_name=length, | |
| load_best_model_at_end=False, | |
| local_rank=0, | |
| log_level=passive, | |
| log_level_replica=warning, | |
| log_on_each_node=True, | |
| logging_dir=./runs/Apr16_17-23-01_ip-26-0-164-187, | |
| logging_first_step=False, | |
| logging_nan_inf_filter=True, | |
| logging_steps=25, | |
| logging_strategy=IntervalStrategy.STEPS, | |
| lr_scheduler_kwargs={}, | |
| lr_scheduler_type=SchedulerType.LINEAR, | |
| max_grad_norm=1.0, | |
| max_steps=50000, | |
| metric_for_best_model=None, | |
| mp_parameters=, | |
| neftune_noise_alpha=None, | |
| no_cuda=False, | |
| num_train_epochs=3.0, | |
| optim=OptimizerNames.ADAMW_TORCH, | |
| optim_args=None, | |
| optim_target_modules=None, | |
| output_dir=./, | |
| output_router_logits=True, | |
| overwrite_output_dir=True, | |
| past_index=-1, | |
| per_device_eval_batch_size=8, | |
| per_device_train_batch_size=8, | |
| predict_with_generate=False, | |
| prediction_loss_only=False, | |
| push_to_hub=False, | |
| push_to_hub_model_id=None, | |
| push_to_hub_organization=None, | |
| push_to_hub_token=<PUSH_TO_HUB_TOKEN>, | |
| ray_scope=last, | |
| remove_unused_columns=True, | |
| report_to=['wandb'], | |
| resume_from_checkpoint=None, | |
| run_name=./, | |
| save_on_each_node=False, | |
| save_only_model=False, | |
| save_safetensors=True, | |
| save_steps=500, | |
| save_strategy=IntervalStrategy.EPOCH, | |
| save_total_limit=1, | |
| seed=42, | |
| skip_memory_metrics=True, | |
| sortish_sampler=False, | |
| split_batches=None, | |
| temperature=2.0, | |
| tf32=None, | |
| torch_compile=False, | |
| torch_compile_backend=None, | |
| torch_compile_mode=None, | |
| torchdynamo=None, | |
| tpu_metrics_debug=False, | |
| tpu_num_cores=None, | |
| use_cpu=False, | |
| use_ipex=False, | |
| use_legacy_prediction_loop=False, | |
| use_mps_device=False, | |
| warmup_ratio=0.0, | |
| warmup_steps=500, | |
| weight_decay=0.0, | |
| ) | |
| Combining datasets...: 0%| | 0/8 [00:00<?, ?it/s] | |
| Resolving data files: 100%|█████████████████████████████████████████████████████████| 18/18 [00:00<00:00, 101885.93it/s] | |
| Loading dataset shards: 100%|█████████████████████████████████████████████████████████| 18/18 [00:00<00:00, 2253.19it/s] | |
| Resolving data files: 100%|██████████████████████████████████████████████████████████| 18/18 [00:00<00:00, 94019.27it/s] | |
| Resolving data files: 100%|████████████████████████████████████████████████████████████| 18/18 [00:00<00:00, 138.64it/s] | |
| Resolving data files: 100%|██████████████████████████████████████████████████████████| 18/18 [00:00<00:00, 97668.14it/s] | |
| Resolving data files: 100%|██████████████████████████████████████████████████████████| 18/18 [00:00<00:00, 96175.12it/s] | |
| Resolving data files: 100%|████████████████████████████████████████████████████████████| 43/43 [00:00<00:00, 414.13it/s] | |
| Loading dataset shards: 100%|█████████████████████████████████████████████████████████| 43/43 [00:00<00:00, 2036.64it/s] | |
| Resolving data files: 100%|████████████████████████████████████████████████████████████| 18/18 [00:00<00:00, 152.49it/s] | |
| Resolving data files: 100%|███████████████████████████████████████████████████████| 139/139 [00:00<00:00, 322460.32it/s] | |
| Loading dataset shards: 100%|███████████████████████████████████████████████████████| 138/138 [00:00<00:00, 2441.33it/s] | |
| Resolving data files: 100%|████████████████████████████████████████████████████████████| 18/18 [00:00<00:00, 158.30it/s] | |
| Resolving data files: 89%|█████████████████████████████████████████████████████▎ | 16/18 [00:00<00:00, 141.58it/s] | |
| Loading dataset shards: 100%|███████████████████████████████████████████████████████| 117/117 [00:00<00:00, 2419.04it/s] | |
| Combining datasets...: 88%|██████████████████████████████████████████████████████▎ | 7/8 [00:24<00:03, 3.60s/it] | |
| Combining datasets...: 100%|██████████████████████████████████████████████████████████████| 8/8 [00:30<00:00, 3.82s/it] | |
| Resolving data files: 100%|█████████████████████████████████████████████████████████| 18/18 [00:00<00:00, 104134.44it/s] | |
| Resolving data files: 100%|██████████████████████████████████████████████████████████| 18/18 [00:00<00:00, 98176.17it/s] | |
| Resolving data files: 100%|██████████████████████████████████████████████████████████| 18/18 [00:00<00:00, 93553.25it/s] | |
| Resolving data files: 100%|██████████████████████████████████████████████████████████| 18/18 [00:00<00:00, 90851.35it/s] | |
| Resolving data files: 100%|██████████████████████████████████████████████████████████| 18/18 [00:00<00:00, 98560.67it/s] | |
| Resolving data files: 100%|██████████████████████████████████████████████████████████| 18/18 [00:00<00:00, 99996.65it/s] | |
| Resolving data files: 100%|█████████████████████████████████████████████████████████| 43/43 [00:00<00:00, 183287.67it/s] | |
| Resolving data files: 100%|██████████████████████████████████████████████████████████| 18/18 [00:00<00:00, 95325.09it/s] | |
| Resolving data files: 100%|███████████████████████████████████████████████████████| 139/139 [00:00<00:00, 366902.62it/s] | |
| Resolving data files: 100%|█████████████████████████████████████████████████████████| 18/18 [00:00<00:00, 103705.32it/s] | |
| Resolving data files: 100%|███████████████████████████████████████████████████████| 118/118 [00:00<00:00, 308174.27it/s] | |
| Resolving data files: 100%|██████████████████████████████████████████████████████████| 18/18 [00:00<00:00, 85211.59it/s] | |
| loading configuration file config.json from cache at /fsx/sanchit/cache/models--sanchit-gandhi--Mistral-7B-v0.1-6-layer/snapshots/d4e2300e8038196385e9106614a4d7b6c5b70211/config.json | |
| Model config MistralConfig { | |
| "_name_or_path": "sanchit-gandhi/Mistral-7B-v0.1-6-layer", | |
| "architectures": [ | |
| "MistralForCausalLM" | |
| ], | |
| "attention_dropout": 0.0, | |
| "bos_token_id": 1, | |
| "eos_token_id": 2, | |
| "hidden_act": "silu", | |
| "hidden_size": 4096, | |
| "initializer_range": 0.02, | |
| "intermediate_size": 14336, | |
| "max_position_embeddings": 32768, | |
| "model_type": "mistral", | |
| "num_attention_heads": 32, | |
| "num_hidden_layers": 6, | |
| "num_key_value_heads": 8, | |
| "rms_norm_eps": 1e-05, | |
| "rope_theta": 10000.0, | |
| "sliding_window": 4096, | |
| "tie_word_embeddings": false, | |
| "torch_dtype": "float32", | |
| "transformers_version": "4.40.0.dev0", | |
| "use_cache": true, | |
| "vocab_size": 32000 | |
| } | |
| loading file tokenizer.model from cache at /fsx/sanchit/cache/models--sanchit-gandhi--Mistral-7B-v0.1-6-layer/snapshots/d4e2300e8038196385e9106614a4d7b6c5b70211/tokenizer.model | |
| loading file tokenizer.json from cache at /fsx/sanchit/cache/models--sanchit-gandhi--Mistral-7B-v0.1-6-layer/snapshots/d4e2300e8038196385e9106614a4d7b6c5b70211/tokenizer.json | |
| loading file added_tokens.json from cache at None | |
| loading file special_tokens_map.json from cache at /fsx/sanchit/cache/models--sanchit-gandhi--Mistral-7B-v0.1-6-layer/snapshots/d4e2300e8038196385e9106614a4d7b6c5b70211/special_tokens_map.json | |
| loading file tokenizer_config.json from cache at /fsx/sanchit/cache/models--sanchit-gandhi--Mistral-7B-v0.1-6-layer/snapshots/d4e2300e8038196385e9106614a4d7b6c5b70211/tokenizer_config.json | |
| loading configuration file config.json from cache at /fsx/sanchit/cache/models--mistralai--Mistral-7B-v0.1/snapshots/26bca36bde8333b5d7f72e9ed20ccda6a618af24/config.json | |
| Model config MistralConfig { | |
| "_name_or_path": "mistralai/Mistral-7B-v0.1", | |
| "architectures": [ | |
| "MistralForCausalLM" | |
| ], | |
| "attention_dropout": 0.0, | |
| "bos_token_id": 1, | |
| "eos_token_id": 2, | |
| "hidden_act": "silu", | |
| "hidden_size": 4096, | |
| "initializer_range": 0.02, | |
| "intermediate_size": 14336, | |
| "max_position_embeddings": 32768, | |
| "model_type": "mistral", | |
| "num_attention_heads": 32, | |
| "num_hidden_layers": 32, | |
| "num_key_value_heads": 8, | |
| "rms_norm_eps": 1e-05, | |
| "rope_theta": 10000.0, | |
| "sliding_window": 4096, | |
| "tie_word_embeddings": false, | |
| "torch_dtype": "bfloat16", | |
| "transformers_version": "4.40.0.dev0", | |
| "use_cache": true, | |
| "vocab_size": 32000 | |
| } | |
| The device_map was not initialized. Setting device_map to {'':torch.cuda.current_device()}. If you want to use the model for inference, please set device_map ='auto' | |
| loading weights file model.safetensors from cache at /fsx/sanchit/cache/models--mistralai--Mistral-7B-v0.1/snapshots/26bca36bde8333b5d7f72e9ed20ccda6a618af24/model.safetensors.index.json | |
| Downloading shards: 100%|███████████████████████████████████████████████████████████████| 2/2 [00:00<00:00, 1479.73it/s] | |
| Instantiating MistralForCausalLM model under default dtype torch.bfloat16. | |
| Generate config GenerationConfig { | |
| "bos_token_id": 1, | |
| "eos_token_id": 2 | |
| } | |
| Loading checkpoint shards: 100%|██████████████████████████████████████████████████████████| 2/2 [00:05<00:00, 2.99s/it] | |
| All model checkpoint weights were used when initializing MistralForCausalLM. | |
| All the weights of MistralForCausalLM were initialized from the model checkpoint at mistralai/Mistral-7B-v0.1. | |
| If your task is similar to the task the model of the checkpoint was trained on, you can already use MistralForCausalLM for predictions without further training. | |
| loading configuration file generation_config.json from cache at /fsx/sanchit/cache/models--mistralai--Mistral-7B-v0.1/snapshots/26bca36bde8333b5d7f72e9ed20ccda6a618af24/generation_config.json | |
| Generate config GenerationConfig { | |
| "bos_token_id": 1, | |
| "eos_token_id": 2 | |
| } | |
| loading weights file model.safetensors from cache at /fsx/sanchit/cache/models--sanchit-gandhi--Mistral-7B-v0.1-6-layer/snapshots/d4e2300e8038196385e9106614a4d7b6c5b70211/model.safetensors.index.json | |
| Downloading shards: 100%|███████████████████████████████████████████████████████████████| 2/2 [00:00<00:00, 1308.47it/s] | |
| Generate config GenerationConfig { | |
| "bos_token_id": 1, | |
| "eos_token_id": 2 | |
| } | |
| Loading checkpoint shards: 100%|██████████████████████████████████████████████████████████| 2/2 [00:00<00:00, 4.66it/s] | |
| All model checkpoint weights were used when initializing MistralForCausalLM. | |
| All the weights of MistralForCausalLM were initialized from the model checkpoint at sanchit-gandhi/Mistral-7B-v0.1-6-layer. | |
| If your task is similar to the task the model of the checkpoint was trained on, you can already use MistralForCausalLM for predictions without further training. | |
| loading configuration file generation_config.json from cache at /fsx/sanchit/cache/models--sanchit-gandhi--Mistral-7B-v0.1-6-layer/snapshots/d4e2300e8038196385e9106614a4d7b6c5b70211/generation_config.json | |
| Generate config GenerationConfig { | |
| "bos_token_id": 1, | |
| "eos_token_id": 2 | |
| } | |
| tokenizer config file saved in ./tokenizer_config.json | |
| Special tokens file saved in ./special_tokens_map.json | |
| Configuration saved in ./config.json | |
| Configuration saved in ./generation_config.json | |
| preprocess train dataset (num_proc=32): 100%|██████████████████████| 31056744/31056744 [54:46<00:00, 9450.26 examples/s] | |
| filtering train dataset (num_proc=32): 100%|██████████████████████| 31056744/31056744 [05:21<00:00, 96735.06 examples/s] | |
| 04/16/2024 18:24:47 - INFO - __main__ - max_steps is given, it will override any value given in num_train_epochs | |
| 04/16/2024 18:25:15 - INFO - __main__ - ***** Running training ***** | |
| 04/16/2024 18:25:15 - INFO - __main__ - Num examples = 3200000 | |
| 04/16/2024 18:25:15 - INFO - __main__ - Num epochs = 1 | |
| 04/16/2024 18:25:15 - INFO - __main__ - Instantaneous batch size per device = 8 | |
| 04/16/2024 18:25:15 - INFO - __main__ - Gradient accumulation steps = 1 | |
| 04/16/2024 18:25:15 - INFO - __main__ - Total train batch size (w. parallel & distributed) = 64 | |
| 04/16/2024 18:25:15 - INFO - __main__ - Total optimization steps = 50000 | |
| Train steps ... : 0%| | 1/50000 [00:10<140:57:09, 10.15s/it]Traceback (most recent call last): | |
| File "/fsx/sanchit/mistral-debug-4bit/run_distillation.py", line 1474, in <module> | |
| main() | |
| File "/fsx/sanchit/mistral-debug-4bit/run_distillation.py", line 1326, in main | |
| loss, train_metric = train_step(batch, temperature=training_args.temperature) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| File "/fsx/sanchit/mistral-debug-4bit/run_distillation.py", line 1211, in train_step | |
| kl_loss = kl_divergence(teacher_distribution, student_distribution, batch["labels"]) * temperature**2 | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| File "/fsx/sanchit/mistral-debug-4bit/run_distillation.py", line 1183, in kl_divergence | |
| divergence = kl_loss(log_predicted_distribution, target_distribution) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| File "/fsx/sanchit/miniconda3/envs/venv/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1527, in _wrapped_call_impl | |
| return self._call_impl(*args, **kwargs) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| File "/fsx/sanchit/miniconda3/envs/venv/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1536, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| File "/fsx/sanchit/miniconda3/envs/venv/lib/python3.11/site-packages/torch/nn/modules/loss.py", line 470, in forward | |
| return F.kl_div(input, target, reduction=self.reduction, log_target=self.log_target) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| File "/fsx/sanchit/miniconda3/envs/venv/lib/python3.11/site-packages/torch/nn/functional.py", line 2990, in kl_div | |
| reduced = torch.kl_div(input, target, reduction_enum, log_target=log_target) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 1.95 GiB. GPU | |
| [rank0]: Traceback (most recent call last): | |
| [rank0]: File "/fsx/sanchit/mistral-debug-4bit/run_distillation.py", line 1474, in <module> | |
| [rank0]: main() | |
| [rank0]: File "/fsx/sanchit/mistral-debug-4bit/run_distillation.py", line 1326, in main | |
| [rank0]: loss, train_metric = train_step(batch, temperature=training_args.temperature) | |
| [rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| [rank0]: File "/fsx/sanchit/mistral-debug-4bit/run_distillation.py", line 1211, in train_step | |
| [rank0]: kl_loss = kl_divergence(teacher_distribution, student_distribution, batch["labels"]) * temperature**2 | |
| [rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| [rank0]: File "/fsx/sanchit/mistral-debug-4bit/run_distillation.py", line 1183, in kl_divergence | |
| [rank0]: divergence = kl_loss(log_predicted_distribution, target_distribution) | |
| [rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| [rank0]: File "/fsx/sanchit/miniconda3/envs/venv/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1527, in _wrapped_call_impl | |
| [rank0]: return self._call_impl(*args, **kwargs) | |
| [rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| [rank0]: File "/fsx/sanchit/miniconda3/envs/venv/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1536, in _call_impl | |
| [rank0]: return forward_call(*args, **kwargs) | |
| [rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| [rank0]: File "/fsx/sanchit/miniconda3/envs/venv/lib/python3.11/site-packages/torch/nn/modules/loss.py", line 470, in forward | |
| [rank0]: return F.kl_div(input, target, reduction=self.reduction, log_target=self.log_target) | |
| [rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| [rank0]: File "/fsx/sanchit/miniconda3/envs/venv/lib/python3.11/site-packages/torch/nn/functional.py", line 2990, in kl_div | |
| [rank0]: reduced = torch.kl_div(input, target, reduction_enum, log_target=log_target) | |
| [rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| [rank0]: torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 1.95 GiB. GPU |