Upload folder using huggingface_hub

README.md

@@ -0,0 +1,60 @@
+---
+library_name: transformers
+license: other
+base_model: deepseek-ai/deepseek-coder-7b-instruct-v1.5
+tags:
+- llama-factory
+- freeze
+- generated_from_trainer
+model-index:
+- name: deepseek_under8_nsx
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# deepseek_under8_nsx
+
+This model is a fine-tuned version of [deepseek-ai/deepseek-coder-7b-instruct-v1.5](https://huggingface.co/deepseek-ai/deepseek-coder-7b-instruct-v1.5) on the codes_nsx_under8 dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-05
+- train_batch_size: 16
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 3
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 384
+- total_eval_batch_size: 24
+- optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- num_epochs: 1.0
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.48.2
+- Pytorch 2.5.1+cu124
+- Datasets 3.2.0
+- Tokenizers 0.21.0

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9907578558225508,
+    "num_input_tokens_seen": 105381888,
+    "total_flos": 4.104162098269913e+18,
+    "train_loss": 0.8075656152483243,
+    "train_runtime": 10309.5741,
+    "train_samples_per_second": 2.518,
+    "train_steps_per_second": 0.006
+}

config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "deepseek-ai/deepseek-coder-7b-instruct-v1.5",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100000,
+  "eos_token_id": 100015,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 30,
+  "num_key_value_heads": 32,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": {
+    "factor": 1.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 4096,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.48.2",
+  "use_cache": false,
+  "vocab_size": 102400
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100000,
+  "eos_token_id": 100015,
+  "transformers_version": "4.48.2"
+}

llamaboard_config.yaml

@@ -0,0 +1,77 @@
+top.booster: liger_kernel
+top.checkpoint_path: null
+top.finetuning_type: freeze
+top.model_name: DeepSeek-Coder-7B-Instruct
+top.quantization_bit: none
+top.quantization_method: bitsandbytes
+top.rope_scaling: llama3
+top.template: deepseekcoder
+train.additional_target: ''
+train.apollo_rank: 256
+train.apollo_scale: 1
+train.apollo_target: all
+train.apollo_update_interval: 200
+train.badam_mode: layer
+train.badam_switch_interval: 50
+train.badam_switch_mode: ascending
+train.badam_update_ratio: 0.05
+train.batch_size: 16
+train.compute_type: bf16
+train.create_new_adapter: false
+train.cutoff_len: 4096
+train.dataset:
+- codes_nsx_under8
+train.dataset_dir: data
+train.ds_offload: false
+train.ds_stage: none
+train.extra_args: '{}'
+train.freeze_extra_modules: ''
+train.freeze_trainable_layers: 2
+train.freeze_trainable_modules: all
+train.galore_rank: 16
+train.galore_scale: 2
+train.galore_target: all
+train.galore_update_interval: 200
+train.gradient_accumulation_steps: 8
+train.learning_rate: 5e-5
+train.logging_steps: 1
+train.lora_alpha: 16
+train.lora_dropout: 0
+train.lora_rank: 8
+train.lora_target: ''
+train.loraplus_lr_ratio: 0
+train.lr_scheduler_type: cosine
+train.mask_history: false
+train.max_grad_norm: '1.0'
+train.max_samples: '50000000'
+train.neat_packing: true
+train.neftune_alpha: 0
+train.num_train_epochs: '1'
+train.packing: true
+train.ppo_score_norm: false
+train.ppo_whiten_rewards: false
+train.pref_beta: 0.1
+train.pref_ftx: 0
+train.pref_loss: sigmoid
+train.report_to:
+- none
+train.resize_vocab: false
+train.reward_model: null
+train.save_steps: 1000
+train.swanlab_api_key: ''
+train.swanlab_mode: cloud
+train.swanlab_project: llamafactory
+train.swanlab_run_name: ''
+train.swanlab_workspace: ''
+train.train_on_prompt: false
+train.training_stage: Supervised Fine-Tuning
+train.use_apollo: true
+train.use_badam: false
+train.use_dora: false
+train.use_galore: false
+train.use_llama_pro: true
+train.use_pissa: false
+train.use_rslora: false
+train.use_swanlab: false
+train.val_size: 0
+train.warmup_steps: 0

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28aba7f82f5ce656674a3e922d68c0b7bd12d2f2e6da09e47fa72eec539c53c2
+size 4987202208

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a14632752c64660abaecefd70025664d95317e19b351e85c08a61c35d2c3dbd9
+size 4980944400

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5925e3a3c3092530a5d595f6643368705428457d37f43b40a0c850464ebbb2d
+size 4662149984

model.safetensors.index.json

@@ -0,0 +1,280 @@
+{
+  "metadata": {
+    "total_size": 14630264832
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

running_log.txt

@@ -0,0 +1,369 @@
+[INFO|2025-07-09 15:57:37] configuration_utils.py:696 >> loading configuration file config.json from cache at /home/kiho/.cache/huggingface/hub/models--deepseek-ai--deepseek-coder-7b-instruct-v1.5/snapshots/2a050a4c59d687a85324d32e147517992117ed30/config.json
+
+[INFO|2025-07-09 15:57:37] configuration_utils.py:768 >> Model config LlamaConfig {
+  "_name_or_path": "deepseek-ai/deepseek-coder-7b-instruct-v1.5",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100000,
+  "eos_token_id": 100015,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 30,
+  "num_key_value_heads": 32,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.48.2",
+  "use_cache": true,
+  "vocab_size": 102400
+}
+
+
+[INFO|2025-07-09 15:57:38] tokenization_utils_base.py:2034 >> loading file tokenizer.model from cache at None
+
+[INFO|2025-07-09 15:57:38] tokenization_utils_base.py:2034 >> loading file tokenizer.json from cache at /home/kiho/.cache/huggingface/hub/models--deepseek-ai--deepseek-coder-7b-instruct-v1.5/snapshots/2a050a4c59d687a85324d32e147517992117ed30/tokenizer.json
+
+[INFO|2025-07-09 15:57:38] tokenization_utils_base.py:2034 >> loading file added_tokens.json from cache at None
+
+[INFO|2025-07-09 15:57:38] tokenization_utils_base.py:2034 >> loading file special_tokens_map.json from cache at None
+
+[INFO|2025-07-09 15:57:38] tokenization_utils_base.py:2034 >> loading file tokenizer_config.json from cache at /home/kiho/.cache/huggingface/hub/models--deepseek-ai--deepseek-coder-7b-instruct-v1.5/snapshots/2a050a4c59d687a85324d32e147517992117ed30/tokenizer_config.json
+
+[INFO|2025-07-09 15:57:38] tokenization_utils_base.py:2034 >> loading file chat_template.jinja from cache at None
+
+[INFO|2025-07-09 15:57:38] tokenization_utils_base.py:2304 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+[INFO|2025-07-09 15:57:39] configuration_utils.py:696 >> loading configuration file config.json from cache at /home/kiho/.cache/huggingface/hub/models--deepseek-ai--deepseek-coder-7b-instruct-v1.5/snapshots/2a050a4c59d687a85324d32e147517992117ed30/config.json
+
+[INFO|2025-07-09 15:57:39] configuration_utils.py:768 >> Model config LlamaConfig {
+  "_name_or_path": "deepseek-ai/deepseek-coder-7b-instruct-v1.5",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100000,
+  "eos_token_id": 100015,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 30,
+  "num_key_value_heads": 32,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.48.2",
+  "use_cache": true,
+  "vocab_size": 102400
+}
+
+
+[INFO|2025-07-09 15:57:39] tokenization_utils_base.py:2034 >> loading file tokenizer.model from cache at None
+
+[INFO|2025-07-09 15:57:39] tokenization_utils_base.py:2034 >> loading file tokenizer.json from cache at /home/kiho/.cache/huggingface/hub/models--deepseek-ai--deepseek-coder-7b-instruct-v1.5/snapshots/2a050a4c59d687a85324d32e147517992117ed30/tokenizer.json
+
+[INFO|2025-07-09 15:57:39] tokenization_utils_base.py:2034 >> loading file added_tokens.json from cache at None
+
+[INFO|2025-07-09 15:57:39] tokenization_utils_base.py:2034 >> loading file special_tokens_map.json from cache at None
+
+[INFO|2025-07-09 15:57:39] tokenization_utils_base.py:2034 >> loading file tokenizer_config.json from cache at /home/kiho/.cache/huggingface/hub/models--deepseek-ai--deepseek-coder-7b-instruct-v1.5/snapshots/2a050a4c59d687a85324d32e147517992117ed30/tokenizer_config.json
+
+[INFO|2025-07-09 15:57:39] tokenization_utils_base.py:2034 >> loading file chat_template.jinja from cache at None
+
+[INFO|2025-07-09 15:57:39] tokenization_utils_base.py:2304 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+[INFO|2025-07-09 15:57:40] logging.py:157 >> Loading dataset Codes3_query_filtered_553474_mark_less_than_8.0.json...
+
+[INFO|2025-07-09 15:58:18] configuration_utils.py:696 >> loading configuration file config.json from cache at /home/kiho/.cache/huggingface/hub/models--deepseek-ai--deepseek-coder-7b-instruct-v1.5/snapshots/2a050a4c59d687a85324d32e147517992117ed30/config.json
+
+[INFO|2025-07-09 15:58:18] configuration_utils.py:768 >> Model config LlamaConfig {
+  "_name_or_path": "deepseek-ai/deepseek-coder-7b-instruct-v1.5",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100000,
+  "eos_token_id": 100015,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 30,
+  "num_key_value_heads": 32,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.48.2",
+  "use_cache": true,
+  "vocab_size": 102400
+}
+
+
+[WARNING|2025-07-09 15:58:18] logging.py:162 >> Input length is smaller than max length. Consider increase input length.
+
+[INFO|2025-07-09 15:58:18] logging.py:157 >> Using llama3 scaling strategy and setting scaling factor to 1.0.
+
+[INFO|2025-07-09 15:58:18] logging.py:157 >> Using block diagonal attention for sequence packing without cross-attention.
+
+[INFO|2025-07-09 15:58:18] logging.py:157 >> Liger kernel has been applied to the model.
+
+[INFO|2025-07-09 15:58:18] modeling_utils.py:3904 >> loading weights file model.safetensors from cache at /home/kiho/.cache/huggingface/hub/models--deepseek-ai--deepseek-coder-7b-instruct-v1.5/snapshots/2a050a4c59d687a85324d32e147517992117ed30/model.safetensors.index.json
+
+[INFO|2025-07-09 15:58:18] modeling_utils.py:1582 >> Instantiating LlamaForCausalLM model under default dtype torch.bfloat16.
+
+[INFO|2025-07-09 15:58:18] configuration_utils.py:1140 >> Generate config GenerationConfig {
+  "bos_token_id": 100000,
+  "eos_token_id": 100015
+}
+
+
+[INFO|2025-07-09 15:58:26] modeling_utils.py:4888 >> All model checkpoint weights were used when initializing LlamaForCausalLM.
+
+
+[INFO|2025-07-09 15:58:26] modeling_utils.py:4896 >> All the weights of LlamaForCausalLM were initialized from the model checkpoint at deepseek-ai/deepseek-coder-7b-instruct-v1.5.
+If your task is similar to the task the model of the checkpoint was trained on, you can already use LlamaForCausalLM for predictions without further training.
+
+[INFO|2025-07-09 15:58:27] configuration_utils.py:1095 >> loading configuration file generation_config.json from cache at /home/kiho/.cache/huggingface/hub/models--deepseek-ai--deepseek-coder-7b-instruct-v1.5/snapshots/2a050a4c59d687a85324d32e147517992117ed30/generation_config.json
+
+[INFO|2025-07-09 15:58:27] configuration_utils.py:1140 >> Generate config GenerationConfig {
+  "bos_token_id": 100000,
+  "eos_token_id": 100015
+}
+
+
+[INFO|2025-07-09 15:58:27] logging.py:157 >> Gradient checkpointing enabled.
+
+[INFO|2025-07-09 15:58:27] logging.py:157 >> Using torch SDPA for faster training and inference.
+
+[INFO|2025-07-09 15:58:27] logging.py:157 >> Upcasting trainable params to float32.
+
+[INFO|2025-07-09 15:58:27] logging.py:157 >> Fine-tuning method: Freeze
+
+[INFO|2025-07-09 15:58:27] logging.py:157 >> Set trainable layers: .14.,.29.
+
+[INFO|2025-07-09 15:58:27] logging.py:157 >> trainable params: 404,766,720 || all params: 6,910,365,696 || trainable%: 5.8574
+
+[INFO|2025-07-09 15:58:27] trainer.py:741 >> Using auto half precision backend
+
+[INFO|2025-07-09 15:58:27] logging.py:157 >> Found linear modules: k_proj,q_proj,v_proj,down_proj,gate_proj,up_proj,o_proj
+
+[INFO|2025-07-09 15:58:27] logging.py:157 >> Using APOLLO optimizer with args: {'rank': 256, 'proj': 'random', 'proj_type': 'std', 'update_proj_gap': 200, 'scale': 1, 'scale_type': 'channel', 'scale_front': False}.
+
+[INFO|2025-07-09 15:58:27] trainer.py:2369 >> ***** Running training *****
+
+[INFO|2025-07-09 15:58:27] trainer.py:2370 >>   Num examples = 25,964
+
+[INFO|2025-07-09 15:58:27] trainer.py:2371 >>   Num Epochs = 1
+
+[INFO|2025-07-09 15:58:27] trainer.py:2372 >>   Instantaneous batch size per device = 16
+
+[INFO|2025-07-09 15:58:27] trainer.py:2375 >>   Total train batch size (w. parallel, distributed & accumulation) = 384
+
+[INFO|2025-07-09 15:58:27] trainer.py:2376 >>   Gradient Accumulation steps = 8
+
+[INFO|2025-07-09 15:58:27] trainer.py:2377 >>   Total optimization steps = 67
+
+[INFO|2025-07-09 15:58:27] trainer.py:2378 >>   Number of trainable parameters = 404,766,720
+
+[INFO|2025-07-09 16:01:10] logging.py:157 >> {'loss': 1.5801, 'learning_rate': 4.9973e-05, 'epoch': 0.01, 'throughput': 9730.57}
+
+[INFO|2025-07-09 16:03:43] logging.py:157 >> {'loss': 1.3696, 'learning_rate': 4.9890e-05, 'epoch': 0.03, 'throughput': 9996.29}
+
+[INFO|2025-07-09 16:06:16] logging.py:157 >> {'loss': 1.2964, 'learning_rate': 4.9753e-05, 'epoch': 0.04, 'throughput': 10081.95}
+
+[INFO|2025-07-09 16:08:49] logging.py:157 >> {'loss': 1.2465, 'learning_rate': 4.9562e-05, 'epoch': 0.06, 'throughput': 10131.27}
+
+[INFO|2025-07-09 16:11:22] logging.py:157 >> {'loss': 1.1284, 'learning_rate': 4.9316e-05, 'epoch': 0.07, 'throughput': 10163.81}
+
+[INFO|2025-07-09 16:13:55] logging.py:157 >> {'loss': 1.0455, 'learning_rate': 4.9017e-05, 'epoch': 0.09, 'throughput': 10183.83}
+
+[INFO|2025-07-09 16:16:28] logging.py:157 >> {'loss': 0.9850, 'learning_rate': 4.8665e-05, 'epoch': 0.10, 'throughput': 10199.69}
+
+[INFO|2025-07-09 16:19:01] logging.py:157 >> {'loss': 0.9570, 'learning_rate': 4.8262e-05, 'epoch': 0.12, 'throughput': 10208.81}
+
+[INFO|2025-07-09 16:21:34] logging.py:157 >> {'loss': 0.9121, 'learning_rate': 4.7807e-05, 'epoch': 0.13, 'throughput': 10215.43}
+
+[INFO|2025-07-09 16:24:07] logging.py:157 >> {'loss': 0.8857, 'learning_rate': 4.7302e-05, 'epoch': 0.15, 'throughput': 10220.33}
+
+[INFO|2025-07-09 16:26:40] logging.py:157 >> {'loss': 0.8871, 'learning_rate': 4.6748e-05, 'epoch': 0.16, 'throughput': 10224.60}
+
+[INFO|2025-07-09 16:29:14] logging.py:157 >> {'loss': 0.8668, 'learning_rate': 4.6146e-05, 'epoch': 0.18, 'throughput': 10228.07}
+
+[INFO|2025-07-09 16:31:47] logging.py:157 >> {'loss': 0.8264, 'learning_rate': 4.5497e-05, 'epoch': 0.19, 'throughput': 10231.09}
+
+[INFO|2025-07-09 16:34:20] logging.py:157 >> {'loss': 0.8316, 'learning_rate': 4.4804e-05, 'epoch': 0.21, 'throughput': 10233.42}
+
+[INFO|2025-07-09 16:36:53] logging.py:157 >> {'loss': 0.8214, 'learning_rate': 4.4067e-05, 'epoch': 0.22, 'throughput': 10236.31}
+
+[INFO|2025-07-09 16:39:26] logging.py:157 >> {'loss': 0.8074, 'learning_rate': 4.3288e-05, 'epoch': 0.24, 'throughput': 10238.37}
+
+[INFO|2025-07-09 16:41:59] logging.py:157 >> {'loss': 0.7947, 'learning_rate': 4.2469e-05, 'epoch': 0.25, 'throughput': 10239.79}
+
+[INFO|2025-07-09 16:44:33] logging.py:157 >> {'loss': 0.7868, 'learning_rate': 4.1612e-05, 'epoch': 0.27, 'throughput': 10240.87}
+
+[INFO|2025-07-09 16:47:06] logging.py:157 >> {'loss': 0.7657, 'learning_rate': 4.0718e-05, 'epoch': 0.28, 'throughput': 10241.86}
+
+[INFO|2025-07-09 16:49:39] logging.py:157 >> {'loss': 0.7821, 'learning_rate': 3.9789e-05, 'epoch': 0.30, 'throughput': 10242.44}
+
+[INFO|2025-07-09 16:52:13] logging.py:157 >> {'loss': 0.7640, 'learning_rate': 3.8828e-05, 'epoch': 0.31, 'throughput': 10242.80}
+
+[INFO|2025-07-09 16:54:46] logging.py:157 >> {'loss': 0.7781, 'learning_rate': 3.7837e-05, 'epoch': 0.33, 'throughput': 10243.48}
+
+[INFO|2025-07-09 16:57:19] logging.py:157 >> {'loss': 0.7663, 'learning_rate': 3.6817e-05, 'epoch': 0.34, 'throughput': 10244.34}
+
+[INFO|2025-07-09 16:59:53] logging.py:157 >> {'loss': 0.7303, 'learning_rate': 3.5772e-05, 'epoch': 0.35, 'throughput': 10245.23}
+
+[INFO|2025-07-09 17:02:26] logging.py:157 >> {'loss': 0.7626, 'learning_rate': 3.4702e-05, 'epoch': 0.37, 'throughput': 10245.49}
+
+[INFO|2025-07-09 17:04:59] logging.py:157 >> {'loss': 0.7709, 'learning_rate': 3.3612e-05, 'epoch': 0.38, 'throughput': 10246.02}
+
+[INFO|2025-07-09 17:07:33] logging.py:157 >> {'loss': 0.7638, 'learning_rate': 3.2502e-05, 'epoch': 0.40, 'throughput': 10246.34}
+
+[INFO|2025-07-09 17:10:06] logging.py:157 >> {'loss': 0.7561, 'learning_rate': 3.1376e-05, 'epoch': 0.41, 'throughput': 10246.39}
+
+[INFO|2025-07-09 17:12:40] logging.py:157 >> {'loss': 0.7372, 'learning_rate': 3.0236e-05, 'epoch': 0.43, 'throughput': 10246.90}
+
+[INFO|2025-07-09 17:15:13] logging.py:157 >> {'loss': 0.7281, 'learning_rate': 2.9084e-05, 'epoch': 0.44, 'throughput': 10246.62}
+
+[INFO|2025-07-09 17:17:47] logging.py:157 >> {'loss': 0.7601, 'learning_rate': 2.7924e-05, 'epoch': 0.46, 'throughput': 10246.53}
+
+[INFO|2025-07-09 17:20:20] logging.py:157 >> {'loss': 0.7296, 'learning_rate': 2.6757e-05, 'epoch': 0.47, 'throughput': 10246.68}
+
+[INFO|2025-07-09 17:22:54] logging.py:157 >> {'loss': 0.7410, 'learning_rate': 2.5586e-05, 'epoch': 0.49, 'throughput': 10246.53}
+
+[INFO|2025-07-09 17:25:27] logging.py:157 >> {'loss': 0.7251, 'learning_rate': 2.4414e-05, 'epoch': 0.50, 'throughput': 10246.23}
+
+[INFO|2025-07-09 17:28:01] logging.py:157 >> {'loss': 0.7385, 'learning_rate': 2.3243e-05, 'epoch': 0.52, 'throughput': 10246.00}
+
+[INFO|2025-07-09 17:30:35] logging.py:157 >> {'loss': 0.7460, 'learning_rate': 2.2076e-05, 'epoch': 0.53, 'throughput': 10245.76}
+
+[INFO|2025-07-09 17:33:08] logging.py:157 >> {'loss': 0.7315, 'learning_rate': 2.0916e-05, 'epoch': 0.55, 'throughput': 10245.41}
+
+[INFO|2025-07-09 17:35:42] logging.py:157 >> {'loss': 0.7267, 'learning_rate': 1.9764e-05, 'epoch': 0.56, 'throughput': 10245.47}
+
+[INFO|2025-07-09 17:38:15] logging.py:157 >> {'loss': 0.7443, 'learning_rate': 1.8624e-05, 'epoch': 0.58, 'throughput': 10246.13}
+
+[INFO|2025-07-09 17:40:48] logging.py:157 >> {'loss': 0.7163, 'learning_rate': 1.7498e-05, 'epoch': 0.59, 'throughput': 10246.67}
+
+[INFO|2025-07-09 17:43:22] logging.py:157 >> {'loss': 0.7300, 'learning_rate': 1.6388e-05, 'epoch': 0.61, 'throughput': 10246.53}
+
+[INFO|2025-07-09 17:45:55] logging.py:157 >> {'loss': 0.7230, 'learning_rate': 1.5298e-05, 'epoch': 0.62, 'throughput': 10246.38}
+
+[INFO|2025-07-09 17:48:29] logging.py:157 >> {'loss': 0.7176, 'learning_rate': 1.4228e-05, 'epoch': 0.64, 'throughput': 10246.13}
+
+[INFO|2025-07-09 17:51:03] logging.py:157 >> {'loss': 0.7066, 'learning_rate': 1.3183e-05, 'epoch': 0.65, 'throughput': 10245.31}
+
+[INFO|2025-07-09 17:53:37] logging.py:157 >> {'loss': 0.7244, 'learning_rate': 1.2163e-05, 'epoch': 0.67, 'throughput': 10245.33}
+
+[INFO|2025-07-09 17:56:10] logging.py:157 >> {'loss': 0.7349, 'learning_rate': 1.1172e-05, 'epoch': 0.68, 'throughput': 10245.14}
+
+[INFO|2025-07-09 17:58:44] logging.py:157 >> {'loss': 0.7487, 'learning_rate': 1.0211e-05, 'epoch': 0.70, 'throughput': 10245.15}
+
+[INFO|2025-07-09 18:01:17] logging.py:157 >> {'loss': 0.7381, 'learning_rate': 9.2822e-06, 'epoch': 0.71, 'throughput': 10245.27}
+
+[INFO|2025-07-09 18:03:51] logging.py:157 >> {'loss': 0.7262, 'learning_rate': 8.3882e-06, 'epoch': 0.72, 'throughput': 10245.11}
+
+[INFO|2025-07-09 18:06:25] logging.py:157 >> {'loss': 0.7473, 'learning_rate': 7.5308e-06, 'epoch': 0.74, 'throughput': 10244.43}
+
+[INFO|2025-07-09 18:08:58] logging.py:157 >> {'loss': 0.6998, 'learning_rate': 6.7117e-06, 'epoch': 0.75, 'throughput': 10244.28}
+
+[INFO|2025-07-09 18:11:32] logging.py:157 >> {'loss': 0.7218, 'learning_rate': 5.9329e-06, 'epoch': 0.77, 'throughput': 10244.28}
+
+[INFO|2025-07-09 18:14:06] logging.py:157 >> {'loss': 0.7314, 'learning_rate': 5.1959e-06, 'epoch': 0.78, 'throughput': 10244.27}
+
+[INFO|2025-07-09 18:16:39] logging.py:157 >> {'loss': 0.7028, 'learning_rate': 4.5025e-06, 'epoch': 0.80, 'throughput': 10244.46}
+
+[INFO|2025-07-09 18:19:13] logging.py:157 >> {'loss': 0.7084, 'learning_rate': 3.8542e-06, 'epoch': 0.81, 'throughput': 10244.41}
+
+[INFO|2025-07-09 18:21:46] logging.py:157 >> {'loss': 0.7223, 'learning_rate': 3.2523e-06, 'epoch': 0.83, 'throughput': 10244.78}
+
+[INFO|2025-07-09 18:24:19] logging.py:157 >> {'loss': 0.7195, 'learning_rate': 2.6983e-06, 'epoch': 0.84, 'throughput': 10244.84}
+
+[INFO|2025-07-09 18:26:53] logging.py:157 >> {'loss': 0.7431, 'learning_rate': 2.1933e-06, 'epoch': 0.86, 'throughput': 10244.76}
+
+[INFO|2025-07-09 18:29:26] logging.py:157 >> {'loss': 0.7320, 'learning_rate': 1.7384e-06, 'epoch': 0.87, 'throughput': 10245.04}
+
+[INFO|2025-07-09 18:32:00] logging.py:157 >> {'loss': 0.7302, 'learning_rate': 1.3346e-06, 'epoch': 0.89, 'throughput': 10245.15}
+
+[INFO|2025-07-09 18:34:33] logging.py:157 >> {'loss': 0.7197, 'learning_rate': 9.8287e-07, 'epoch': 0.90, 'throughput': 10244.99}
+
+[INFO|2025-07-09 18:37:07] logging.py:157 >> {'loss': 0.7203, 'learning_rate': 6.8393e-07, 'epoch': 0.92, 'throughput': 10244.88}
+
+[INFO|2025-07-09 18:39:40] logging.py:157 >> {'loss': 0.7474, 'learning_rate': 4.3844e-07, 'epoch': 0.93, 'throughput': 10244.81}
+
+[INFO|2025-07-09 18:42:14] logging.py:157 >> {'loss': 0.7403, 'learning_rate': 2.4694e-07, 'epoch': 0.95, 'throughput': 10244.76}
+
+[INFO|2025-07-09 18:44:48] logging.py:157 >> {'loss': 0.7327, 'learning_rate': 1.0985e-07, 'epoch': 0.96, 'throughput': 10244.72}
+
+[INFO|2025-07-09 18:47:21] logging.py:157 >> {'loss': 0.7250, 'learning_rate': 2.7478e-08, 'epoch': 0.98, 'throughput': 10244.49}
+
+[INFO|2025-07-09 18:49:55] logging.py:157 >> {'loss': 0.7209, 'learning_rate': 0.0000e+00, 'epoch': 0.99, 'throughput': 10244.41}
+
+[INFO|2025-07-09 18:49:55] trainer.py:3910 >> Saving model checkpoint to saves/DeepSeek-Coder-7B-Instruct/freeze/deepseek_under8_nsx/checkpoint-67
+
+[INFO|2025-07-09 18:49:55] configuration_utils.py:420 >> Configuration saved in saves/DeepSeek-Coder-7B-Instruct/freeze/deepseek_under8_nsx/checkpoint-67/config.json
+
+[INFO|2025-07-09 18:49:55] configuration_utils.py:909 >> Configuration saved in saves/DeepSeek-Coder-7B-Instruct/freeze/deepseek_under8_nsx/checkpoint-67/generation_config.json
+
+[INFO|2025-07-09 18:50:16] modeling_utils.py:2996 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 3 checkpoint shards. You can find where each parameters has been saved in the index located at saves/DeepSeek-Coder-7B-Instruct/freeze/deepseek_under8_nsx/checkpoint-67/model.safetensors.index.json.
+
+[INFO|2025-07-09 18:50:16] tokenization_utils_base.py:2491 >> tokenizer config file saved in saves/DeepSeek-Coder-7B-Instruct/freeze/deepseek_under8_nsx/checkpoint-67/tokenizer_config.json
+
+[INFO|2025-07-09 18:50:16] tokenization_utils_base.py:2500 >> Special tokens file saved in saves/DeepSeek-Coder-7B-Instruct/freeze/deepseek_under8_nsx/checkpoint-67/special_tokens_map.json
+
+[INFO|2025-07-09 18:50:17] trainer.py:2643 >> 
+
+Training completed. Do not forget to share your model on huggingface.co/models =)
+
+
+
+[INFO|2025-07-09 18:50:17] trainer.py:3910 >> Saving model checkpoint to saves/DeepSeek-Coder-7B-Instruct/freeze/deepseek_under8_nsx
+
+[INFO|2025-07-09 18:50:17] configuration_utils.py:420 >> Configuration saved in saves/DeepSeek-Coder-7B-Instruct/freeze/deepseek_under8_nsx/config.json
+
+[INFO|2025-07-09 18:50:17] configuration_utils.py:909 >> Configuration saved in saves/DeepSeek-Coder-7B-Instruct/freeze/deepseek_under8_nsx/generation_config.json
+
+[INFO|2025-07-09 18:50:39] modeling_utils.py:2996 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 3 checkpoint shards. You can find where each parameters has been saved in the index located at saves/DeepSeek-Coder-7B-Instruct/freeze/deepseek_under8_nsx/model.safetensors.index.json.
+
+[INFO|2025-07-09 18:50:39] tokenization_utils_base.py:2491 >> tokenizer config file saved in saves/DeepSeek-Coder-7B-Instruct/freeze/deepseek_under8_nsx/tokenizer_config.json
+
+[INFO|2025-07-09 18:50:39] tokenization_utils_base.py:2500 >> Special tokens file saved in saves/DeepSeek-Coder-7B-Instruct/freeze/deepseek_under8_nsx/special_tokens_map.json
+
+[WARNING|2025-07-09 18:50:39] logging.py:162 >> No metric eval_loss to plot.
+
+[WARNING|2025-07-09 18:50:39] logging.py:162 >> No metric eval_accuracy to plot.
+
+[INFO|2025-07-09 18:50:39] modelcard.py:449 >> Dropping the following result as it does not have all the necessary fields:
+{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}
+

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<｜begin▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|EOT|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<｜end▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,149 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "100000": {
+      "content": "<｜begin▁of▁sentence｜>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100001": {
+      "content": "<｜end▁of▁sentence｜>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100002": {
+      "content": "ø",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100003": {
+      "content": "ö",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100004": {
+      "content": "ú",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100005": {
+      "content": "ÿ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100006": {
+      "content": "õ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100007": {
+      "content": "÷",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100008": {
+      "content": "û",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100009": {
+      "content": "ý",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100010": {
+      "content": "À",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100011": {
+      "content": "ù",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100012": {
+      "content": "Á",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100013": {
+      "content": "þ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100014": {
+      "content": "ü",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100015": {
+      "content": "<|EOT|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<｜begin▁of▁sentence｜>",
+  "chat_template": "{% if not add_generation_prompt is defined %}\n{% set add_generation_prompt = false %}\n{% endif %}\n{%- set ns = namespace(found=false) -%}\n{%- for message in messages -%}\n    {%- if message['role'] == 'system' -%}\n        {%- set ns.found = true -%}\n    {%- endif -%}\n{%- endfor -%}\n{{bos_token}}{%- if not ns.found -%}\n{{'You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\\n'}}\n{%- endif %}\n{%- for message in messages %}\n    {%- if message['role'] == 'system' %}\n{{ message['content'] }}\n    {%- else %}\n        {%- if message['role'] == 'user' %}\n{{'### Instruction:\\n' + message['content'] + '\\n'}}\n        {%- else %}\n{{'### Response:\\n' + message['content'] + '\\n<|EOT|>\\n'}}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{% if add_generation_prompt %}\n{{'### Response:'}}\n{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|EOT|>",
+  "extra_special_tokens": {},
+  "legacy": true,
+  "model_max_length": 4096,
+  "pad_token": "<｜end▁of▁sentence｜>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "split_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": null,
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9907578558225508,
+    "num_input_tokens_seen": 105381888,
+    "total_flos": 4.104162098269913e+18,
+    "train_loss": 0.8075656152483243,
+    "train_runtime": 10309.5741,
+    "train_samples_per_second": 2.518,
+    "train_steps_per_second": 0.006
+}

trainer_log.jsonl

@@ -0,0 +1,68 @@
+{"current_steps": 1, "total_steps": 67, "loss": 1.5801, "lr": 4.997252228714279e-05, "epoch": 0.014787430683918669, "percentage": 1.49, "elapsed_time": "0:02:41", "remaining_time": "2:57:48", "throughput": 9730.57, "total_tokens": 1572864}
+{"current_steps": 2, "total_steps": 67, "loss": 1.3696, "lr": 4.9890149550547454e-05, "epoch": 0.029574861367837338, "percentage": 2.99, "elapsed_time": "0:05:14", "remaining_time": "2:50:27", "throughput": 9996.29, "total_tokens": 3145728}
+{"current_steps": 3, "total_steps": 67, "loss": 1.2964, "lr": 4.9753062863366276e-05, "epoch": 0.04436229205175601, "percentage": 4.48, "elapsed_time": "0:07:48", "remaining_time": "2:46:24", "throughput": 10081.95, "total_tokens": 4718592}
+{"current_steps": 4, "total_steps": 67, "loss": 1.2465, "lr": 4.95615635718894e-05, "epoch": 0.059149722735674676, "percentage": 5.97, "elapsed_time": "0:10:20", "remaining_time": "2:43:00", "throughput": 10131.27, "total_tokens": 6291456}
+{"current_steps": 5, "total_steps": 67, "loss": 1.1284, "lr": 4.931607263312032e-05, "epoch": 0.07393715341959335, "percentage": 7.46, "elapsed_time": "0:12:53", "remaining_time": "2:39:54", "throughput": 10163.81, "total_tokens": 7864320}
+{"current_steps": 6, "total_steps": 67, "loss": 1.0455, "lr": 4.9017129689421e-05, "epoch": 0.08872458410351201, "percentage": 8.96, "elapsed_time": "0:15:26", "remaining_time": "2:37:01", "throughput": 10183.83, "total_tokens": 9437184}
+{"current_steps": 7, "total_steps": 67, "loss": 0.985, "lr": 4.8665391882260856e-05, "epoch": 0.10351201478743069, "percentage": 10.45, "elapsed_time": "0:17:59", "remaining_time": "2:34:12", "throughput": 10199.69, "total_tokens": 11010048}
+{"current_steps": 8, "total_steps": 67, "loss": 0.957, "lr": 4.8261632407677174e-05, "epoch": 0.11829944547134935, "percentage": 11.94, "elapsed_time": "0:20:32", "remaining_time": "2:31:30", "throughput": 10208.81, "total_tokens": 12582912}
+{"current_steps": 9, "total_steps": 67, "loss": 0.9121, "lr": 4.780673881662242e-05, "epoch": 0.133086876155268, "percentage": 13.43, "elapsed_time": "0:23:05", "remaining_time": "2:28:50", "throughput": 10215.43, "total_tokens": 14155776}
+{"current_steps": 10, "total_steps": 67, "loss": 0.8857, "lr": 4.730171106393466e-05, "epoch": 0.1478743068391867, "percentage": 14.93, "elapsed_time": "0:25:38", "remaining_time": "2:26:12", "throughput": 10220.33, "total_tokens": 15728640}
+{"current_steps": 11, "total_steps": 67, "loss": 0.8871, "lr": 4.674765931021976e-05, "epoch": 0.16266173752310537, "percentage": 16.42, "elapsed_time": "0:28:12", "remaining_time": "2:23:34", "throughput": 10224.6, "total_tokens": 17301504}
+{"current_steps": 12, "total_steps": 67, "loss": 0.8668, "lr": 4.614580148147744e-05, "epoch": 0.17744916820702403, "percentage": 17.91, "elapsed_time": "0:30:45", "remaining_time": "2:20:57", "throughput": 10228.07, "total_tokens": 18874368}
+{"current_steps": 13, "total_steps": 67, "loss": 0.8264, "lr": 4.5497460591835615e-05, "epoch": 0.1922365988909427, "percentage": 19.4, "elapsed_time": "0:33:18", "remaining_time": "2:18:21", "throughput": 10231.09, "total_tokens": 20447232}
+{"current_steps": 14, "total_steps": 67, "loss": 0.8316, "lr": 4.480406183527823e-05, "epoch": 0.20702402957486138, "percentage": 20.9, "elapsed_time": "0:35:51", "remaining_time": "2:15:46", "throughput": 10233.42, "total_tokens": 22020096}
+{"current_steps": 15, "total_steps": 67, "loss": 0.8214, "lr": 4.406712945275955e-05, "epoch": 0.22181146025878004, "percentage": 22.39, "elapsed_time": "0:38:24", "remaining_time": "2:13:10", "throughput": 10236.31, "total_tokens": 23592960}
+{"current_steps": 16, "total_steps": 67, "loss": 0.8074, "lr": 4.328828338159173e-05, "epoch": 0.2365988909426987, "percentage": 23.88, "elapsed_time": "0:40:57", "remaining_time": "2:10:34", "throughput": 10238.37, "total_tokens": 25165824}
+{"current_steps": 17, "total_steps": 67, "loss": 0.7947, "lr": 4.2469235694471043e-05, "epoch": 0.2513863216266174, "percentage": 25.37, "elapsed_time": "0:43:31", "remaining_time": "2:08:00", "throughput": 10239.79, "total_tokens": 26738688}
+{"current_steps": 18, "total_steps": 67, "loss": 0.7868, "lr": 4.161178683597054e-05, "epoch": 0.266173752310536, "percentage": 26.87, "elapsed_time": "0:46:04", "remaining_time": "2:05:25", "throughput": 10240.87, "total_tokens": 28311552}
+{"current_steps": 19, "total_steps": 67, "loss": 0.7657, "lr": 4.071782166477213e-05, "epoch": 0.2809611829944547, "percentage": 28.36, "elapsed_time": "0:48:37", "remaining_time": "2:02:51", "throughput": 10241.86, "total_tokens": 29884416}
+{"current_steps": 20, "total_steps": 67, "loss": 0.7821, "lr": 3.978930531033807e-05, "epoch": 0.2957486136783734, "percentage": 29.85, "elapsed_time": "0:51:11", "remaining_time": "2:00:17", "throughput": 10242.44, "total_tokens": 31457280}
+{"current_steps": 21, "total_steps": 67, "loss": 0.764, "lr": 3.882827885312999e-05, "epoch": 0.31053604436229204, "percentage": 31.34, "elapsed_time": "0:53:44", "remaining_time": "1:57:43", "throughput": 10242.8, "total_tokens": 33030144}
+{"current_steps": 22, "total_steps": 67, "loss": 0.7781, "lr": 3.783685483787105e-05, "epoch": 0.32532347504621073, "percentage": 32.84, "elapsed_time": "0:56:18", "remaining_time": "1:55:09", "throughput": 10243.48, "total_tokens": 34603008}
+{"current_steps": 23, "total_steps": 67, "loss": 0.7663, "lr": 3.681721262971413e-05, "epoch": 0.34011090573012936, "percentage": 34.33, "elapsed_time": "0:58:51", "remaining_time": "1:52:35", "throughput": 10244.34, "total_tokens": 36175872}
+{"current_steps": 24, "total_steps": 67, "loss": 0.7303, "lr": 3.5771593623524265e-05, "epoch": 0.35489833641404805, "percentage": 35.82, "elapsed_time": "1:01:24", "remaining_time": "1:50:01", "throughput": 10245.23, "total_tokens": 37748736}
+{"current_steps": 25, "total_steps": 67, "loss": 0.7626, "lr": 3.4702296316806244e-05, "epoch": 0.36968576709796674, "percentage": 37.31, "elapsed_time": "1:03:57", "remaining_time": "1:47:27", "throughput": 10245.49, "total_tokens": 39321600}
+{"current_steps": 26, "total_steps": 67, "loss": 0.7709, "lr": 3.361167125710832e-05, "epoch": 0.3844731977818854, "percentage": 38.81, "elapsed_time": "1:06:31", "remaining_time": "1:44:53", "throughput": 10246.02, "total_tokens": 40894464}
+{"current_steps": 27, "total_steps": 67, "loss": 0.7638, "lr": 3.2502115875008524e-05, "epoch": 0.39926062846580407, "percentage": 40.3, "elapsed_time": "1:09:04", "remaining_time": "1:42:20", "throughput": 10246.34, "total_tokens": 42467328}
+{"current_steps": 28, "total_steps": 67, "loss": 0.7561, "lr": 3.1376069214041913e-05, "epoch": 0.41404805914972276, "percentage": 41.79, "elapsed_time": "1:11:38", "remaining_time": "1:39:46", "throughput": 10246.39, "total_tokens": 44040192}
+{"current_steps": 29, "total_steps": 67, "loss": 0.7372, "lr": 3.0236006569153617e-05, "epoch": 0.4288354898336414, "percentage": 43.28, "elapsed_time": "1:14:11", "remaining_time": "1:37:12", "throughput": 10246.9, "total_tokens": 45613056}
+{"current_steps": 30, "total_steps": 67, "loss": 0.7281, "lr": 2.9084434045463255e-05, "epoch": 0.4436229205175601, "percentage": 44.78, "elapsed_time": "1:16:45", "remaining_time": "1:34:39", "throughput": 10246.62, "total_tokens": 47185920}
+{"current_steps": 31, "total_steps": 67, "loss": 0.7601, "lr": 2.792388304930207e-05, "epoch": 0.4584103512014787, "percentage": 46.27, "elapsed_time": "1:19:18", "remaining_time": "1:32:06", "throughput": 10246.53, "total_tokens": 48758784}
+{"current_steps": 32, "total_steps": 67, "loss": 0.7296, "lr": 2.6756904723632324e-05, "epoch": 0.4731977818853974, "percentage": 47.76, "elapsed_time": "1:21:51", "remaining_time": "1:29:32", "throughput": 10246.68, "total_tokens": 50331648}
+{"current_steps": 33, "total_steps": 67, "loss": 0.741, "lr": 2.5586064340081516e-05, "epoch": 0.4879852125693161, "percentage": 49.25, "elapsed_time": "1:24:25", "remaining_time": "1:26:59", "throughput": 10246.53, "total_tokens": 51904512}
+{"current_steps": 34, "total_steps": 67, "loss": 0.7251, "lr": 2.441393565991849e-05, "epoch": 0.5027726432532348, "percentage": 50.75, "elapsed_time": "1:26:59", "remaining_time": "1:24:25", "throughput": 10246.23, "total_tokens": 53477376}
+{"current_steps": 35, "total_steps": 67, "loss": 0.7385, "lr": 2.3243095276367685e-05, "epoch": 0.5175600739371534, "percentage": 52.24, "elapsed_time": "1:29:32", "remaining_time": "1:21:52", "throughput": 10246.0, "total_tokens": 55050240}
+{"current_steps": 36, "total_steps": 67, "loss": 0.746, "lr": 2.207611695069794e-05, "epoch": 0.532347504621072, "percentage": 53.73, "elapsed_time": "1:32:06", "remaining_time": "1:19:18", "throughput": 10245.76, "total_tokens": 56623104}
+{"current_steps": 37, "total_steps": 67, "loss": 0.7315, "lr": 2.0915565954536744e-05, "epoch": 0.5471349353049908, "percentage": 55.22, "elapsed_time": "1:34:40", "remaining_time": "1:16:45", "throughput": 10245.41, "total_tokens": 58195968}
+{"current_steps": 38, "total_steps": 67, "loss": 0.7267, "lr": 1.9763993430846395e-05, "epoch": 0.5619223659889094, "percentage": 56.72, "elapsed_time": "1:37:13", "remaining_time": "1:14:12", "throughput": 10245.47, "total_tokens": 59768832}
+{"current_steps": 39, "total_steps": 67, "loss": 0.7443, "lr": 1.8623930785958092e-05, "epoch": 0.5767097966728281, "percentage": 58.21, "elapsed_time": "1:39:46", "remaining_time": "1:11:38", "throughput": 10246.13, "total_tokens": 61341696}
+{"current_steps": 40, "total_steps": 67, "loss": 0.7163, "lr": 1.749788412499149e-05, "epoch": 0.5914972273567468, "percentage": 59.7, "elapsed_time": "1:42:19", "remaining_time": "1:09:04", "throughput": 10246.67, "total_tokens": 62914560}
+{"current_steps": 41, "total_steps": 67, "loss": 0.73, "lr": 1.638832874289168e-05, "epoch": 0.6062846580406654, "percentage": 61.19, "elapsed_time": "1:44:53", "remaining_time": "1:06:31", "throughput": 10246.53, "total_tokens": 64487424}
+{"current_steps": 42, "total_steps": 67, "loss": 0.723, "lr": 1.5297703683193752e-05, "epoch": 0.6210720887245841, "percentage": 62.69, "elapsed_time": "1:47:27", "remaining_time": "1:03:57", "throughput": 10246.38, "total_tokens": 66060288}
+{"current_steps": 43, "total_steps": 67, "loss": 0.7176, "lr": 1.4228406376475742e-05, "epoch": 0.6358595194085028, "percentage": 64.18, "elapsed_time": "1:50:00", "remaining_time": "1:01:24", "throughput": 10246.13, "total_tokens": 67633152}
+{"current_steps": 44, "total_steps": 67, "loss": 0.7066, "lr": 1.3182787370285865e-05, "epoch": 0.6506469500924215, "percentage": 65.67, "elapsed_time": "1:52:34", "remaining_time": "0:58:50", "throughput": 10245.31, "total_tokens": 69206016}
+{"current_steps": 45, "total_steps": 67, "loss": 0.7244, "lr": 1.2163145162128947e-05, "epoch": 0.6654343807763401, "percentage": 67.16, "elapsed_time": "1:55:08", "remaining_time": "0:56:17", "throughput": 10245.33, "total_tokens": 70778880}
+{"current_steps": 46, "total_steps": 67, "loss": 0.7349, "lr": 1.1171721146870015e-05, "epoch": 0.6802218114602587, "percentage": 68.66, "elapsed_time": "1:57:42", "remaining_time": "0:53:43", "throughput": 10245.14, "total_tokens": 72351744}
+{"current_steps": 47, "total_steps": 67, "loss": 0.7487, "lr": 1.021069468966194e-05, "epoch": 0.6950092421441775, "percentage": 70.15, "elapsed_time": "2:00:15", "remaining_time": "0:51:10", "throughput": 10245.15, "total_tokens": 73924608}
+{"current_steps": 48, "total_steps": 67, "loss": 0.7381, "lr": 9.282178335227884e-06, "epoch": 0.7097966728280961, "percentage": 71.64, "elapsed_time": "2:02:49", "remaining_time": "0:48:36", "throughput": 10245.27, "total_tokens": 75497472}
+{"current_steps": 49, "total_steps": 67, "loss": 0.7262, "lr": 8.38821316402946e-06, "epoch": 0.7245841035120147, "percentage": 73.13, "elapsed_time": "2:05:22", "remaining_time": "0:46:03", "throughput": 10245.11, "total_tokens": 77070336}
+{"current_steps": 50, "total_steps": 67, "loss": 0.7473, "lr": 7.530764305528959e-06, "epoch": 0.7393715341959335, "percentage": 74.63, "elapsed_time": "2:07:56", "remaining_time": "0:43:30", "throughput": 10244.43, "total_tokens": 78643200}
+{"current_steps": 51, "total_steps": 67, "loss": 0.6998, "lr": 6.711716618408281e-06, "epoch": 0.7541589648798521, "percentage": 76.12, "elapsed_time": "2:10:30", "remaining_time": "0:40:56", "throughput": 10244.28, "total_tokens": 80216064}
+{"current_steps": 52, "total_steps": 67, "loss": 0.7218, "lr": 5.932870547240454e-06, "epoch": 0.7689463955637708, "percentage": 77.61, "elapsed_time": "2:13:03", "remaining_time": "0:38:23", "throughput": 10244.28, "total_tokens": 81788928}
+{"current_steps": 53, "total_steps": 67, "loss": 0.7314, "lr": 5.1959381647217666e-06, "epoch": 0.7837338262476895, "percentage": 79.1, "elapsed_time": "2:15:37", "remaining_time": "0:35:49", "throughput": 10244.27, "total_tokens": 83361792}
+{"current_steps": 54, "total_steps": 67, "loss": 0.7028, "lr": 4.502539408164386e-06, "epoch": 0.7985212569316081, "percentage": 80.6, "elapsed_time": "2:18:10", "remaining_time": "0:33:15", "throughput": 10244.46, "total_tokens": 84934656}
+{"current_steps": 55, "total_steps": 67, "loss": 0.7084, "lr": 3.8541985185225645e-06, "epoch": 0.8133086876155268, "percentage": 82.09, "elapsed_time": "2:20:44", "remaining_time": "0:30:42", "throughput": 10244.41, "total_tokens": 86507520}
+{"current_steps": 56, "total_steps": 67, "loss": 0.7223, "lr": 3.252340689780245e-06, "epoch": 0.8280961182994455, "percentage": 83.58, "elapsed_time": "2:23:17", "remaining_time": "0:28:08", "throughput": 10244.78, "total_tokens": 88080384}
+{"current_steps": 57, "total_steps": 67, "loss": 0.7195, "lr": 2.6982889360653377e-06, "epoch": 0.8428835489833642, "percentage": 85.07, "elapsed_time": "2:25:51", "remaining_time": "0:25:35", "throughput": 10244.84, "total_tokens": 89653248}
+{"current_steps": 58, "total_steps": 67, "loss": 0.7431, "lr": 2.1932611833775846e-06, "epoch": 0.8576709796672828, "percentage": 86.57, "elapsed_time": "2:28:24", "remaining_time": "0:23:01", "throughput": 10244.76, "total_tokens": 91226112}
+{"current_steps": 59, "total_steps": 67, "loss": 0.732, "lr": 1.738367592322837e-06, "epoch": 0.8724584103512015, "percentage": 88.06, "elapsed_time": "2:30:57", "remaining_time": "0:20:28", "throughput": 10245.04, "total_tokens": 92798976}
+{"current_steps": 60, "total_steps": 67, "loss": 0.7302, "lr": 1.3346081177391472e-06, "epoch": 0.8872458410351202, "percentage": 89.55, "elapsed_time": "2:33:31", "remaining_time": "0:17:54", "throughput": 10245.15, "total_tokens": 94371840}
+{"current_steps": 61, "total_steps": 67, "loss": 0.7197, "lr": 9.828703105789983e-07, "epoch": 0.9020332717190388, "percentage": 91.04, "elapsed_time": "2:36:05", "remaining_time": "0:15:21", "throughput": 10244.99, "total_tokens": 95944704}
+{"current_steps": 62, "total_steps": 67, "loss": 0.7203, "lr": 6.839273668796747e-07, "epoch": 0.9168207024029574, "percentage": 92.54, "elapsed_time": "2:38:38", "remaining_time": "0:12:47", "throughput": 10244.88, "total_tokens": 97517568}
+{"current_steps": 63, "total_steps": 67, "loss": 0.7474, "lr": 4.3843642811059737e-07, "epoch": 0.9316081330868762, "percentage": 94.03, "elapsed_time": "2:41:12", "remaining_time": "0:10:14", "throughput": 10244.81, "total_tokens": 99090432}
+{"current_steps": 64, "total_steps": 67, "loss": 0.7403, "lr": 2.4693713663372644e-07, "epoch": 0.9463955637707948, "percentage": 95.52, "elapsed_time": "2:43:45", "remaining_time": "0:07:40", "throughput": 10244.76, "total_tokens": 100663296}
+{"current_steps": 65, "total_steps": 67, "loss": 0.7327, "lr": 1.0985044945254764e-07, "epoch": 0.9611829944547134, "percentage": 97.01, "elapsed_time": "2:46:19", "remaining_time": "0:05:07", "throughput": 10244.72, "total_tokens": 102236160}
+{"current_steps": 66, "total_steps": 67, "loss": 0.725, "lr": 2.7477712857215677e-08, "epoch": 0.9759704251386322, "percentage": 98.51, "elapsed_time": "2:48:53", "remaining_time": "0:02:33", "throughput": 10244.49, "total_tokens": 103809024}
+{"current_steps": 67, "total_steps": 67, "loss": 0.7209, "lr": 0.0, "epoch": 0.9907578558225508, "percentage": 100.0, "elapsed_time": "2:51:26", "remaining_time": "0:00:00", "throughput": 10244.41, "total_tokens": 105381888}
+{"current_steps": 67, "total_steps": 67, "epoch": 0.9907578558225508, "percentage": 100.0, "elapsed_time": "2:51:48", "remaining_time": "0:00:00", "throughput": 10222.76, "total_tokens": 105381888}

trainer_state.json

@@ -0,0 +1,579 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9907578558225508,
+  "eval_steps": 500,
+  "global_step": 67,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.014787430683918669,
+      "grad_norm": 2.2905805110931396,
+      "learning_rate": 4.997252228714279e-05,
+      "loss": 1.5801,
+      "num_input_tokens_seen": 1572864,
+      "step": 1
+    },
+    {
+      "epoch": 0.029574861367837338,
+      "grad_norm": 1.8597325086593628,
+      "learning_rate": 4.9890149550547454e-05,
+      "loss": 1.3696,
+      "num_input_tokens_seen": 3145728,
+      "step": 2
+    },
+    {
+      "epoch": 0.04436229205175601,
+      "grad_norm": 1.7161543369293213,
+      "learning_rate": 4.9753062863366276e-05,
+      "loss": 1.2964,
+      "num_input_tokens_seen": 4718592,
+      "step": 3
+    },
+    {
+      "epoch": 0.059149722735674676,
+      "grad_norm": 1.556989073753357,
+      "learning_rate": 4.95615635718894e-05,
+      "loss": 1.2465,
+      "num_input_tokens_seen": 6291456,
+      "step": 4
+    },
+    {
+      "epoch": 0.07393715341959335,
+      "grad_norm": 1.2705891132354736,
+      "learning_rate": 4.931607263312032e-05,
+      "loss": 1.1284,
+      "num_input_tokens_seen": 7864320,
+      "step": 5
+    },
+    {
+      "epoch": 0.08872458410351201,
+      "grad_norm": 1.1708149909973145,
+      "learning_rate": 4.9017129689421e-05,
+      "loss": 1.0455,
+      "num_input_tokens_seen": 9437184,
+      "step": 6
+    },
+    {
+      "epoch": 0.10351201478743069,
+      "grad_norm": 1.0333069562911987,
+      "learning_rate": 4.8665391882260856e-05,
+      "loss": 0.985,
+      "num_input_tokens_seen": 11010048,
+      "step": 7
+    },
+    {
+      "epoch": 0.11829944547134935,
+      "grad_norm": 1.0009125471115112,
+      "learning_rate": 4.8261632407677174e-05,
+      "loss": 0.957,
+      "num_input_tokens_seen": 12582912,
+      "step": 8
+    },
+    {
+      "epoch": 0.133086876155268,
+      "grad_norm": 0.7598341107368469,
+      "learning_rate": 4.780673881662242e-05,
+      "loss": 0.9121,
+      "num_input_tokens_seen": 14155776,
+      "step": 9
+    },
+    {
+      "epoch": 0.1478743068391867,
+      "grad_norm": 0.366968035697937,
+      "learning_rate": 4.730171106393466e-05,
+      "loss": 0.8857,
+      "num_input_tokens_seen": 15728640,
+      "step": 10
+    },
+    {
+      "epoch": 0.16266173752310537,
+      "grad_norm": 0.2760382890701294,
+      "learning_rate": 4.674765931021976e-05,
+      "loss": 0.8871,
+      "num_input_tokens_seen": 17301504,
+      "step": 11
+    },
+    {
+      "epoch": 0.17744916820702403,
+      "grad_norm": 0.23743785917758942,
+      "learning_rate": 4.614580148147744e-05,
+      "loss": 0.8668,
+      "num_input_tokens_seen": 18874368,
+      "step": 12
+    },
+    {
+      "epoch": 0.1922365988909427,
+      "grad_norm": 0.21216906607151031,
+      "learning_rate": 4.5497460591835615e-05,
+      "loss": 0.8264,
+      "num_input_tokens_seen": 20447232,
+      "step": 13
+    },
+    {
+      "epoch": 0.20702402957486138,
+      "grad_norm": 0.202525332570076,
+      "learning_rate": 4.480406183527823e-05,
+      "loss": 0.8316,
+      "num_input_tokens_seen": 22020096,
+      "step": 14
+    },
+    {
+      "epoch": 0.22181146025878004,
+      "grad_norm": 0.1785624921321869,
+      "learning_rate": 4.406712945275955e-05,
+      "loss": 0.8214,
+      "num_input_tokens_seen": 23592960,
+      "step": 15
+    },
+    {
+      "epoch": 0.2365988909426987,
+      "grad_norm": 0.17254720628261566,
+      "learning_rate": 4.328828338159173e-05,
+      "loss": 0.8074,
+      "num_input_tokens_seen": 25165824,
+      "step": 16
+    },
+    {
+      "epoch": 0.2513863216266174,
+      "grad_norm": 0.15905120968818665,
+      "learning_rate": 4.2469235694471043e-05,
+      "loss": 0.7947,
+      "num_input_tokens_seen": 26738688,
+      "step": 17
+    },
+    {
+      "epoch": 0.266173752310536,
+      "grad_norm": 0.13924936950206757,
+      "learning_rate": 4.161178683597054e-05,
+      "loss": 0.7868,
+      "num_input_tokens_seen": 28311552,
+      "step": 18
+    },
+    {
+      "epoch": 0.2809611829944547,
+      "grad_norm": 0.12500160932540894,
+      "learning_rate": 4.071782166477213e-05,
+      "loss": 0.7657,
+      "num_input_tokens_seen": 29884416,
+      "step": 19
+    },
+    {
+      "epoch": 0.2957486136783734,
+      "grad_norm": 0.11662258207798004,
+      "learning_rate": 3.978930531033807e-05,
+      "loss": 0.7821,
+      "num_input_tokens_seen": 31457280,
+      "step": 20
+    },
+    {
+      "epoch": 0.31053604436229204,
+      "grad_norm": 0.10671637207269669,
+      "learning_rate": 3.882827885312999e-05,
+      "loss": 0.764,
+      "num_input_tokens_seen": 33030144,
+      "step": 21
+    },
+    {
+      "epoch": 0.32532347504621073,
+      "grad_norm": 0.10863461345434189,
+      "learning_rate": 3.783685483787105e-05,
+      "loss": 0.7781,
+      "num_input_tokens_seen": 34603008,
+      "step": 22
+    },
+    {
+      "epoch": 0.34011090573012936,
+      "grad_norm": 0.09509966522455215,
+      "learning_rate": 3.681721262971413e-05,
+      "loss": 0.7663,
+      "num_input_tokens_seen": 36175872,
+      "step": 23
+    },
+    {
+      "epoch": 0.35489833641404805,
+      "grad_norm": 0.08742663264274597,
+      "learning_rate": 3.5771593623524265e-05,
+      "loss": 0.7303,
+      "num_input_tokens_seen": 37748736,
+      "step": 24
+    },
+    {
+      "epoch": 0.36968576709796674,
+      "grad_norm": 0.09325291961431503,
+      "learning_rate": 3.4702296316806244e-05,
+      "loss": 0.7626,
+      "num_input_tokens_seen": 39321600,
+      "step": 25
+    },
+    {
+      "epoch": 0.3844731977818854,
+      "grad_norm": 0.08703341335058212,
+      "learning_rate": 3.361167125710832e-05,
+      "loss": 0.7709,
+      "num_input_tokens_seen": 40894464,
+      "step": 26
+    },
+    {
+      "epoch": 0.39926062846580407,
+      "grad_norm": 0.09138187021017075,
+      "learning_rate": 3.2502115875008524e-05,
+      "loss": 0.7638,
+      "num_input_tokens_seen": 42467328,
+      "step": 27
+    },
+    {
+      "epoch": 0.41404805914972276,
+      "grad_norm": 0.08183719217777252,
+      "learning_rate": 3.1376069214041913e-05,
+      "loss": 0.7561,
+      "num_input_tokens_seen": 44040192,
+      "step": 28
+    },
+    {
+      "epoch": 0.4288354898336414,
+      "grad_norm": 0.08267659693956375,
+      "learning_rate": 3.0236006569153617e-05,
+      "loss": 0.7372,
+      "num_input_tokens_seen": 45613056,
+      "step": 29
+    },
+    {
+      "epoch": 0.4436229205175601,
+      "grad_norm": 0.07969928532838821,
+      "learning_rate": 2.9084434045463255e-05,
+      "loss": 0.7281,
+      "num_input_tokens_seen": 47185920,
+      "step": 30
+    },
+    {
+      "epoch": 0.4584103512014787,
+      "grad_norm": 0.07706066220998764,
+      "learning_rate": 2.792388304930207e-05,
+      "loss": 0.7601,
+      "num_input_tokens_seen": 48758784,
+      "step": 31
+    },
+    {
+      "epoch": 0.4731977818853974,
+      "grad_norm": 0.07695123553276062,
+      "learning_rate": 2.6756904723632324e-05,
+      "loss": 0.7296,
+      "num_input_tokens_seen": 50331648,
+      "step": 32
+    },
+    {
+      "epoch": 0.4879852125693161,
+      "grad_norm": 0.07787525653839111,
+      "learning_rate": 2.5586064340081516e-05,
+      "loss": 0.741,
+      "num_input_tokens_seen": 51904512,
+      "step": 33
+    },
+    {
+      "epoch": 0.5027726432532348,
+      "grad_norm": 0.08225582540035248,
+      "learning_rate": 2.441393565991849e-05,
+      "loss": 0.7251,
+      "num_input_tokens_seen": 53477376,
+      "step": 34
+    },
+    {
+      "epoch": 0.5175600739371534,
+      "grad_norm": 0.0731961578130722,
+      "learning_rate": 2.3243095276367685e-05,
+      "loss": 0.7385,
+      "num_input_tokens_seen": 55050240,
+      "step": 35
+    },
+    {
+      "epoch": 0.532347504621072,
+      "grad_norm": 0.08208758383989334,
+      "learning_rate": 2.207611695069794e-05,
+      "loss": 0.746,
+      "num_input_tokens_seen": 56623104,
+      "step": 36
+    },
+    {
+      "epoch": 0.5471349353049908,
+      "grad_norm": 0.07385499030351639,
+      "learning_rate": 2.0915565954536744e-05,
+      "loss": 0.7315,
+      "num_input_tokens_seen": 58195968,
+      "step": 37
+    },
+    {
+      "epoch": 0.5619223659889094,
+      "grad_norm": 0.06937970221042633,
+      "learning_rate": 1.9763993430846395e-05,
+      "loss": 0.7267,
+      "num_input_tokens_seen": 59768832,
+      "step": 38
+    },
+    {
+      "epoch": 0.5767097966728281,
+      "grad_norm": 0.0702456384897232,
+      "learning_rate": 1.8623930785958092e-05,
+      "loss": 0.7443,
+      "num_input_tokens_seen": 61341696,
+      "step": 39
+    },
+    {
+      "epoch": 0.5914972273567468,
+      "grad_norm": 0.0659838542342186,
+      "learning_rate": 1.749788412499149e-05,
+      "loss": 0.7163,
+      "num_input_tokens_seen": 62914560,
+      "step": 40
+    },
+    {
+      "epoch": 0.6062846580406654,
+      "grad_norm": 0.07357968389987946,
+      "learning_rate": 1.638832874289168e-05,
+      "loss": 0.73,
+      "num_input_tokens_seen": 64487424,
+      "step": 41
+    },
+    {
+      "epoch": 0.6210720887245841,
+      "grad_norm": 0.0689447820186615,
+      "learning_rate": 1.5297703683193752e-05,
+      "loss": 0.723,
+      "num_input_tokens_seen": 66060288,
+      "step": 42
+    },
+    {
+      "epoch": 0.6358595194085028,
+      "grad_norm": 0.0671798512339592,
+      "learning_rate": 1.4228406376475742e-05,
+      "loss": 0.7176,
+      "num_input_tokens_seen": 67633152,
+      "step": 43
+    },
+    {
+      "epoch": 0.6506469500924215,
+      "grad_norm": 0.06556376069784164,
+      "learning_rate": 1.3182787370285865e-05,
+      "loss": 0.7066,
+      "num_input_tokens_seen": 69206016,
+      "step": 44
+    },
+    {
+      "epoch": 0.6654343807763401,
+      "grad_norm": 0.07168299704790115,
+      "learning_rate": 1.2163145162128947e-05,
+      "loss": 0.7244,
+      "num_input_tokens_seen": 70778880,
+      "step": 45
+    },
+    {
+      "epoch": 0.6802218114602587,
+      "grad_norm": 0.06598961353302002,
+      "learning_rate": 1.1171721146870015e-05,
+      "loss": 0.7349,
+      "num_input_tokens_seen": 72351744,
+      "step": 46
+    },
+    {
+      "epoch": 0.6950092421441775,
+      "grad_norm": 0.06602618843317032,
+      "learning_rate": 1.021069468966194e-05,
+      "loss": 0.7487,
+      "num_input_tokens_seen": 73924608,
+      "step": 47
+    },
+    {
+      "epoch": 0.7097966728280961,
+      "grad_norm": 0.0713997408747673,
+      "learning_rate": 9.282178335227884e-06,
+      "loss": 0.7381,
+      "num_input_tokens_seen": 75497472,
+      "step": 48
+    },
+    {
+      "epoch": 0.7245841035120147,
+      "grad_norm": 0.06555724889039993,
+      "learning_rate": 8.38821316402946e-06,
+      "loss": 0.7262,
+      "num_input_tokens_seen": 77070336,
+      "step": 49
+    },
+    {
+      "epoch": 0.7393715341959335,
+      "grad_norm": 0.06813663244247437,
+      "learning_rate": 7.530764305528959e-06,
+      "loss": 0.7473,
+      "num_input_tokens_seen": 78643200,
+      "step": 50
+    },
+    {
+      "epoch": 0.7541589648798521,
+      "grad_norm": 0.06930514425039291,
+      "learning_rate": 6.711716618408281e-06,
+      "loss": 0.6998,
+      "num_input_tokens_seen": 80216064,
+      "step": 51
+    },
+    {
+      "epoch": 0.7689463955637708,
+      "grad_norm": 0.06492163240909576,
+      "learning_rate": 5.932870547240454e-06,
+      "loss": 0.7218,
+      "num_input_tokens_seen": 81788928,
+      "step": 52
+    },
+    {
+      "epoch": 0.7837338262476895,
+      "grad_norm": 0.07155918329954147,
+      "learning_rate": 5.1959381647217666e-06,
+      "loss": 0.7314,
+      "num_input_tokens_seen": 83361792,
+      "step": 53
+    },
+    {
+      "epoch": 0.7985212569316081,
+      "grad_norm": 0.06532897800207138,
+      "learning_rate": 4.502539408164386e-06,
+      "loss": 0.7028,
+      "num_input_tokens_seen": 84934656,
+      "step": 54
+    },
+    {
+      "epoch": 0.8133086876155268,
+      "grad_norm": 0.06727246940135956,
+      "learning_rate": 3.8541985185225645e-06,
+      "loss": 0.7084,
+      "num_input_tokens_seen": 86507520,
+      "step": 55
+    },
+    {
+      "epoch": 0.8280961182994455,
+      "grad_norm": 0.06698304414749146,
+      "learning_rate": 3.252340689780245e-06,
+      "loss": 0.7223,
+      "num_input_tokens_seen": 88080384,
+      "step": 56
+    },
+    {
+      "epoch": 0.8428835489833642,
+      "grad_norm": 0.06450291723012924,
+      "learning_rate": 2.6982889360653377e-06,
+      "loss": 0.7195,
+      "num_input_tokens_seen": 89653248,
+      "step": 57
+    },
+    {
+      "epoch": 0.8576709796672828,
+      "grad_norm": 0.06992805004119873,
+      "learning_rate": 2.1932611833775846e-06,
+      "loss": 0.7431,
+      "num_input_tokens_seen": 91226112,
+      "step": 58
+    },
+    {
+      "epoch": 0.8724584103512015,
+      "grad_norm": 0.06958083808422089,
+      "learning_rate": 1.738367592322837e-06,
+      "loss": 0.732,
+      "num_input_tokens_seen": 92798976,
+      "step": 59
+    },
+    {
+      "epoch": 0.8872458410351202,
+      "grad_norm": 0.0694640502333641,
+      "learning_rate": 1.3346081177391472e-06,
+      "loss": 0.7302,
+      "num_input_tokens_seen": 94371840,
+      "step": 60
+    },
+    {
+      "epoch": 0.9020332717190388,
+      "grad_norm": 0.07005713880062103,
+      "learning_rate": 9.828703105789983e-07,
+      "loss": 0.7197,
+      "num_input_tokens_seen": 95944704,
+      "step": 61
+    },
+    {
+      "epoch": 0.9168207024029574,
+      "grad_norm": 0.07030840963125229,
+      "learning_rate": 6.839273668796747e-07,
+      "loss": 0.7203,
+      "num_input_tokens_seen": 97517568,
+      "step": 62
+    },
+    {
+      "epoch": 0.9316081330868762,
+      "grad_norm": 0.0708225816488266,
+      "learning_rate": 4.3843642811059737e-07,
+      "loss": 0.7474,
+      "num_input_tokens_seen": 99090432,
+      "step": 63
+    },
+    {
+      "epoch": 0.9463955637707948,
+      "grad_norm": 0.0676749050617218,
+      "learning_rate": 2.4693713663372644e-07,
+      "loss": 0.7403,
+      "num_input_tokens_seen": 100663296,
+      "step": 64
+    },
+    {
+      "epoch": 0.9611829944547134,
+      "grad_norm": 0.06782912462949753,
+      "learning_rate": 1.0985044945254764e-07,
+      "loss": 0.7327,
+      "num_input_tokens_seen": 102236160,
+      "step": 65
+    },
+    {
+      "epoch": 0.9759704251386322,
+      "grad_norm": 0.06357243657112122,
+      "learning_rate": 2.7477712857215677e-08,
+      "loss": 0.725,
+      "num_input_tokens_seen": 103809024,
+      "step": 66
+    },
+    {
+      "epoch": 0.9907578558225508,
+      "grad_norm": 0.07199209183454514,
+      "learning_rate": 0.0,
+      "loss": 0.7209,
+      "num_input_tokens_seen": 105381888,
+      "step": 67
+    },
+    {
+      "epoch": 0.9907578558225508,
+      "num_input_tokens_seen": 105381888,
+      "step": 67,
+      "total_flos": 4.104162098269913e+18,
+      "train_loss": 0.8075656152483243,
+      "train_runtime": 10309.5741,
+      "train_samples_per_second": 2.518,
+      "train_steps_per_second": 0.006
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 67,
+  "num_input_tokens_seen": 105381888,
+  "num_train_epochs": 1,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4.104162098269913e+18,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3744866dbbc6f131cffa9f029f7c79843f64cdc0b6148cbf4456291d5471e5e3
+size 5688

training_args.yaml

@@ -0,0 +1,39 @@
+apollo_rank: 256
+apollo_scale: 1
+apollo_target: all
+apollo_update_interval: 200
+bf16: true
+cutoff_len: 4096
+dataset: codes_nsx_under8
+dataset_dir: data
+ddp_timeout: 180000000
+do_train: true
+enable_liger_kernel: true
+finetuning_type: freeze
+flash_attn: auto
+freeze_trainable_layers: 2
+freeze_trainable_modules: all
+gradient_accumulation_steps: 8
+include_num_input_tokens_seen: true
+learning_rate: 5.0e-05
+logging_steps: 1
+lr_scheduler_type: cosine
+max_grad_norm: 1.0
+max_samples: 50000000
+model_name_or_path: deepseek-ai/deepseek-coder-7b-instruct-v1.5
+neat_packing: true
+num_train_epochs: 1.0
+output_dir: saves/DeepSeek-Coder-7B-Instruct/freeze/deepseek_under8_nsx
+packing: true
+per_device_train_batch_size: 16
+plot_loss: true
+preprocessing_num_workers: 16
+report_to: none
+rope_scaling: llama3
+save_steps: 1000
+stage: sft
+template: deepseekcoder
+trust_remote_code: true
+use_apollo: true
+use_llama_pro: true
+warmup_steps: 0