Training in progress, step 120

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "HuggingFaceTB/SmolLM2-135M",
+  "_name_or_path": "HuggingFaceTB/SmolLM2-360M",
   "architectures": [
     "LlamaForCausalLM"
   ],
@@ -7,16 +7,18 @@
   "attention_dropout": 0.0,
   "bos_token_id": 0,
   "eos_token_id": 0,
+  "head_dim": 64,
   "hidden_act": "silu",
-  "hidden_size": 576,
-  "initializer_range": 0.041666666666666664,
-  "intermediate_size": 1536,
+  "hidden_size": 960,
+  "initializer_range": 0.02,
+  "intermediate_size": 2560,
   "is_llama_config": true,
   "max_position_embeddings": 8192,
+  "mlp_bias": false,
   "model_type": "llama",
-  "num_attention_heads": 9,
-  "num_hidden_layers": 30,
-  "num_key_value_heads": 3,
+  "num_attention_heads": 15,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 5,
   "pretraining_tp": 1,
   "rms_norm_eps": 1e-05,
   "rope_interleaved": false,
@@ -24,7 +26,7 @@
   "rope_theta": 100000,
   "tie_word_embeddings": true,
   "torch_dtype": "float32",
-  "transformers_version": "4.39.3",
+  "transformers_version": "4.46.2",
   "use_cache": true,
   "vocab_size": 49152
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6e82bfa45ce95a78b0f0abcc0248ec7f5bb617938d94c9579bb871c1d7a00fe3
-size 538090408
+oid sha256:7e52bfea8452cab61d95bea475831af3fbf67e86d2c86f0e1dcc94a129c827e1
+size 1447317080

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b673bdad4b4bd458f503a7d7be6cf83d3db8e9d42106875f0af15f6fb04cfa70
-size 4920
+oid sha256:e709baa5bc37e2fadc4915c500d2445e79ac8282fb8e08d9e07e05ac435d5f85
+size 5240