pico-decoder-large-1 trained to 50k steps

README.md

@@ -13,6 +13,8 @@ pipeline_tag: text-generation
 
 **pico-decoder-large** is the largest model (570M) in the current `pico-decoder` suite. It is a full-scale research model designed for in-depth interpretability studies of transformer learning. Trained with [`pico-train`](https://github.com/pico-lm) and fully compatible with [`pico-analyze`](https://github.com/pico-lm), it offers rich checkpointing and analytical insight into large-scale LM behavior.
 
+> NOTE: The `pico-decoder-large-1` branch contains the full commit history for the training run. 
+
 ## 🔧 Model Details
 
 | Field               | Value                              |

config.json

@@ -0,0 +1,22 @@
+{
+  "activation_hidden_dim": 6144,
+  "architectures": [
+    "PicoHF"
+  ],
+  "attention_n_heads": 12,
+  "attention_n_kv_heads": 4,
+  "auto_map": {
+    "AutoConfig": "pico.PicoHFConfig",
+    "AutoModelForCausalLM": "pico.PicoHF"
+  },
+  "batch_size": 1024,
+  "d_model": 1536,
+  "max_seq_len": 2048,
+  "model_type": "pico",
+  "n_layers": 12,
+  "norm_eps": 1e-06,
+  "position_emb_theta": 10000.0,
+  "torch_dtype": "float32",
+  "transformers_version": "4.48.1",
+  "vocab_size": 50304
+}

eval_results/step_0.json

@@ -0,0 +1 @@
+{"paloma": 60407.55679170296}

eval_results/step_1000.json

@@ -0,0 +1 @@
+{"paloma": 325.7754169842923}

eval_results/step_10000.json

@@ -0,0 +1 @@
+{"paloma": 39.63131210396929}

eval_results/step_11000.json

@@ -0,0 +1 @@
+{"paloma": 38.16542526977818}

eval_results/step_12000.json

@@ -0,0 +1 @@
+{"paloma": 37.00609017996838}

eval_results/step_13000.json

@@ -0,0 +1 @@
+{"paloma": 36.12258010915763}

eval_results/step_14000.json

@@ -0,0 +1 @@
+{"paloma": 35.30092575084873}

eval_results/step_15000.json

@@ -0,0 +1 @@
+{"paloma": 34.9831400311367}

eval_results/step_16000.json

@@ -0,0 +1 @@
+{"paloma": 33.944966741887534}

eval_results/step_17000.json

@@ -0,0 +1 @@
+{"paloma": 33.525249324692254}

eval_results/step_18000.json

@@ -0,0 +1 @@
+{"paloma": 33.06738855614479}

eval_results/step_19000.json

@@ -0,0 +1 @@
+{"paloma": 32.68154775822204}

eval_results/step_2000.json

@@ -0,0 +1 @@
+{"paloma": 893.744832137902}

eval_results/step_20000.json

@@ -0,0 +1 @@
+{"paloma": 32.365150609082875}

eval_results/step_21000.json

@@ -0,0 +1 @@
+{"paloma": 31.893736724238778}

eval_results/step_22000.json

@@ -0,0 +1 @@
+{"paloma": 31.542816529955182}

eval_results/step_23000.json

@@ -0,0 +1 @@
+{"paloma": 31.126356617498896}

eval_results/step_24000.json

@@ -0,0 +1 @@
+{"paloma": 30.837729862010438}

eval_results/step_25000.json

@@ -0,0 +1 @@
+{"paloma": 30.55359834328761}

eval_results/step_26000.json

@@ -0,0 +1 @@
+{"paloma": 30.223841104607132}

eval_results/step_27000.json

@@ -0,0 +1 @@
+{"paloma": 30.00440268965133}

eval_results/step_28000.json

@@ -0,0 +1 @@
+{"paloma": 29.728886223836227}

eval_results/step_29000.json

@@ -0,0 +1 @@
+{"paloma": 29.550568088920276}

eval_results/step_3000.json

@@ -0,0 +1 @@
+{"paloma": 86.66753888420942}

eval_results/step_30000.json

@@ -0,0 +1 @@
+{"paloma": 29.28618069741784}

eval_results/step_31000.json

@@ -0,0 +1 @@
+{"paloma": 29.13321706517648}

eval_results/step_32000.json

@@ -0,0 +1 @@
+{"paloma": 28.90883405690409}

eval_results/step_33000.json

@@ -0,0 +1 @@
+{"paloma": 28.689674922979666}

eval_results/step_34000.json

@@ -0,0 +1 @@
+{"paloma": 28.554471499828512}

eval_results/step_35000.json

@@ -0,0 +1 @@
+{"paloma": 28.37717256438026}

eval_results/step_36000.json

@@ -0,0 +1 @@
+{"paloma": 28.19421185649646}

eval_results/step_37000.json

@@ -0,0 +1 @@
+{"paloma": 28.05642763455155}

eval_results/step_38000.json

@@ -0,0 +1 @@
+{"paloma": 27.91100605555943}

eval_results/step_39000.json

@@ -0,0 +1 @@
+{"paloma": 27.77313469752202}

eval_results/step_4000.json

@@ -0,0 +1 @@
+{"paloma": 61.31575394249959}

eval_results/step_40000.json

@@ -0,0 +1 @@
+{"paloma": 27.60404353183321}

eval_results/step_41000.json

@@ -0,0 +1 @@
+{"paloma": 27.54711822325344}

eval_results/step_42000.json

@@ -0,0 +1 @@
+{"paloma": 27.389416195789696}

eval_results/step_43000.json

@@ -0,0 +1 @@
+{"paloma": 27.207264288792626}

eval_results/step_44000.json

@@ -0,0 +1 @@
+{"paloma": 27.154732140514493}

eval_results/step_45000.json

@@ -0,0 +1 @@
+{"paloma": 26.99035127520146}

eval_results/step_46000.json

@@ -0,0 +1 @@
+{"paloma": 26.9613054703337}

eval_results/step_47000.json

@@ -0,0 +1 @@
+{"paloma": 26.828856883730207}

eval_results/step_48000.json

@@ -0,0 +1 @@
+{"paloma": 26.728875775287374}

eval_results/step_49000.json

@@ -0,0 +1 @@
+{"paloma": 26.63958778913013}

eval_results/step_5000.json

@@ -0,0 +1 @@
+{"paloma": 56.45454279595551}

eval_results/step_50000.json

@@ -0,0 +1 @@
+{"paloma": 26.524442167398405}

eval_results/step_6000.json

@@ -0,0 +1 @@
+{"paloma": 49.864106077267316}